From 4eefad42e8650d1ffc1c8c29392fe75b44f39d6e Mon Sep 17 00:00:00 2001 From: subhamkrai Date: Tue, 10 Mar 2026 15:41:57 +0530 Subject: [PATCH] csi: move csi management to admin Going forward, admin will manage the csi operator CR's and rook will only manage Ceph Connection cr and client Profile cr. The old csi driver is completely removed from Rook and can no longer be used starting in Rook v1.20. The upgrade guide will contain the needed transition steps for managing the csi operator settings. Signed-off-by: subhamkrai --- .github/workflows/canary-integration-test.yml | 16 +- .../encryption-pvc-kms-ibm-kp/action.yml | 1 - .../Getting-Started/example-configurations.md | 2 +- Documentation/Getting-Started/quickstart.md | 7 +- Documentation/Helm-Charts/.pages | 1 + .../Helm-Charts/csi-drivers-chart.md | 22 + Documentation/Helm-Charts/helm-charts.md | 8 +- .../Helm-Charts/operator-chart.gotmpl.md | 1 + Documentation/Helm-Charts/operator-chart.md | 79 +- .../Ceph-CSI/custom-images.md | 58 +- .../NFS/nfs-csi-driver.md | 16 +- Makefile | 1 - PendingReleaseNotes.md | 5 +- build/rbac/gen-nfs-rbac.sh | 23 - build/rbac/get-helm-rbac.sh | 2 +- build/rbac/nfs-rbac.yaml.header | 3 - deploy/charts/rook-ceph/Chart.yaml | 2 +- deploy/charts/rook-ceph/templates/NOTES.txt | 2 +- .../rook-ceph/templates/clusterrole.yaml | 286 ------ .../templates/clusterrolebinding.yaml | 94 -- .../charts/rook-ceph/templates/configmap.yaml | 260 +---- deploy/charts/rook-ceph/templates/role.yaml | 75 -- .../rook-ceph/templates/rolebinding.yaml | 50 - .../templates/securityContextConstraints.yaml | 11 +- .../rook-ceph/templates/serviceaccount.yaml | 62 -- .../rook-ceph/templates/servicemonitor.yaml | 4 +- deploy/charts/rook-ceph/values.yaml | 471 +-------- deploy/examples/common.yaml | 490 --------- deploy/examples/csi/nfs/driver.yaml | 12 + deploy/examples/csi/nfs/rbac.yaml | 144 --- deploy/examples/images.txt | 24 +- deploy/examples/kustomization.yaml | 1 - deploy/examples/operator-openshift.yaml | 489 +-------- deploy/examples/operator.yaml | 565 ++-------- images/ceph/Makefile | 12 +- pkg/operator/ceph/cluster/cluster.go | 5 - pkg/operator/ceph/cluster/cluster_external.go | 33 +- pkg/operator/ceph/cluster/controller.go | 8 +- pkg/operator/ceph/cluster/mon/health_test.go | 20 +- pkg/operator/ceph/cluster/mon/mon.go | 17 +- pkg/operator/ceph/cluster/mon/mon_test.go | 16 +- pkg/operator/ceph/cluster/predicate.go | 23 +- pkg/operator/ceph/csi/ceph_config.go | 45 - pkg/operator/ceph/csi/ceph_config_test.go | 56 - pkg/operator/ceph/csi/cluster_config.go | 187 +--- pkg/operator/ceph/csi/cluster_config_test.go | 415 +------- pkg/operator/ceph/csi/config.go | 19 + pkg/operator/ceph/csi/controller.go | 133 +-- pkg/operator/ceph/csi/controller_test.go | 112 +- pkg/operator/ceph/csi/csi.go | 394 ------- pkg/operator/ceph/csi/csidriver.go | 118 --- pkg/operator/ceph/csi/operator_config.go | 181 ---- pkg/operator/ceph/csi/operator_config_test.go | 99 -- pkg/operator/ceph/csi/operator_driver.go | 485 --------- pkg/operator/ceph/csi/operator_driver_test.go | 233 ----- pkg/operator/ceph/csi/spec.go | 970 ------------------ pkg/operator/ceph/csi/spec_test.go | 117 --- .../csi-cephfsplugin-provisioner-dep.yaml | 341 ------ .../template/cephfs/csi-cephfsplugin-svc.yaml | 16 - .../csi/template/cephfs/csi-cephfsplugin.yaml | 242 ----- .../csi/template/csi-logrotate-sidecar.yaml | 41 - .../nfs/csi-nfsplugin-provisioner-dep.yaml | 247 ----- .../ceph/csi/template/nfs/csi-nfsplugin.yaml | 182 ---- .../rbd/csi-rbdplugin-provisioner-dep.yaml | 391 ------- .../csi/template/rbd/csi-rbdplugin-svc.yaml | 16 - .../ceph/csi/template/rbd/csi-rbdplugin.yaml | 315 ------ pkg/operator/ceph/csi/util.go | 278 ----- pkg/operator/ceph/csi/util_test.go | 314 ------ .../ceph/file/subvolumegroup/controller.go | 63 +- .../file/subvolumegroup/controller_test.go | 25 +- .../ceph/pool/radosnamespace/controller.go | 67 +- .../pool/radosnamespace/controller_test.go | 23 +- .../installer/ceph_helm_installer.go | 111 +- tests/framework/installer/ceph_installer.go | 35 +- tests/framework/installer/ceph_manifests.go | 5 - .../installer/ceph_manifests_previous.go | 4 - tests/framework/installer/ceph_settings.go | 5 +- tests/framework/utils/helm_helper.go | 32 + tests/framework/utils/snapshot.go | 16 +- tests/integration/ceph_base_block_test.go | 4 +- tests/integration/ceph_base_file_test.go | 5 +- tests/integration/ceph_upgrade_test.go | 1 - tests/scripts/create-dev-cluster.sh | 3 +- tests/scripts/github-action-helper.sh | 10 +- 84 files changed, 515 insertions(+), 9262 deletions(-) create mode 100644 Documentation/Helm-Charts/csi-drivers-chart.md delete mode 100755 build/rbac/gen-nfs-rbac.sh delete mode 100644 build/rbac/nfs-rbac.yaml.header create mode 100644 deploy/examples/csi/nfs/driver.yaml delete mode 100644 deploy/examples/csi/nfs/rbac.yaml delete mode 100644 pkg/operator/ceph/csi/ceph_config.go delete mode 100644 pkg/operator/ceph/csi/ceph_config_test.go delete mode 100644 pkg/operator/ceph/csi/csi.go delete mode 100644 pkg/operator/ceph/csi/csidriver.go delete mode 100644 pkg/operator/ceph/csi/operator_config.go delete mode 100644 pkg/operator/ceph/csi/operator_config_test.go delete mode 100644 pkg/operator/ceph/csi/operator_driver.go delete mode 100644 pkg/operator/ceph/csi/operator_driver_test.go delete mode 100644 pkg/operator/ceph/csi/spec.go delete mode 100644 pkg/operator/ceph/csi/spec_test.go delete mode 100644 pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml delete mode 100644 pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml delete mode 100644 pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml delete mode 100644 pkg/operator/ceph/csi/template/csi-logrotate-sidecar.yaml delete mode 100644 pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml delete mode 100644 pkg/operator/ceph/csi/template/nfs/csi-nfsplugin.yaml delete mode 100644 pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml delete mode 100644 pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml delete mode 100644 pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml delete mode 100644 pkg/operator/ceph/csi/util.go delete mode 100644 pkg/operator/ceph/csi/util_test.go diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index af93d951f..eb56ee951 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -901,7 +901,6 @@ jobs: run: | yq write -i deploy/examples/operator.yaml "data.ROOK_CEPH_ALLOW_LOOP_DEVICES" --style=double "true" tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 3 yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi @@ -936,7 +935,6 @@ jobs: - name: redeploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml @@ -1007,7 +1005,6 @@ jobs: - name: deploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml @@ -1068,7 +1065,6 @@ jobs: - name: deploy rook run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml @@ -1142,7 +1138,6 @@ jobs: - name: deploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml @@ -1174,7 +1169,6 @@ jobs: - name: redeploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml @@ -1246,7 +1240,6 @@ jobs: - name: deploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml @@ -1309,7 +1302,6 @@ jobs: - name: deploy rook run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml @@ -1387,7 +1379,6 @@ jobs: - name: deploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml cat tests/manifests/test-kms-vault.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 @@ -1485,7 +1476,6 @@ jobs: - name: deploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec-k8s-auth.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi @@ -1561,7 +1551,6 @@ jobs: - name: deploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml @@ -2022,7 +2011,6 @@ jobs: - name: deploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build "deploy/examples/operator.yaml" - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build "deploy/examples/csi-operator.yaml" tests/scripts/github-action-helper.sh deploy_manifest_with_local_build "deploy/examples/csi/nvmeof/driver.yaml" export BLOCK="$(tests/scripts/github-action-helper.sh find_extra_block_dev)" @@ -2135,9 +2123,7 @@ jobs: - name: deploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml - - + tests/scripts/github-action-helper.sh deploy_manifest_with_local_build "deploy/examples/csi/nfs/driver.yaml" export BLOCK="$(tests/scripts/github-action-helper.sh find_extra_block_dev)" yq write -i deploy/examples/cluster-multus-test.yaml "spec.storage.deviceFilter" "${BLOCK}" kubectl create -f deploy/examples/cluster-multus-test.yaml diff --git a/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml b/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml index b0a873342..3c2a4232a 100644 --- a/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml +++ b/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml @@ -57,7 +57,6 @@ runs: IBM_KP_SERVICE_API_KEY: ${{ inputs.ibm-service-api-key }} run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml envsubst < "tests/manifests/test-kms-ibm-kp-secret.in" > "tests/manifests/test-kms-ibm-kp-secret.yaml" envsubst < "tests/manifests/test-kms-ibm-kp-spec.in" > "tests/manifests/test-kms-ibm-kp-spec.yaml" cat tests/manifests/test-kms-ibm-kp-secret.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml diff --git a/Documentation/Getting-Started/example-configurations.md b/Documentation/Getting-Started/example-configurations.md index 1ea1a98e4..17055af2f 100644 --- a/Documentation/Getting-Started/example-configurations.md +++ b/Documentation/Getting-Started/example-configurations.md @@ -24,7 +24,7 @@ After the common resources are created, the next step is to create the Operator * [`operator.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/operator.yaml): The most common settings for production deployments * `kubectl create -f operator.yaml` -* [`csi-operator.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/csi-operator.yaml): Deploys the CSI operator, CRDs, and RBAC for the CSI driver (block and shared filesystem provisioning). Deploy alongside operator.yaml for CSI-based storage. +* [`csi-operator.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/csi-operator.yaml): Resources required for the Ceph-CSI operator, including RBAC, CRDs, and the deployment spec. * `kubectl create -f csi-operator.yaml` * [`operator-openshift.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/operator-openshift.yaml): Includes all of the operator settings for running a basic Rook cluster in an OpenShift environment. You will also want to review the [OpenShift Prerequisites](../Getting-Started/ceph-openshift.md) to confirm the settings. * `oc create -f operator-openshift.yaml` diff --git a/Documentation/Getting-Started/quickstart.md b/Documentation/Getting-Started/quickstart.md index 50ffd1f07..1a9607f3b 100644 --- a/Documentation/Getting-Started/quickstart.md +++ b/Documentation/Getting-Started/quickstart.md @@ -38,8 +38,8 @@ A simple Rook cluster is created for Kubernetes with the following `kubectl` com ```console $ git clone --single-branch --branch master https://github.com/rook/rook.git cd rook/deploy/examples -kubectl create -f crds.yaml -f common.yaml -f csi-operator.yaml -f operator.yaml -kubectl create -f cluster.yaml +kubectl create -f crds.yaml -f common.yaml -f csi-operator.yaml +kubectl create -f operator.yaml -f cluster.yaml ``` After the cluster is running, applications can consume [block, object, or file](#storage) storage. @@ -59,7 +59,8 @@ The first step is to deploy the Rook operator. ```console cd deploy/examples -kubectl create -f crds.yaml -f common.yaml -f csi-operator.yaml -f operator.yaml +kubectl create -f crds.yaml -f common.yaml -f csi-operator.yaml +kubectl create -f operator.yaml # verify the rook-ceph-operator is in the `Running` state before proceeding kubectl -n rook-ceph get pod diff --git a/Documentation/Helm-Charts/.pages b/Documentation/Helm-Charts/.pages index 17d4548ca..ff052bcb3 100644 --- a/Documentation/Helm-Charts/.pages +++ b/Documentation/Helm-Charts/.pages @@ -1,5 +1,6 @@ nav: - helm-charts.md - operator-chart.md + - csi-drivers-chart.md - ceph-cluster-chart.md - ... diff --git a/Documentation/Helm-Charts/csi-drivers-chart.md b/Documentation/Helm-Charts/csi-drivers-chart.md new file mode 100644 index 000000000..2190f0fa6 --- /dev/null +++ b/Documentation/Helm-Charts/csi-drivers-chart.md @@ -0,0 +1,22 @@ +--- +title: Ceph-CSI Driver Helm Chart +--- + +To configure the Ceph-CSI drivers, Rook requires the installation of the [Ceph-CSI Driver chart](https://github.com/ceph/ceph-csi-operator/blob/main/docs/helm-charts/drivers-chart.md). This chart configures the CSI drivers to provision and mount volumes to make available the Ceph storage to your applications. + +## Prerequisites + +* The `rook-ceph` chart must be installed before the Ceph-CSI drivers chart, to install the required Ceph-CSI operator and CRDs. + +## Installing + +The Ceph-CSI drivers Helm chart installs the resources needed for [ceph-csi](https://github.com/ceph/ceph-csi) to run under the ceph-csi-operator. + +The `helm install` command deploys the drivers in the default configuration from the chart. For more configuration options, see the [Ceph-CSI Drivers Configuration](https://github.com/ceph/ceph-csi-operator/blob/main/docs/helm-charts/drivers-chart.md#configuration). + +Ceph-CSI publishes the drivers chart from the `ceph-csi-operator` Helm repository. + +```console +helm repo add ceph-csi-operator https://ceph.github.io/ceph-csi-operator +helm install ceph-csi-drivers --namespace rook-ceph ceph-csi-operator/ceph-csi-drivers +``` diff --git a/Documentation/Helm-Charts/helm-charts.md b/Documentation/Helm-Charts/helm-charts.md index c430fd38c..29ca58606 100644 --- a/Documentation/Helm-Charts/helm-charts.md +++ b/Documentation/Helm-Charts/helm-charts.md @@ -2,10 +2,12 @@ title: Helm Charts Overview --- -Rook has published the following Helm charts for the Ceph storage provider: +The following charts are available to configure Ceph storage: + +1. [Rook Ceph Operator](operator-chart.md): Starts the Ceph Operator, which will watch for Ceph CRs (custom resources). Also installs the Ceph-CSI operator as a Helm dependency. +1. [Ceph-CSI drivers chart](csi-drivers-chart.md): Installs the Ceph-CSI drivers to provision and mount volumes. +1. [Rook Ceph Cluster](ceph-cluster-chart.md): Creates Ceph CRs that the operator will use to configure the cluster. -* [Rook Ceph Operator](operator-chart.md): Starts the Ceph Operator, which will watch for Ceph CRs (custom resources) -* [Rook Ceph Cluster](ceph-cluster-chart.md): Creates Ceph CRs that the operator will use to configure the cluster The Helm charts are intended to simplify deployment and upgrades. Configuring the Rook resources without Helm is also fully supported by creating the diff --git a/Documentation/Helm-Charts/operator-chart.gotmpl.md b/Documentation/Helm-Charts/operator-chart.gotmpl.md index 997d35e9d..748e31452 100644 --- a/Documentation/Helm-Charts/operator-chart.gotmpl.md +++ b/Documentation/Helm-Charts/operator-chart.gotmpl.md @@ -20,6 +20,7 @@ See the [Helm support matrix](https://helm.sh/docs/topics/version_skew/) for mor The Ceph Operator helm chart will install the basic components necessary to create a storage platform for your Kubernetes cluster. 1. Install the Helm chart +1. Install the [Ceph-CSI drivers chart](csi-drivers-chart.md) so CSI can provision and mount volumes 1. [Create a Rook cluster](../Getting-Started/quickstart.md#create-a-ceph-cluster). The `helm install` command deploys rook on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. It is recommended that the rook operator be installed into the `rook-ceph` namespace (you will install your clusters into separate namespaces). diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md index 5a6354c10..a95daf5b8 100644 --- a/Documentation/Helm-Charts/operator-chart.md +++ b/Documentation/Helm-Charts/operator-chart.md @@ -23,6 +23,7 @@ See the [Helm support matrix](https://helm.sh/docs/topics/version_skew/) for mor The Ceph Operator helm chart will install the basic components necessary to create a storage platform for your Kubernetes cluster. 1. Install the Helm chart +1. Install the [Ceph-CSI drivers chart](csi-drivers-chart.md) so CSI can provision and mount volumes 1. [Create a Rook cluster](../Getting-Started/quickstart.md#create-a-ceph-cluster). The `helm install` command deploys rook on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. It is recommended that the rook operator be installed into the `rook-ceph` namespace (you will install your clusters into separate namespaces). @@ -54,91 +55,23 @@ The following table lists the configurable parameters of the rook-operator chart | `crds.enabled` | Whether the helm chart should create and update the CRDs. If false, the CRDs must be managed independently with deploy/examples/crds.yaml. **WARNING** Only set during first deployment. If later disabled the cluster may be DESTROYED. If the CRDs are deleted in this case, see [the disaster recovery guide](https://rook.io/docs/rook/latest/Troubleshooting/disaster-recovery/#restoring-crds-after-deletion) to restore them. | `true` | | `csi.attacher.repository` | Kubernetes CSI Attacher image repository | `"registry.k8s.io/sig-storage/csi-attacher"` | | `csi.attacher.tag` | Attacher image tag | `"v4.11.0"` | -| `csi.cephFSAttachRequired` | Whether to skip any attach operation altogether for CephFS PVCs. See more details [here](https://kubernetes-csi.github.io/docs/skip-attach.html#skip-attach-with-csi-driver-object). If cephFSAttachRequired is set to false it skips the volume attachments and makes the creation of pods using the CephFS PVC fast. **WARNING** It's highly discouraged to use this for CephFS RWO volumes. Refer to this [issue](https://github.com/kubernetes/kubernetes/issues/103305) for more details. | `true` | -| `csi.cephFSFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted. supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html | `"File"` | -| `csi.cephFSKernelMountOptions` | Set CephFS Kernel mount options to use https://docs.ceph.com/en/latest/man/8/mount.ceph/#options. Set to "ms_mode=secure" when connections.encrypted is enabled in CephCluster CR | `nil` | -| `csi.cephFSPluginUpdateStrategy` | CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate | `RollingUpdate` | -| `csi.cephFSPluginUpdateStrategyMaxUnavailable` | A maxUnavailable parameter of CSI cephFS plugin daemonset update strategy. | `1` | | `csi.cephcsi.repository` | Ceph CSI image repository | `"quay.io/cephcsi/cephcsi"` | | `csi.cephcsi.tag` | Ceph CSI image tag | `"v3.16.2"` | -| `csi.cephfsLivenessMetricsPort` | CSI CephFS driver metrics port | `9081` | -| `csi.cephfsPodLabels` | Labels to add to the CSI CephFS Deployments and DaemonSets Pods | `nil` | -| `csi.clusterName` | Cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful in cases like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster | `nil` | -| `csi.crossNamespaceVolumeDataSource.enabled` | Enable cross namespace volume data source provisioning | `false` | -| `csi.csiAddons.enabled` | Enable CSIAddons | `false` | | `csi.csiAddons.repository` | CSIAddons sidecar image repository | `"quay.io/csiaddons/k8s-sidecar"` | | `csi.csiAddons.tag` | CSIAddons sidecar image tag | `"v0.14.0"` | -| `csi.csiAddonsCephFSProvisionerPort` | CSI Addons server port for the Ceph FS provisioner | `9070` | -| `csi.csiAddonsPort` | CSI Addons server port | `9070` | -| `csi.csiAddonsRBDProvisionerPort` | CSI Addons server port for the RBD provisioner | `9070` | -| `csi.csiCephFSPluginResource` | CEPH CSI CephFS plugin resource requirement list | see values.yaml | -| `csi.csiCephFSPluginVolume` | The volume of the CephCSI CephFS plugin DaemonSet | `nil` | -| `csi.csiCephFSPluginVolumeMount` | The volume mounts of the CephCSI CephFS plugin DaemonSet | `nil` | -| `csi.csiCephFSProvisionerResource` | CEPH CSI CephFS provisioner resource requirement list | see values.yaml | -| `csi.csiDriverNamePrefix` | CSI driver name prefix for cephfs, rbd and nfs. | `namespace name where rook-ceph operator is deployed` | -| `csi.csiLeaderElectionLeaseDuration` | Duration in seconds that non-leader candidates will wait to force acquire leadership. | `137s` | -| `csi.csiLeaderElectionRenewDeadline` | Deadline in seconds that the acting leader will retry refreshing leadership before giving up. | `107s` | -| `csi.csiLeaderElectionRetryPeriod` | Retry period in seconds the LeaderElector clients should wait between tries of actions. | `26s` | -| `csi.csiNFSPluginResource` | CEPH CSI NFS plugin resource requirement list | see values.yaml | -| `csi.csiNFSProvisionerResource` | CEPH CSI NFS provisioner resource requirement list | see values.yaml | -| `csi.csiRBDPluginResource` | CEPH CSI RBD plugin resource requirement list | see values.yaml | -| `csi.csiRBDPluginVolume` | The volume of the CephCSI RBD plugin DaemonSet | `nil` | -| `csi.csiRBDPluginVolumeMount` | The volume mounts of the CephCSI RBD plugin DaemonSet | `nil` | -| `csi.csiRBDProvisionerResource` | CEPH CSI RBD provisioner resource requirement list csi-omap-generator resources will be applied only if `enableOMAPGenerator` is set to `true` | see values.yaml | -| `csi.disableCsiDriver` | Disable the CSI driver. | `"false"` | -| `csi.enableCSIEncryption` | Enable Ceph CSI PVC encryption support | `false` | -| `csi.enableCSIHostNetwork` | Enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary in some network configurations where the SDN does not provide access to an external cluster or there is significant drop in read/write performance | `true` | -| `csi.enableCephfsDriver` | Enable Ceph CSI CephFS driver | `true` | -| `csi.enableCephfsSnapshotter` | Enable Snapshotter in CephFS provisioner pod | `true` | -| `csi.enableLiveness` | Enable Ceph CSI Liveness sidecar deployment | `false` | -| `csi.enableMetadata` | Enable adding volume metadata on the CephFS subvolumes and RBD images. Not all users might be interested in getting volume/snapshot details as metadata on CephFS subvolume and RBD images. Hence enable metadata is false by default | `false` | -| `csi.enableNFSSnapshotter` | Enable Snapshotter in NFS provisioner pod | `true` | -| `csi.enableOMAPGenerator` | OMAP generator generates the omap mapping between the PV name and the RBD image which helps CSI to identify the rbd images for CSI operations. `CSI_ENABLE_OMAP_GENERATOR` needs to be enabled when we are using rbd mirroring feature. By default OMAP generator is disabled and when enabled, it will be deployed as a sidecar with CSI provisioner pod, to enable set it to true. | `false` | -| `csi.enablePluginSelinuxHostMount` | Enable Host mount for `/etc/selinux` directory for Ceph CSI nodeplugins | `false` | -| `csi.enableRBDSnapshotter` | Enable Snapshotter in RBD provisioner pod | `true` | -| `csi.enableRbdDriver` | Enable Ceph CSI RBD driver | `true` | -| `csi.enableVolumeGroupSnapshot` | Enable volume group snapshot feature. This feature is enabled by default as long as the necessary CRDs are available in the cluster. | `true` | -| `csi.forceCephFSKernelClient` | Enable Ceph Kernel clients on kernel < 4.17. If your kernel does not support quotas for CephFS you may want to disable this setting. However, this will cause an issue during upgrades with the FUSE client. See the [upgrade guide](https://rook.io/docs/rook/v1.2/ceph-upgrade.html) | `true` | -| `csi.grpcTimeoutInSeconds` | Set GRPC timeout for csi containers (in seconds). It should be >= 120. If this value is not set or is invalid, it defaults to 150 | `150` | -| `csi.imagePullPolicy` | Image pull policy | `"IfNotPresent"` | -| `csi.kubeApiBurst` | Burst to use while communicating with the kubernetes apiserver. | `nil` | -| `csi.kubeApiQPS` | QPS to use while communicating with the kubernetes apiserver. | `nil` | -| `csi.kubeletDirPath` | Kubelet root directory path (if the Kubelet uses a different path for the `--root-dir` flag) | `/var/lib/kubelet` | -| `csi.logLevel` | Set logging level for cephCSI containers maintained by the cephCSI. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. | `0` | -| `csi.nfs.enabled` | Enable the nfs csi driver | `false` | -| `csi.nfsAttachRequired` | Whether to skip any attach operation altogether for NFS PVCs. See more details [here](https://kubernetes-csi.github.io/docs/skip-attach.html#skip-attach-with-csi-driver-object). If cephFSAttachRequired is set to false it skips the volume attachments and makes the creation of pods using the NFS PVC fast. **WARNING** It's highly discouraged to use this for NFS RWO volumes. Refer to this [issue](https://github.com/kubernetes/kubernetes/issues/103305) for more details. | `true` | -| `csi.nfsFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted. supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html | `"File"` | -| `csi.nfsPluginUpdateStrategy` | CSI NFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate | `RollingUpdate` | -| `csi.nfsPodLabels` | Labels to add to the CSI NFS Deployments and DaemonSets Pods | `nil` | -| `csi.pluginNodeAffinity` | The node labels for affinity of the CephCSI RBD plugin DaemonSet [^1] | `nil` | -| `csi.pluginPriorityClassName` | PriorityClassName to be set on csi driver plugin pods | `"system-node-critical"` | -| `csi.pluginTolerations` | Array of tolerations in YAML format which will be added to CephCSI plugin DaemonSet | `nil` | +| `csi.installCsiOperator` | When true, install the ceph-csi-operator subchart (see Chart.yaml `condition`). | `true` | | `csi.provisioner.repository` | Kubernetes CSI provisioner image repository | `"registry.k8s.io/sig-storage/csi-provisioner"` | | `csi.provisioner.tag` | Provisioner image tag | `"v6.1.1"` | -| `csi.provisionerNodeAffinity` | The node labels for affinity of the CSI provisioner deployment [^1] | `nil` | -| `csi.provisionerPriorityClassName` | PriorityClassName to be set on csi driver provisioner pods | `"system-cluster-critical"` | -| `csi.provisionerReplicas` | Set replicas for csi provisioner deployment | `2` | -| `csi.provisionerTolerations` | Array of tolerations in YAML format which will be added to CSI provisioner deployment | `nil` | -| `csi.rbdAttachRequired` | Whether to skip any attach operation altogether for RBD PVCs. See more details [here](https://kubernetes-csi.github.io/docs/skip-attach.html#skip-attach-with-csi-driver-object). If set to false it skips the volume attachments and makes the creation of pods using the RBD PVC fast. **WARNING** It's highly discouraged to use this for RWO volumes as it can cause data corruption. csi-addons operations like Reclaimspace and PVC Keyrotation will also not be supported if set to false since we'll have no VolumeAttachments to determine which node the PVC is mounted on. Refer to this [issue](https://github.com/kubernetes/kubernetes/issues/103305) for more details. | `true` | -| `csi.rbdFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted. supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html | `"File"` | -| `csi.rbdLivenessMetricsPort` | Ceph CSI RBD driver metrics port | `8080` | -| `csi.rbdPluginUpdateStrategy` | CSI RBD plugin daemonset update strategy, supported values are OnDelete and RollingUpdate | `RollingUpdate` | -| `csi.rbdPluginUpdateStrategyMaxUnavailable` | A maxUnavailable parameter of CSI RBD plugin daemonset update strategy. | `1` | -| `csi.rbdPodLabels` | Labels to add to the CSI RBD Deployments and DaemonSets Pods | `nil` | | `csi.registrar.repository` | Kubernetes CSI registrar image repository | `"registry.k8s.io/sig-storage/csi-node-driver-registrar"` | | `csi.registrar.tag` | Registrar image tag | `"v2.16.0"` | | `csi.resizer.repository` | Kubernetes CSI resizer image repository | `"registry.k8s.io/sig-storage/csi-resizer"` | | `csi.resizer.tag` | Resizer image tag | `"v2.1.0"` | -| `csi.rookUseCsiOperator` | | `true` | -| `csi.serviceMonitor.enabled` | Enable ServiceMonitor for Ceph CSI drivers | `false` | -| `csi.serviceMonitor.interval` | Service monitor scrape interval | `"10s"` | -| `csi.serviceMonitor.labels` | ServiceMonitor additional labels | `{}` | -| `csi.serviceMonitor.namespace` | Use a different namespace for the ServiceMonitor | `nil` | -| `csi.sidecarLogLevel` | Set logging level for Kubernetes-csi sidecar containers. Supported values from 0 to 5. 0 for general useful logs (the default), 5 for trace level verbosity. | `0` | +| `csi.serviceMonitor.enabled` | Enable ServiceMonitor for CSI metrics | `false` | +| `csi.serviceMonitor.interval` | Interval at which metrics should be scraped | `"5s"` | +| `csi.serviceMonitor.labels` | Additional labels for the ServiceMonitor | `{}` | +| `csi.serviceMonitor.namespace` | Namespace in which to deploy the ServiceMonitor | the release namespace | | `csi.snapshotter.repository` | Kubernetes CSI snapshotter image repository | `"registry.k8s.io/sig-storage/csi-snapshotter"` | | `csi.snapshotter.tag` | Snapshotter image tag | `"v8.5.0"` | -| `csi.topology.domainLabels` | domainLabels define which node labels to use as domains for CSI nodeplugins to advertise their domains | `nil` | -| `csi.topology.enabled` | Enable topology based provisioning | `false` | | `currentNamespaceOnly` | Whether the operator should watch cluster CRD in its own namespace or not | `false` | | `customHostnameLabel` | Custom label to identify node hostname. If not set `kubernetes.io/hostname` will be used | `nil` | | `deleteUnusedCrushRules` | If true, delete unused generated CRUSH rules after the mgr starts | `true` | diff --git a/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md b/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md index c7a144166..73b41dadf 100644 --- a/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md +++ b/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md @@ -2,51 +2,49 @@ title: Custom Images --- -By default, Rook will deploy the latest stable version of the Ceph CSI driver. -Commonly, there is no need to change this default version that is deployed. -For scenarios that require deploying a custom image (e.g. downstream releases), -the defaults can be overridden with the following settings. - -The CSI configuration variables are found in the `rook-ceph-operator-config` ConfigMap. -These settings can also be specified as environment variables on the operator deployment, though -the configmap values will override the env vars if both are specified. +CSI drivers are now managed by the [ceph-csi-operator](https://github.com/ceph/ceph-csi-operator). +The admin configures CSI container images through an `ImageSet` ConfigMap that is +referenced by the `OperatorConfig` CR. By default, Rook's Helm chart creates this +ConfigMap with the latest stable versions. Commonly, there is no need to change +the defaults. For scenarios that require custom images (e.g. downstream releases), +edit the `ImageSet` ConfigMap directly. ```console -kubectl -n $ROOK_OPERATOR_NAMESPACE edit configmap rook-ceph-operator-config +kubectl -n $ROOK_OPERATOR_NAMESPACE edit configmap rook-csi-operator-image-set-configmap ``` -The default upstream images are included below, which you can change to your desired images. +The default upstream images are included below, which can be customized to the desired images. ```yaml -ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.16.2" -ROOK_CSI_REGISTRAR_IMAGE: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.16.0" -ROOK_CSI_PROVISIONER_IMAGE: "registry.k8s.io/sig-storage/csi-provisioner:v6.1.1" -ROOK_CSI_ATTACHER_IMAGE: "registry.k8s.io/sig-storage/csi-attacher:v4.11.0" -ROOK_CSI_RESIZER_IMAGE: "registry.k8s.io/sig-storage/csi-resizer:v2.1.0" -ROOK_CSI_SNAPSHOTTER_IMAGE: "registry.k8s.io/sig-storage/csi-snapshotter:v8.5.0" -ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.14.0" +plugin: "quay.io/cephcsi/cephcsi:v3.16.2" +provisioner: "registry.k8s.io/sig-storage/csi-provisioner:v6.1.1" +attacher: "registry.k8s.io/sig-storage/csi-attacher:v4.11.0" +resizer: "registry.k8s.io/sig-storage/csi-resizer:v2.1.0" +snapshotter: "registry.k8s.io/sig-storage/csi-snapshotter:v8.5.0" +registrar: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.16.0" +addons: "quay.io/csiaddons/k8s-sidecar:v0.14.0" ``` -### **Use private repository** - -If image version is not passed along with the image name in any of the variables above, -Rook will add the corresponding default version to that image. -Example: if `ROOK_CSI_CEPH_IMAGE: "quay.io/private-repo/cephcsi"` is passed, -Rook will add internal default version and consume it as `"quay.io/private-repo/cephcsi:v3.12.0"`. +When using Helm, the images are configured under the `csi` section of `values.yaml` +(e.g. `csi.cephcsi.repository`, `csi.cephcsi.tag`) and are rendered into the +`ImageSet` ConfigMap automatically. ### **Use default images** -If you would like Rook to use the default upstream images, then you may simply remove all -variables matching `ROOK_CSI_*_IMAGE` from the above ConfigMap and/or the operator deployment. +To use the default upstream images, leave the `ImageSet` ConfigMap +unchanged or, for Helm installs, do not override any `csi.*` image values. ### **Verifying updates** -You can use the below command to see the CSI images currently being used in the cluster. Note that -not all images (like `volumereplication-operator`) may be present in every cluster depending on -which CSI features are enabled. +Use the below command to see the CSI images currently being used in the cluster. +Not all images may be present depending on which CSI features are enabled. ```console -kubectl --namespace rook-ceph get pod -o jsonpath='{range .items[*]}{range .spec.containers[*]}{.image}{"\n"}' -l 'app in (csi-rbdplugin,csi-rbdplugin-provisioner,csi-cephfsplugin,csi-cephfsplugin-provisioner)' | sort | uniq +kubectl --namespace rook-ceph get pod -o jsonpath='{range .items[*]}{range .spec.containers[*]}{.image}{"\n"}' -l 'app.kubernetes.io/part-of in (ceph-csi-rbd, ceph-csi-cephfs, ceph-csi-nfs)' | sort | uniq ``` -The default images can also be found with each release in the [images list](https://github.com/rook/rook/blob/master/deploy/examples/images.txt) +To inspect the `ImageSet` ConfigMap directly: + +```console +kubectl -n $ROOK_OPERATOR_NAMESPACE get configmap rook-csi-operator-image-set-configmap -o yaml +``` diff --git a/Documentation/Storage-Configuration/NFS/nfs-csi-driver.md b/Documentation/Storage-Configuration/NFS/nfs-csi-driver.md index ec38c4db7..c22cdabf1 100644 --- a/Documentation/Storage-Configuration/NFS/nfs-csi-driver.md +++ b/Documentation/Storage-Configuration/NFS/nfs-csi-driver.md @@ -10,20 +10,8 @@ For this section, we will refer to Rook's deployment examples in the ## Enabling the CSI drivers -The Ceph CSI NFS provisioner and driver require additional RBAC to operate. Apply the -`deploy/examples/csi/nfs/rbac.yaml` manifest to deploy the additional resources. - -Rook will only deploy the Ceph CSI NFS provisioner and driver components when the -`ROOK_CSI_ENABLE_NFS` config is set to `"true"` in the `rook-ceph-operator-config` configmap. Change -the value in your manifest, or patch the resource as below. - -```console -kubectl --namespace rook-ceph patch configmap rook-ceph-operator-config --type merge --patch '{"data":{"ROOK_CSI_ENABLE_NFS": "true"}}' -``` - -!!! note - The rook-ceph operator Helm chart will deploy the required RBAC and enable the driver - components if `csi.nfs.enabled` is set to `true`. +The NFS CSI driver is managed by the ceph-csi-operator. The required RBAC is included in +`deploy/examples/csi-operator.yaml` and the NFS Driver CR is defined in `deploy/examples/csi/nfs/driver.yaml`. ## Creating NFS exports via PVC diff --git a/Makefile b/Makefile index 98844e826..5bcbe82f0 100644 --- a/Makefile +++ b/Makefile @@ -281,7 +281,6 @@ gen.rbac: gen-rbac gen-rbac: $(HELM) $(YQ) helm.dependency.build ## Generate RBAC from Helm charts @# output only stdout to the file; stderr for debugging should keep going to stderr HELM=$(HELM) ./build/rbac/gen-common.sh - HELM=$(HELM) ./build/rbac/gen-nfs-rbac.sh .PHONY: gen.docs gen.docs: docs ## generate docs diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 92e7c2b99..14dc9232d 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -3,7 +3,10 @@ ## Breaking Changes - The minimum supported Kubernetes version is v1.31. - +- The CSI operator is required for managing CSI driver settings. + - The CSI settings are removed from the Rook operator configmap and helm chart + - New installs must configure the CSI settings with the CSI CRs instead of Rook operator settings. + - Upgrades will continue working with the existing settings that had been applied by Rook previously. Further updates to CSI settings will need to be updated by the Rook admin. ## Features diff --git a/build/rbac/gen-nfs-rbac.sh b/build/rbac/gen-nfs-rbac.sh deleted file mode 100755 index 4d1c55e4c..000000000 --- a/build/rbac/gen-nfs-rbac.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -set -xeEuo pipefail - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -pushd "$SCRIPT_DIR" &>/dev/stderr - -NFS_RBAC_YAML_FILE="$SCRIPT_DIR/../../deploy/examples/csi/nfs/rbac.yaml" - -tmpdir="$(mktemp -d)" -WITHOUT_FILE="${tmpdir}"/without-nfs.yaml # intermediate file of yaml that doesn't include NFS RBAC -WITH_FILE="${tmpdir}"/with-nfs.yaml # intermediate file of yaml that includes previous plus NFS RBAC - -./get-helm-rbac.sh > "$WITHOUT_FILE" - -export ADDITIONAL_HELM_CLI_OPTIONS="--set csi.nfs.enabled=true" -./get-helm-rbac.sh > "$WITH_FILE" - -rm -f "$NFS_RBAC_YAML_FILE" -cat nfs-rbac.yaml.header >> "$NFS_RBAC_YAML_FILE" -./keep-added.sh "$WITHOUT_FILE" "$WITH_FILE" >> "$NFS_RBAC_YAML_FILE" - -rm -rf "$tmpdir" -popd &>/dev/stderr diff --git a/build/rbac/get-helm-rbac.sh b/build/rbac/get-helm-rbac.sh index 45b9b7bb5..04f3c8872 100755 --- a/build/rbac/get-helm-rbac.sh +++ b/build/rbac/get-helm-rbac.sh @@ -19,7 +19,7 @@ options=( --namespace rook-ceph --set crds.enabled=false --set csi.csiAddons.enabled=true - --set csi.rookUseCsiOperator=false + --set csi.installCsiOperator=false ) for option in ${ADDITIONAL_HELM_CLI_OPTIONS}; do diff --git a/build/rbac/nfs-rbac.yaml.header b/build/rbac/nfs-rbac.yaml.header deleted file mode 100644 index 618c53cff..000000000 --- a/build/rbac/nfs-rbac.yaml.header +++ /dev/null @@ -1,3 +0,0 @@ -#################################################################################################### -# These RBAC resources are required to allow Rook to run the NFS export CSI driver components. -#################################################################################################### diff --git a/deploy/charts/rook-ceph/Chart.yaml b/deploy/charts/rook-ceph/Chart.yaml index 72f8fa70c..7e6aa46a6 100644 --- a/deploy/charts/rook-ceph/Chart.yaml +++ b/deploy/charts/rook-ceph/Chart.yaml @@ -15,4 +15,4 @@ dependencies: version: 0.6.0 repository: https://ceph.github.io/ceph-csi-operator alias: ceph-csi-operator - condition: csi.rookUseCsiOperator + condition: csi.installCsiOperator diff --git a/deploy/charts/rook-ceph/templates/NOTES.txt b/deploy/charts/rook-ceph/templates/NOTES.txt index 9eeb4a8ca..f236ad683 100644 --- a/deploy/charts/rook-ceph/templates/NOTES.txt +++ b/deploy/charts/rook-ceph/templates/NOTES.txt @@ -8,6 +8,6 @@ Important Notes: - The sample manifests assume you also installed the rook-ceph operator in the `rook-ceph` namespace. - The helm chart includes all the RBAC required to create a CephCluster CRD in the same namespace. - Any disk devices you add to the cluster in the 'CephCluster' must be empty (no filesystem and no partitions). -{{- if .Values.csi.rookUseCsiOperator }} +{{- if .Values.csi.installCsiOperator }} - The CSI operator will manage the CSI driver lifecycle for RBD, CephFS, and NFS drivers. {{- end }} diff --git a/deploy/charts/rook-ceph/templates/clusterrole.yaml b/deploy/charts/rook-ceph/templates/clusterrole.yaml index 7a0356938..6a5274664 100644 --- a/deploy/charts/rook-ceph/templates/clusterrole.yaml +++ b/deploy/charts/rook-ceph/templates/clusterrole.yaml @@ -412,292 +412,6 @@ rules: --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts/token"] - verbs: ["create"] -{{- if .Values.csi.nfs.enabled }} ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ceph-nfs-external-provisioner-runner - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "patch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] - - apiGroups: ["events.k8s.io"] - resources: ["events"] - verbs: ["create", "patch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] ---- -# TODO: remove this, once https://github.com/rook/rook/issues/10141 -# is resolved. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ceph-nfs-csi-nodeplugin - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get"] -{{- end }} ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-provisioner-runner - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "patch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["events.k8s.io"] - resources: ["events"] - verbs: ["create", "patch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list", "watch", "patch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update", "patch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotcontents"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotcontents/status"] - verbs: ["update", "patch"] - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts/token"] - verbs: ["create"] - - apiGroups: ["authentication.k8s.io"] - resources: ["tokenreviews"] - verbs: ["create"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts/token"] - verbs: ["create"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get"] - - apiGroups: ["authentication.k8s.io"] - resources: ["tokenreviews"] - verbs: ["create"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-provisioner-runner - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["events.k8s.io"] - resources: ["events"] - verbs: ["create", "patch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list", "watch", "patch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update", "patch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotcontents"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotcontents/status"] - verbs: ["update", "patch"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts/token"] - verbs: ["create"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["gateway.networking.k8s.io"] - resources: ["referencegrants"] - verbs: ["get", "list", "watch"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumegroupreplicationcontents"] - verbs: ["get", "list", "watch"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumegroupreplicationclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["authentication.k8s.io"] - resources: ["tokenreviews"] - verbs: ["create"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 metadata: name: objectstorage-provisioner-role labels: diff --git a/deploy/charts/rook-ceph/templates/clusterrolebinding.yaml b/deploy/charts/rook-ceph/templates/clusterrolebinding.yaml index 10c32fdd7..0e90c264b 100644 --- a/deploy/charts/rook-ceph/templates/clusterrolebinding.yaml +++ b/deploy/charts/rook-ceph/templates/clusterrolebinding.yaml @@ -47,100 +47,6 @@ subjects: name: rook-ceph-system namespace: {{ .Release.Namespace }} # namespace:operator --- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-plugin-sa - namespace: {{ .Release.Namespace }} # namespace:operator -roleRef: - kind: ClusterRole - name: rbd-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-provisioner-role - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: {{ .Release.Namespace }} # namespace:operator -roleRef: - kind: ClusterRole - name: cephfs-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin-role - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-plugin-sa - namespace: {{ .Release.Namespace }} # namespace:operator -roleRef: - kind: ClusterRole - name: cephfs-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io -{{- if .Values.csi.nfs.enabled }} ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ceph-nfs-csi-provisioner-role - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -subjects: - - kind: ServiceAccount - name: rook-csi-nfs-provisioner-sa - namespace: {{ .Release.Namespace }} # namespace:operator -roleRef: - kind: ClusterRole - name: ceph-nfs-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- -# TODO: remove this, once https://github.com/rook/rook/issues/10141 -# is resolved. -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ceph-nfs-csi-nodeplugin-role - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -subjects: - - kind: ServiceAccount - name: rook-csi-nfs-plugin-sa - namespace: {{ .Release.Namespace }} # namespace:operator -roleRef: - kind: ClusterRole - name: ceph-nfs-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io -{{- end }} ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-provisioner-role - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: {{ .Release.Namespace }} # namespace:operator -roleRef: - kind: ClusterRole - name: rbd-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- # RBAC for ceph cosi driver service account kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml index fcf9f1583..31bda4d0e 100644 --- a/deploy/charts/rook-ceph/templates/configmap.yaml +++ b/deploy/charts/rook-ceph/templates/configmap.yaml @@ -33,250 +33,50 @@ data: {{- with .Values.enforceHostNetwork }} ROOK_ENFORCE_HOST_NETWORK: {{ . | quote }} {{- end }} - - {{- with .Values.csi }} - ROOK_USE_CSI_OPERATOR: {{ .rookUseCsiOperator | quote }} - ROOK_CSI_ENABLE_RBD: {{ .enableRbdDriver | quote }} - ROOK_CSI_ENABLE_CEPHFS: {{ .enableCephfsDriver | quote }} - ROOK_CSI_DISABLE_DRIVER: {{ .disableCsiDriver | quote }} - CSI_ENABLE_CEPHFS_SNAPSHOTTER: {{ .enableCephfsSnapshotter | quote }} - CSI_ENABLE_NFS_SNAPSHOTTER: {{ .enableNFSSnapshotter | quote }} - CSI_ENABLE_RBD_SNAPSHOTTER: {{ .enableRBDSnapshotter | quote }} - CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT: {{ .enablePluginSelinuxHostMount | quote }} - CSI_ENABLE_ENCRYPTION: {{ .enableCSIEncryption | quote }} - CSI_ENABLE_OMAP_GENERATOR: {{ .enableOMAPGenerator | quote }} - CSI_ENABLE_HOST_NETWORK: {{ .enableCSIHostNetwork | quote }} - CSI_ENABLE_METADATA: {{ .enableMetadata | quote }} - CSI_ENABLE_VOLUME_GROUP_SNAPSHOT: {{ .enableVolumeGroupSnapshot | quote }} - {{- with .csiDriverNamePrefix }} - CSI_DRIVER_NAME_PREFIX: {{ . | quote }} - {{- end }} - {{- with .pluginPriorityClassName }} - CSI_PLUGIN_PRIORITY_CLASSNAME: {{ . | quote }} - {{- end }} - {{- with .provisionerPriorityClassName }} - CSI_PROVISIONER_PRIORITY_CLASSNAME: {{ . | quote }} - {{- end }} - {{- with .cephFSPluginUpdateStrategy }} - CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY: {{ . | quote }} - {{- end }} - {{- with .cephFSPluginUpdateStrategyMaxUnavailable }} - CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE: {{ . | quote }} - {{- end }} - {{- with .nfsPluginUpdateStrategy }} - CSI_NFS_PLUGIN_UPDATE_STRATEGY: {{ . | quote }} - {{- end }} - {{- with .rbdFSGroupPolicy }} - CSI_RBD_FSGROUPPOLICY: {{ . | quote }} - {{- end }} - {{- with .cephFSFSGroupPolicy }} - CSI_CEPHFS_FSGROUPPOLICY: {{ . | quote }} - {{- end }} - {{- with .nfsFSGroupPolicy }} - CSI_NFS_FSGROUPPOLICY: {{ . | quote }} - {{- end }} - {{- with .rbdPluginUpdateStrategy }} - CSI_RBD_PLUGIN_UPDATE_STRATEGY: {{ . | quote }} - {{- end }} - {{- with .cephFSKernelMountOptions }} - CSI_CEPHFS_KERNEL_MOUNT_OPTIONS: {{ . | quote }} - {{- end }} - {{- with .rbdPluginUpdateStrategyMaxUnavailable }} - CSI_RBD_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE: {{ . | quote }} - {{- end }} - {{- with .kubeletDirPath }} - ROOK_CSI_KUBELET_DIR_PATH: {{ . | quote }} - {{- end }} - {{- with .csiLeaderElectionLeaseDuration }} - CSI_LEADER_ELECTION_LEASE_DURATION: {{ . | quote }} - {{- end }} - {{- with .csiLeaderElectionRenewDeadline }} - CSI_LEADER_ELECTION_RENEW_DEADLINE: {{ . | quote }} - {{- end }} - {{- with .csiLeaderElectionRetryPeriod }} - CSI_LEADER_ELECTION_RETRY_PERIOD: {{ . | quote }} - {{- end }} - {{- with .cephcsi }} - {{- if and .repository .tag }} - ROOK_CSI_CEPH_IMAGE: "{{ .repository }}:{{ .tag }}" - {{- end }} - {{- end }} - {{- with .registrar }} - {{- if and .repository .tag }} - ROOK_CSI_REGISTRAR_IMAGE: "{{ .repository }}:{{ .tag }}" - {{- end }} - {{- end }} +{{- with .Values.csi }} +--- +# ImageSet ConfigMap defines the container images used by the CSI drivers. +apiVersion: v1 +kind: ConfigMap +metadata: + name: rook-csi-operator-image-set-configmap + namespace: {{ $.Release.Namespace }} # namespace:operator + labels: + {{- include "library.rook-ceph.labels" $ | nindent 4 }} +data: {{- with .provisioner }} {{- if and .repository .tag }} - ROOK_CSI_PROVISIONER_IMAGE: "{{ .repository }}:{{ .tag }}" - {{- end }} - {{- end }} - {{- with .snapshotter }} - {{- if and .repository .tag }} - ROOK_CSI_SNAPSHOTTER_IMAGE: "{{ .repository }}:{{ .tag }}" + provisioner: "{{ .repository }}:{{ .tag }}" {{- end }} {{- end }} {{- with .attacher }} {{- if and .repository .tag }} - ROOK_CSI_ATTACHER_IMAGE: "{{ .repository }}:{{ .tag }}" + attacher: "{{ .repository }}:{{ .tag }}" {{- end }} {{- end }} {{- with .resizer }} {{- if and .repository .tag }} - ROOK_CSI_RESIZER_IMAGE: "{{ .repository }}:{{ .tag }}" + resizer: "{{ .repository }}:{{ .tag }}" {{- end }} {{- end }} - {{- with .imagePullPolicy }} - ROOK_CSI_IMAGE_PULL_POLICY: {{ . | quote }} + {{- with .snapshotter }} + {{- if and .repository .tag }} + snapshotter: "{{ .repository }}:{{ .tag }}" + {{- end }} + {{- end }} + {{- with .registrar }} + {{- if and .repository .tag }} + registrar: "{{ .repository }}:{{ .tag }}" + {{- end }} + {{- end }} + {{- with .cephcsi }} + {{- if and .repository .tag }} + plugin: "{{ .repository }}:{{ .tag }}" + {{- end }} {{- end }} {{- with .csiAddons }} - CSI_ENABLE_CSIADDONS: {{ .enabled | quote }} {{- if and .repository .tag }} - ROOK_CSIADDONS_IMAGE: "{{ .repository }}:{{ .tag }}" - {{- end }} - {{- end }} - {{- with .crossNamespaceVolumeDataSource }} - CSI_ENABLE_CROSS_NAMESPACE_VOLUME_DATA_SOURCE: {{ .enabled | quote }} - {{- end }} - {{- with .topology }} - CSI_ENABLE_TOPOLOGY: {{ .enabled | quote }} - {{- with .domainLabels }} - CSI_TOPOLOGY_DOMAIN_LABELS: {{ . | join "," | quote }} - {{- end }} - {{- end }} - {{- with .nfs }} - ROOK_CSI_ENABLE_NFS: {{ .enabled | quote }} - {{- end }} - {{- with .cephfsPodLabels }} - ROOK_CSI_CEPHFS_POD_LABELS: {{ . | quote }} - {{- end }} - {{- with .nfsPodLabels }} - ROOK_CSI_NFS_POD_LABELS: {{ . | quote }} - {{- end }} - {{- with .rbdPodLabels }} - ROOK_CSI_RBD_POD_LABELS: {{ . | quote }} - {{- end }} - {{- with .provisionerTolerations }} - CSI_PROVISIONER_TOLERATIONS: {{ . | toYaml | quote }} - {{- end }} - {{- with .provisionerNodeAffinity }} - CSI_PROVISIONER_NODE_AFFINITY: {{ . | quote }} - {{- end }} - {{- with .rbdProvisionerTolerations }} - CSI_RBD_PROVISIONER_TOLERATIONS: {{ . | toYaml | quote }} - {{- end }} - {{- with .rbdProvisionerNodeAffinity }} - CSI_RBD_PROVISIONER_NODE_AFFINITY: {{ . | quote }} - {{- end }} - {{- with .cephFSProvisionerTolerations }} - CSI_CEPHFS_PROVISIONER_TOLERATIONS: {{ . | toYaml | quote }} - {{- end }} - {{- with .cephFSProvisionerNodeAffinity }} - CSI_CEPHFS_PROVISIONER_NODE_AFFINITY: {{ . | quote }} - {{- end }} - {{- with .nfsProvisionerTolerations }} - CSI_NFS_PROVISIONER_TOLERATIONS: {{ . | toYaml | quote }} - {{- end }} - {{- with .nfsProvisionerNodeAffinity }} - CSI_NFS_PROVISIONER_NODE_AFFINITY: {{ . | quote }} - {{- end }} - {{- with .pluginTolerations }} - CSI_PLUGIN_TOLERATIONS: {{ . | toYaml | quote }} - {{- end }} - {{- with .pluginNodeAffinity }} - CSI_PLUGIN_NODE_AFFINITY: {{ . | quote }} - {{- end }} - {{- with .rbdPluginTolerations }} - CSI_RBD_PLUGIN_TOLERATIONS: {{ . | toYaml | quote }} - {{- end }} - {{- with .rbdPluginNodeAffinity }} - CSI_RBD_PLUGIN_NODE_AFFINITY: {{ . | quote }} - {{- end }} - {{- with .cephFSPluginTolerations }} - CSI_CEPHFS_PLUGIN_TOLERATIONS: {{ . | toYaml | quote }} - {{- end }} - {{- with .cephFSPluginNodeAffinity }} - CSI_CEPHFS_PLUGIN_NODE_AFFINITY: {{ . | quote }} - {{- end }} - {{- with .nfsPluginTolerations }} - CSI_NFS_PLUGIN_TOLERATIONS: {{ . | toYaml | quote }} - {{- end }} - {{- with .nfsPluginNodeAffinity }} - CSI_NFS_PLUGIN_NODE_AFFINITY: {{ . | quote }} - {{- end }} - {{- with .cephfsLivenessMetricsPort }} - CSI_CEPHFS_LIVENESS_METRICS_PORT: {{ . | quote }} - {{- end }} - {{- with .enableLiveness }} - CSI_ENABLE_LIVENESS: {{ . | quote }} - {{- end }} - {{- with .rbdLivenessMetricsPort }} - CSI_RBD_LIVENESS_METRICS_PORT: {{ . | quote }} - {{- end }} - {{- with .csiAddonsPort }} - CSIADDONS_PORT: {{ . | quote }} - {{- end }} - {{- with .csiAddonsRBDProvisionerPort }} - CSIADDONS_RBD_PROVISIONER_PORT: {{ . | quote }} - {{- end }} - {{- with .csiAddonsCephFSProvisionerPort }} - CSIADDONS_CEPHFS_PROVISIONER_PORT: {{ . | quote }} - {{- end }} - {{- with .forceCephFSKernelClient }} - CSI_FORCE_CEPHFS_KERNEL_CLIENT: {{ . | quote }} - {{- end }} - {{- with .logLevel }} - CSI_LOG_LEVEL: {{ . | quote }} - {{- end }} - {{- with .sidecarLogLevel }} - CSI_SIDECAR_LOG_LEVEL: {{ . | quote }} - {{- end }} - {{- with .clusterName }} - CSI_CLUSTER_NAME: {{ . | quote }} - {{- end }} - {{- with .grpcTimeoutInSeconds }} - CSI_GRPC_TIMEOUT_SECONDS: {{ . | quote }} - {{- end }} - {{- with .provisionerReplicas }} - CSI_PROVISIONER_REPLICAS: {{ . | quote }} - {{- end }} - {{- with .csiRBDProvisionerResource }} - CSI_RBD_PROVISIONER_RESOURCE: {{ . | quote }} - {{- end }} - {{- with .csiRBDPluginResource }} - CSI_RBD_PLUGIN_RESOURCE: {{ . | quote }} - {{- end }} - {{- with .csiCephFSProvisionerResource }} - CSI_CEPHFS_PROVISIONER_RESOURCE: {{ . | quote }} - {{- end }} - {{- with .csiCephFSPluginResource }} - CSI_CEPHFS_PLUGIN_RESOURCE: {{ . | quote }} - {{- end }} - {{- with .csiNFSProvisionerResource }} - CSI_NFS_PROVISIONER_RESOURCE: {{ . | quote }} - {{- end }} - {{- with .csiNFSPluginResource }} - CSI_NFS_PLUGIN_RESOURCE: {{ . | quote }} - {{- end }} - {{- with .csiRBDPluginVolume }} - CSI_RBD_PLUGIN_VOLUME: {{ . | toYaml | quote }} - {{- end }} - {{- with .csiRBDPluginVolumeMount }} - CSI_RBD_PLUGIN_VOLUME_MOUNT: {{ . | toYaml | quote }} - {{- end }} - {{- with .csiCephFSPluginVolume }} - CSI_CEPHFS_PLUGIN_VOLUME: {{ . | toYaml | quote }} - {{- end }} - {{- with .csiCephFSPluginVolumeMount }} - CSI_CEPHFS_PLUGIN_VOLUME_MOUNT: {{ . | toYaml | quote }} - {{- end }} - CSI_CEPHFS_ATTACH_REQUIRED: {{ .cephFSAttachRequired | quote }} - CSI_RBD_ATTACH_REQUIRED: {{ .rbdAttachRequired | quote }} - CSI_NFS_ATTACH_REQUIRED: {{ .nfsAttachRequired | quote }} - {{- with .kubeApiBurst }} - CSI_KUBE_API_BURST: {{ . | quote }} - {{- end }} - {{- with .kubeApiQPS }} - CSI_KUBE_API_QPS: {{ . | quote }} + addons: "{{ .repository }}:{{ .tag }}" {{- end }} {{- end }} +{{- end }} diff --git a/deploy/charts/rook-ceph/templates/role.yaml b/deploy/charts/rook-ceph/templates/role.yaml index ab0997c4b..e3454ae83 100644 --- a/deploy/charts/rook-ceph/templates/role.yaml +++ b/deploy/charts/rook-ceph/templates/role.yaml @@ -60,79 +60,4 @@ rules: verbs: - get - create ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-provisioner-cfg - namespace: {{ .Release.Namespace }} # namespace:operator - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -rules: - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - {{- if and .Values.csi.csiAddons .Values.csi.csiAddons.enabled }} - - apiGroups: ["csiaddons.openshift.io"] - resources: ["csiaddonsnodes"] - verbs: ["get", "watch", "list", "create", "update", "delete"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get"] - - apiGroups: ["apps"] - resources: ["replicasets"] - verbs: ["get"] - - apiGroups: ["apps"] - resources: ["deployments/finalizers", "daemonsets/finalizers"] - verbs: ["update"] - {{- end }} -{{- if and .Values.csi.csiAddons .Values.csi.csiAddons.enabled }} ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin - namespace: {{ .Release.Namespace }} # namespace:operator - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -rules: - - apiGroups: ["csiaddons.openshift.io"] - resources: ["csiaddonsnodes"] - verbs: ["get", "watch", "list", "create", "update", "delete"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get"] - - apiGroups: ["apps"] - resources: ["replicasets"] - verbs: ["get"] - - apiGroups: ["apps"] - resources: ["deployments/finalizers", "daemonsets/finalizers"] - verbs: ["update"] -{{- end }} ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-provisioner-cfg - namespace: {{ .Release.Namespace }} # namespace:operator - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -rules: - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - {{- if and .Values.csi.csiAddons .Values.csi.csiAddons.enabled }} - - apiGroups: ["csiaddons.openshift.io"] - resources: ["csiaddonsnodes"] - verbs: ["get", "watch", "list", "create", "update", "delete"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get"] - - apiGroups: ["apps"] - resources: ["replicasets"] - verbs: ["get"] - - apiGroups: ["apps"] - resources: ["deployments/finalizers", "daemonsets/finalizers"] - verbs: ["update"] - {{- end }} {{- end }} diff --git a/deploy/charts/rook-ceph/templates/rolebinding.yaml b/deploy/charts/rook-ceph/templates/rolebinding.yaml index ae2dc54a4..1d60696ad 100644 --- a/deploy/charts/rook-ceph/templates/rolebinding.yaml +++ b/deploy/charts/rook-ceph/templates/rolebinding.yaml @@ -16,54 +16,4 @@ subjects: - kind: ServiceAccount name: rook-ceph-system namespace: {{ .Release.Namespace }} # namespace:operator ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-provisioner-role-cfg - namespace: {{ .Release.Namespace }} # namespace:operator - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: {{ .Release.Namespace }} # namespace:operator -roleRef: - kind: Role - name: cephfs-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io -{{- if and .Values.csi.csiAddons .Values.csi.csiAddons.enabled }} ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin-role-cfg - namespace: {{ .Release.Namespace }} # namespace:operator - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-plugin-sa - namespace: {{ .Release.Namespace }} # namespace:operator -roleRef: - kind: Role - name: rbd-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io -{{- end }} ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-provisioner-role-cfg - namespace: {{ .Release.Namespace }} # namespace:operator - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: {{ .Release.Namespace }} # namespace:operator -roleRef: - kind: Role - name: rbd-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io {{- end }} diff --git a/deploy/charts/rook-ceph/templates/securityContextConstraints.yaml b/deploy/charts/rook-ceph/templates/securityContextConstraints.yaml index 207401b3f..bf74f5e08 100644 --- a/deploy/charts/rook-ceph/templates/securityContextConstraints.yaml +++ b/deploy/charts/rook-ceph/templates/securityContextConstraints.yaml @@ -89,20 +89,11 @@ volumes: - hostPath - projected users: - # A user needs to be added for each csi service account. - {{- if .Values.csi.rookUseCsiOperator }} + # A user needs to be added for each csi service account managed by the ceph-csi-operator. - system:serviceaccount:{{ .Release.Namespace }}:ceph-csi-cephfs-ctrlplugin-sa - system:serviceaccount:{{ .Release.Namespace }}:ceph-csi-cephfs-nodeplugin-sa - system:serviceaccount:{{ .Release.Namespace }}:ceph-csi-nfs-ctrlplugin-sa - system:serviceaccount:{{ .Release.Namespace }}:ceph-csi-nfs-nodeplugin-sa - system:serviceaccount:{{ .Release.Namespace }}:ceph-csi-rbd-ctrlplugin-sa - system:serviceaccount:{{ .Release.Namespace }}:ceph-csi-rbd-nodeplugin-sa - {{- else }} - - system:serviceaccount:{{ .Release.Namespace }}:rook-csi-rbd-plugin-sa - - system:serviceaccount:{{ .Release.Namespace }}:rook-csi-rbd-provisioner-sa - - system:serviceaccount:{{ .Release.Namespace }}:rook-csi-cephfs-plugin-sa - - system:serviceaccount:{{ .Release.Namespace }}:rook-csi-cephfs-provisioner-sa - - system:serviceaccount:{{ .Release.Namespace }}:rook-csi-nfs-plugin-sa - - system:serviceaccount:{{ .Release.Namespace }}:rook-csi-nfs-provisioner-sa - {{- end }} {{- end }} diff --git a/deploy/charts/rook-ceph/templates/serviceaccount.yaml b/deploy/charts/rook-ceph/templates/serviceaccount.yaml index 3e9bd3b34..a86e568cc 100644 --- a/deploy/charts/rook-ceph/templates/serviceaccount.yaml +++ b/deploy/charts/rook-ceph/templates/serviceaccount.yaml @@ -9,68 +9,6 @@ metadata: {{- include "library.rook-ceph.labels" . | nindent 4 }} {{- include "library.imagePullSecrets" . | nindent 0 }} --- -# Service account for the CephFS CSI driver -kind: ServiceAccount -apiVersion: v1 -metadata: - name: rook-csi-cephfs-plugin-sa - namespace: {{ .Release.Namespace }} # namespace:operator - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -{{- include "library.imagePullSecrets" . | nindent 0 }} -{{- if .Values.csi.nfs.enabled }} ---- -# Service account for the NFS CSI driver -kind: ServiceAccount -apiVersion: v1 -metadata: - name: rook-csi-nfs-plugin-sa - namespace: {{ .Release.Namespace }} # namespace:operator - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -{{- include "library.imagePullSecrets" . | nindent 0 }} ---- -# Service account for the NFS CSI provisioner -kind: ServiceAccount -apiVersion: v1 -metadata: - name: rook-csi-nfs-provisioner-sa - namespace: {{ .Release.Namespace }} # namespace:operator - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -{{- include "library.imagePullSecrets" . | nindent 0 }} -{{- end }} ---- -# Service account for the CephFS CSI provisioner -kind: ServiceAccount -apiVersion: v1 -metadata: - name: rook-csi-cephfs-provisioner-sa - namespace: {{ .Release.Namespace }} # namespace:operator - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -{{- include "library.imagePullSecrets" . | nindent 0 }} ---- -# Service account for the RBD CSI driver -kind: ServiceAccount -apiVersion: v1 -metadata: - name: rook-csi-rbd-plugin-sa - namespace: {{ .Release.Namespace }} # namespace:operator - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -{{- include "library.imagePullSecrets" . | nindent 0 }} ---- -# Service account for the RBD CSI provisioner -kind: ServiceAccount -apiVersion: v1 -metadata: - name: rook-csi-rbd-provisioner-sa - namespace: {{ .Release.Namespace }} # namespace:operator - labels: - {{- include "library.rook-ceph.labels" . | nindent 4 }} -{{- include "library.imagePullSecrets" . | nindent 0 }} ---- # Service account for Ceph COSI driver kind: ServiceAccount apiVersion: v1 diff --git a/deploy/charts/rook-ceph/templates/servicemonitor.yaml b/deploy/charts/rook-ceph/templates/servicemonitor.yaml index 8f19d8a79..1780bc5af 100644 --- a/deploy/charts/rook-ceph/templates/servicemonitor.yaml +++ b/deploy/charts/rook-ceph/templates/servicemonitor.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.monitoring.enabled .Values.csi.serviceMonitor.enabled (or .Values.csi.enableLiveness .Values.csi.enableGrpcMetrics) -}} +{{- if and .Values.monitoring.enabled .Values.csi.serviceMonitor.enabled -}} --- kind: ServiceMonitor apiVersion: monitoring.coreos.com/v1 @@ -18,9 +18,7 @@ spec: matchLabels: app: csi-metrics endpoints: - {{- if .Values.csi.enableLiveness }} - port: csi-http-metrics path: /metrics interval: {{ .Values.csi.serviceMonitor.interval }} - {{- end }} {{- end }} diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index 4cdddd18a..42da28b2f 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -88,414 +88,9 @@ ceph-csi-operator: env: csiServiceAccountPrefix: "ceph-csi-" -# Settings for whether to disable the drivers or other daemons if they are not -# needed csi: - rookUseCsiOperator: true - # -- Enable Ceph CSI RBD driver - enableRbdDriver: true - # -- Enable Ceph CSI CephFS driver - enableCephfsDriver: true - # -- Disable the CSI driver. - disableCsiDriver: "false" - - # -- Enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary - # in some network configurations where the SDN does not provide access to an external cluster or - # there is significant drop in read/write performance - enableCSIHostNetwork: true - # -- Enable Snapshotter in CephFS provisioner pod - enableCephfsSnapshotter: true - # -- Enable Snapshotter in NFS provisioner pod - enableNFSSnapshotter: true - # -- Enable Snapshotter in RBD provisioner pod - enableRBDSnapshotter: true - # -- Enable Host mount for `/etc/selinux` directory for Ceph CSI nodeplugins - enablePluginSelinuxHostMount: false - # -- Enable Ceph CSI PVC encryption support - enableCSIEncryption: false - - # -- Enable volume group snapshot feature. This feature is - # enabled by default as long as the necessary CRDs are available in the cluster. - enableVolumeGroupSnapshot: true - # -- PriorityClassName to be set on csi driver plugin pods - pluginPriorityClassName: system-node-critical - - # -- PriorityClassName to be set on csi driver provisioner pods - provisionerPriorityClassName: system-cluster-critical - - # -- Policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted. - # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html - rbdFSGroupPolicy: "File" - - # -- Policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted. - # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html - cephFSFSGroupPolicy: "File" - - # -- Policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted. - # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html - nfsFSGroupPolicy: "File" - - # -- OMAP generator generates the omap mapping between the PV name and the RBD image - # which helps CSI to identify the rbd images for CSI operations. - # `CSI_ENABLE_OMAP_GENERATOR` needs to be enabled when we are using rbd mirroring feature. - # By default OMAP generator is disabled and when enabled, it will be deployed as a - # sidecar with CSI provisioner pod, to enable set it to true. - enableOMAPGenerator: false - - # -- Set CephFS Kernel mount options to use https://docs.ceph.com/en/latest/man/8/mount.ceph/#options. - # Set to "ms_mode=secure" when connections.encrypted is enabled in CephCluster CR - cephFSKernelMountOptions: - - # -- Enable adding volume metadata on the CephFS subvolumes and RBD images. - # Not all users might be interested in getting volume/snapshot details as metadata on CephFS subvolume and RBD images. - # Hence enable metadata is false by default - enableMetadata: false - - # -- Set replicas for csi provisioner deployment - provisionerReplicas: 2 - - # -- Cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful - # in cases like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster - clusterName: - - # -- Set logging level for cephCSI containers maintained by the cephCSI. - # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. - logLevel: 0 - - # -- Set logging level for Kubernetes-csi sidecar containers. - # Supported values from 0 to 5. 0 for general useful logs (the default), 5 for trace level verbosity. - # @default -- `0` - sidecarLogLevel: - - # -- CSI driver name prefix for cephfs, rbd and nfs. - # @default -- `namespace name where rook-ceph operator is deployed` - csiDriverNamePrefix: - - # -- CSI RBD plugin daemonset update strategy, supported values are OnDelete and RollingUpdate - # @default -- `RollingUpdate` - rbdPluginUpdateStrategy: - - # -- A maxUnavailable parameter of CSI RBD plugin daemonset update strategy. - # @default -- `1` - rbdPluginUpdateStrategyMaxUnavailable: - - # -- CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate - # @default -- `RollingUpdate` - cephFSPluginUpdateStrategy: - - # -- A maxUnavailable parameter of CSI cephFS plugin daemonset update strategy. - # @default -- `1` - cephFSPluginUpdateStrategyMaxUnavailable: - - # -- CSI NFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate - # @default -- `RollingUpdate` - nfsPluginUpdateStrategy: - - # -- Set GRPC timeout for csi containers (in seconds). It should be >= 120. If this value is not set or is invalid, it defaults to 150 - grpcTimeoutInSeconds: 150 - - # -- Burst to use while communicating with the kubernetes apiserver. - kubeApiBurst: - - # -- QPS to use while communicating with the kubernetes apiserver. - kubeApiQPS: - - # -- The volume of the CephCSI RBD plugin DaemonSet - csiRBDPluginVolume: - # - name: lib-modules - # hostPath: - # path: /run/booted-system/kernel-modules/lib/modules/ - # - name: host-nix - # hostPath: - # path: /nix - - # -- The volume mounts of the CephCSI RBD plugin DaemonSet - csiRBDPluginVolumeMount: - # - name: host-nix - # mountPath: /nix - # readOnly: true - - # -- The volume of the CephCSI CephFS plugin DaemonSet - csiCephFSPluginVolume: - # - name: lib-modules - # hostPath: - # path: /run/booted-system/kernel-modules/lib/modules/ - # - name: host-nix - # hostPath: - # path: /nix - - # -- The volume mounts of the CephCSI CephFS plugin DaemonSet - csiCephFSPluginVolumeMount: - # - name: host-nix - # mountPath: /nix - # readOnly: true - - # -- CEPH CSI RBD provisioner resource requirement list - # csi-omap-generator resources will be applied only if `enableOMAPGenerator` is set to `true` - # @default -- see values.yaml - csiRBDProvisionerResource: | - - name : csi-provisioner - resource: - requests: - memory: 128Mi - cpu: 100m - limits: - memory: 256Mi - - name : csi-resizer - resource: - requests: - memory: 128Mi - cpu: 100m - limits: - memory: 256Mi - - name : csi-attacher - resource: - requests: - memory: 128Mi - cpu: 100m - limits: - memory: 256Mi - - name : csi-snapshotter - resource: - requests: - memory: 128Mi - cpu: 100m - limits: - memory: 256Mi - - name : csi-rbdplugin - resource: - requests: - memory: 512Mi - limits: - memory: 1Gi - - name : csi-omap-generator - resource: - requests: - memory: 512Mi - cpu: 250m - limits: - memory: 1Gi - - name : liveness-prometheus - resource: - requests: - memory: 128Mi - cpu: 50m - limits: - memory: 256Mi - - # -- CEPH CSI RBD plugin resource requirement list - # @default -- see values.yaml - csiRBDPluginResource: | - - name : driver-registrar - resource: - requests: - memory: 128Mi - cpu: 50m - limits: - memory: 256Mi - - name : csi-rbdplugin - resource: - requests: - memory: 512Mi - cpu: 250m - limits: - memory: 1Gi - - name : liveness-prometheus - resource: - requests: - memory: 128Mi - cpu: 50m - limits: - memory: 256Mi - - # -- CEPH CSI CephFS provisioner resource requirement list - # @default -- see values.yaml - csiCephFSProvisionerResource: | - - name : csi-provisioner - resource: - requests: - memory: 128Mi - cpu: 100m - limits: - memory: 256Mi - - name : csi-resizer - resource: - requests: - memory: 128Mi - cpu: 100m - limits: - memory: 256Mi - - name : csi-attacher - resource: - requests: - memory: 128Mi - cpu: 100m - limits: - memory: 256Mi - - name : csi-snapshotter - resource: - requests: - memory: 128Mi - cpu: 100m - limits: - memory: 256Mi - - name : csi-cephfsplugin - resource: - requests: - memory: 512Mi - cpu: 250m - limits: - memory: 1Gi - - name : liveness-prometheus - resource: - requests: - memory: 128Mi - cpu: 50m - limits: - memory: 256Mi - - # -- CEPH CSI CephFS plugin resource requirement list - # @default -- see values.yaml - csiCephFSPluginResource: | - - name : driver-registrar - resource: - requests: - memory: 128Mi - cpu: 50m - limits: - memory: 256Mi - - name : csi-cephfsplugin - resource: - requests: - memory: 512Mi - cpu: 250m - limits: - memory: 1Gi - - name : liveness-prometheus - resource: - requests: - memory: 128Mi - cpu: 50m - limits: - memory: 256Mi - - # -- CEPH CSI NFS provisioner resource requirement list - # @default -- see values.yaml - csiNFSProvisionerResource: | - - name : csi-provisioner - resource: - requests: - memory: 128Mi - cpu: 100m - limits: - memory: 256Mi - - name : csi-nfsplugin - resource: - requests: - memory: 512Mi - cpu: 250m - limits: - memory: 1Gi - - name : csi-attacher - resource: - requests: - memory: 512Mi - cpu: 250m - limits: - memory: 1Gi - - # -- CEPH CSI NFS plugin resource requirement list - # @default -- see values.yaml - csiNFSPluginResource: | - - name : driver-registrar - resource: - requests: - memory: 128Mi - cpu: 50m - limits: - memory: 256Mi - - name : csi-nfsplugin - resource: - requests: - memory: 512Mi - cpu: 250m - limits: - memory: 1Gi - - # Set provisionerTolerations and provisionerNodeAffinity for provisioner pod. - # The CSI provisioner would be best to start on the same nodes as other ceph daemons. - - # -- Array of tolerations in YAML format which will be added to CSI provisioner deployment - provisionerTolerations: - # - key: key - # operator: Exists - # effect: NoSchedule - - # -- The node labels for affinity of the CSI provisioner deployment [^1] - provisionerNodeAffinity: #key1=value1,value2; key2=value3 - # Set pluginTolerations and pluginNodeAffinity for plugin daemonset pods. - # The CSI plugins need to be started on all the nodes where the clients need to mount the storage. - - # -- Array of tolerations in YAML format which will be added to CephCSI plugin DaemonSet - pluginTolerations: - # - key: key - # operator: Exists - # effect: NoSchedule - - # -- The node labels for affinity of the CephCSI RBD plugin DaemonSet [^1] - pluginNodeAffinity: # key1=value1,value2; key2=value3 - - # -- Enable Ceph CSI Liveness sidecar deployment - enableLiveness: false - - # -- CSI CephFS driver metrics port - # @default -- `9081` - cephfsLivenessMetricsPort: - - # -- CSI Addons server port - # @default -- `9070` - csiAddonsPort: - # -- CSI Addons server port for the RBD provisioner - # @default -- `9070` - csiAddonsRBDProvisionerPort: - # -- CSI Addons server port for the Ceph FS provisioner - # @default -- `9070` - csiAddonsCephFSProvisionerPort: - - # -- Enable Ceph Kernel clients on kernel < 4.17. If your kernel does not support quotas for CephFS - # you may want to disable this setting. However, this will cause an issue during upgrades - # with the FUSE client. See the [upgrade guide](https://rook.io/docs/rook/v1.2/ceph-upgrade.html) - forceCephFSKernelClient: true - - # -- Ceph CSI RBD driver metrics port - # @default -- `8080` - rbdLivenessMetricsPort: - - serviceMonitor: - # -- Enable ServiceMonitor for Ceph CSI drivers - enabled: false - # -- Service monitor scrape interval - interval: 10s - # -- ServiceMonitor additional labels - labels: {} - # -- Use a different namespace for the ServiceMonitor - namespace: - - # -- Kubelet root directory path (if the Kubelet uses a different path for the `--root-dir` flag) - # @default -- `/var/lib/kubelet` - kubeletDirPath: - - # -- Duration in seconds that non-leader candidates will wait to force acquire leadership. - # @default -- `137s` - csiLeaderElectionLeaseDuration: - - # -- Deadline in seconds that the acting leader will retry refreshing leadership before giving up. - # @default -- `107s` - csiLeaderElectionRenewDeadline: - - # -- Retry period in seconds the LeaderElector clients should wait between tries of actions. - # @default -- `26s` - csiLeaderElectionRetryPeriod: + # -- When true, install the ceph-csi-operator subchart (see Chart.yaml `condition`). + installCsiOperator: true cephcsi: # -- Ceph CSI image repository @@ -533,66 +128,22 @@ csi: # -- Resizer image tag tag: v2.1.0 - # -- Image pull policy - imagePullPolicy: IfNotPresent - - # -- Labels to add to the CSI CephFS Deployments and DaemonSets Pods - cephfsPodLabels: #"key1=value1,key2=value2" - - # -- Labels to add to the CSI NFS Deployments and DaemonSets Pods - nfsPodLabels: #"key1=value1,key2=value2" - - # -- Labels to add to the CSI RBD Deployments and DaemonSets Pods - rbdPodLabels: #"key1=value1,key2=value2" - csiAddons: - # -- Enable CSIAddons - enabled: false # -- CSIAddons sidecar image repository repository: quay.io/csiaddons/k8s-sidecar # -- CSIAddons sidecar image tag tag: v0.14.0 - nfs: - # -- Enable the nfs csi driver + serviceMonitor: + # -- Enable ServiceMonitor for CSI metrics enabled: false - - topology: - # -- Enable topology based provisioning - enabled: false - # NOTE: the value here serves as an example and needs to be - # updated with node labels that define domains of interest - # -- domainLabels define which node labels to use as domains - # for CSI nodeplugins to advertise their domains - domainLabels: - # - kubernetes.io/hostname - # - topology.kubernetes.io/zone - # - topology.rook.io/rack - - crossNamespaceVolumeDataSource: - # -- Enable cross namespace volume data source provisioning - enabled: false - - # -- Whether to skip any attach operation altogether for CephFS PVCs. See more details - # [here](https://kubernetes-csi.github.io/docs/skip-attach.html#skip-attach-with-csi-driver-object). - # If cephFSAttachRequired is set to false it skips the volume attachments and makes the creation - # of pods using the CephFS PVC fast. **WARNING** It's highly discouraged to use this for - # CephFS RWO volumes. Refer to this [issue](https://github.com/kubernetes/kubernetes/issues/103305) for more details. - cephFSAttachRequired: true - # -- Whether to skip any attach operation altogether for RBD PVCs. See more details - # [here](https://kubernetes-csi.github.io/docs/skip-attach.html#skip-attach-with-csi-driver-object). - # If set to false it skips the volume attachments and makes the creation of pods using the RBD PVC fast. - # **WARNING** It's highly discouraged to use this for RWO volumes as it can cause data corruption. - # csi-addons operations like Reclaimspace and PVC Keyrotation will also not be supported if set - # to false since we'll have no VolumeAttachments to determine which node the PVC is mounted on. - # Refer to this [issue](https://github.com/kubernetes/kubernetes/issues/103305) for more details. - rbdAttachRequired: true - # -- Whether to skip any attach operation altogether for NFS PVCs. See more details - # [here](https://kubernetes-csi.github.io/docs/skip-attach.html#skip-attach-with-csi-driver-object). - # If cephFSAttachRequired is set to false it skips the volume attachments and makes the creation - # of pods using the NFS PVC fast. **WARNING** It's highly discouraged to use this for - # NFS RWO volumes. Refer to this [issue](https://github.com/kubernetes/kubernetes/issues/103305) for more details. - nfsAttachRequired: true + # -- Namespace in which to deploy the ServiceMonitor + # @default -- the release namespace + namespace: + # -- Additional labels for the ServiceMonitor + labels: {} + # -- Interval at which metrics should be scraped + interval: 5s # -- Enable discovery daemon enableDiscoveryDaemon: false diff --git a/deploy/examples/common.yaml b/deploy/examples/common.yaml index d3a6b2415..94072f2dc 100644 --- a/deploy/examples/common.yaml +++ b/deploy/examples/common.yaml @@ -13,109 +13,6 @@ metadata: --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts/token"] - verbs: ["create"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-provisioner-runner - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "patch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["events.k8s.io"] - resources: ["events"] - verbs: ["create", "patch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list", "watch", "patch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update", "patch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotcontents"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotcontents/status"] - verbs: ["update", "patch"] - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts/token"] - verbs: ["create"] - - apiGroups: ["authentication.k8s.io"] - resources: ["tokenreviews"] - verbs: ["create"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 metadata: name: objectstorage-provisioner-role labels: @@ -136,130 +33,6 @@ rules: resources: ["events"] verbs: ["create", "patch", "update"] --- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts/token"] - verbs: ["create"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get"] - - apiGroups: ["authentication.k8s.io"] - resources: ["tokenreviews"] - verbs: ["create"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-provisioner-runner - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["events.k8s.io"] - resources: ["events"] - verbs: ["create", "patch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list", "watch", "patch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update", "patch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotcontents"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotcontents/status"] - verbs: ["update", "patch"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts/token"] - verbs: ["create"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["gateway.networking.k8s.io"] - resources: ["referencegrants"] - verbs: ["get", "list", "watch"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumegroupreplicationcontents"] - verbs: ["get", "list", "watch"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumegroupreplicationclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["authentication.k8s.io"] - resources: ["tokenreviews"] - verbs: ["create"] ---- # The cluster role for managing all the cluster-specific resources in a namespace kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -691,42 +464,6 @@ rules: resources: ["drivers"] verbs: ["create", "delete", "get", "list", "update", "watch"] --- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin-role - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-plugin-sa - namespace: rook-ceph # namespace:operator -roleRef: - kind: ClusterRole - name: cephfs-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-provisioner-role - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: rook-ceph # namespace:operator -roleRef: - kind: ClusterRole - name: cephfs-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- # RBAC for ceph cosi driver service account kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -745,42 +482,6 @@ roleRef: name: objectstorage-provisioner-role apiGroup: rbac.authorization.k8s.io --- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-plugin-sa - namespace: rook-ceph # namespace:operator -roleRef: - kind: ClusterRole - name: rbd-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-provisioner-role - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: rook-ceph # namespace:operator -roleRef: - kind: ClusterRole - name: rbd-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- # Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -877,84 +578,6 @@ subjects: --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-provisioner-cfg - namespace: rook-ceph # namespace:operator - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -rules: - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: ["csiaddons.openshift.io"] - resources: ["csiaddonsnodes"] - verbs: ["get", "watch", "list", "create", "update", "delete"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get"] - - apiGroups: ["apps"] - resources: ["replicasets"] - verbs: ["get"] - - apiGroups: ["apps"] - resources: ["deployments/finalizers", "daemonsets/finalizers"] - verbs: ["update"] ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin - namespace: rook-ceph # namespace:operator - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -rules: - - apiGroups: ["csiaddons.openshift.io"] - resources: ["csiaddonsnodes"] - verbs: ["get", "watch", "list", "create", "update", "delete"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get"] - - apiGroups: ["apps"] - resources: ["replicasets"] - verbs: ["get"] - - apiGroups: ["apps"] - resources: ["deployments/finalizers", "daemonsets/finalizers"] - verbs: ["update"] ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-provisioner-cfg - namespace: rook-ceph # namespace:operator - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -rules: - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: ["csiaddons.openshift.io"] - resources: ["csiaddonsnodes"] - verbs: ["get", "watch", "list", "create", "update", "delete"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get"] - - apiGroups: ["apps"] - resources: ["replicasets"] - verbs: ["get"] - - apiGroups: ["apps"] - resources: ["deployments/finalizers", "daemonsets/finalizers"] - verbs: ["update"] ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-cmd-reporter namespace: rook-ceph # namespace:cluster @@ -1170,63 +793,6 @@ rules: - get - create --- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-provisioner-role-cfg - namespace: rook-ceph # namespace:operator - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: rook-ceph # namespace:operator -roleRef: - kind: Role - name: cephfs-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin-role-cfg - namespace: rook-ceph # namespace:operator - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-plugin-sa - namespace: rook-ceph # namespace:operator -roleRef: - kind: Role - name: rbd-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-provisioner-role-cfg - namespace: rook-ceph # namespace:operator - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: rook-ceph # namespace:operator -roleRef: - kind: Role - name: rbd-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io ---- # Allow the operator to create resources in this cluster's namespace kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -1490,59 +1056,3 @@ metadata: app.kubernetes.io/part-of: rook-ceph-operator # imagePullSecrets: # - name: my-registry-secret ---- -# Service account for the CephFS CSI driver -kind: ServiceAccount -apiVersion: v1 -metadata: - name: rook-csi-cephfs-plugin-sa - namespace: rook-ceph # namespace:operator - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -# imagePullSecrets: -# - name: my-registry-secret ---- -# Service account for the CephFS CSI provisioner -kind: ServiceAccount -apiVersion: v1 -metadata: - name: rook-csi-cephfs-provisioner-sa - namespace: rook-ceph # namespace:operator - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -# imagePullSecrets: -# - name: my-registry-secret ---- -# Service account for the RBD CSI driver -kind: ServiceAccount -apiVersion: v1 -metadata: - name: rook-csi-rbd-plugin-sa - namespace: rook-ceph # namespace:operator - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -# imagePullSecrets: -# - name: my-registry-secret ---- -# Service account for the RBD CSI provisioner -kind: ServiceAccount -apiVersion: v1 -metadata: - name: rook-csi-rbd-provisioner-sa - namespace: rook-ceph # namespace:operator - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -# imagePullSecrets: -# - name: my-registry-secret diff --git a/deploy/examples/csi/nfs/driver.yaml b/deploy/examples/csi/nfs/driver.yaml new file mode 100644 index 000000000..58871a734 --- /dev/null +++ b/deploy/examples/csi/nfs/driver.yaml @@ -0,0 +1,12 @@ +# NFS CSI Driver +apiVersion: csi.ceph.io/v1 +kind: Driver +metadata: + name: rook-ceph.nfs.csi.ceph.com # csi-provisioner-name + namespace: rook-ceph # namespace:operator +spec: + fsGroupPolicy: File + nodePlugin: + updateStrategy: + type: RollingUpdate + controllerPlugin: {} diff --git a/deploy/examples/csi/nfs/rbac.yaml b/deploy/examples/csi/nfs/rbac.yaml deleted file mode 100644 index 4a74678bf..000000000 --- a/deploy/examples/csi/nfs/rbac.yaml +++ /dev/null @@ -1,144 +0,0 @@ -#################################################################################################### -# These RBAC resources are required to allow Rook to run the NFS export CSI driver components. -#################################################################################################### ---- -# TODO: remove this, once https://github.com/rook/rook/issues/10141 -# is resolved. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ceph-nfs-csi-nodeplugin - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ceph-nfs-external-provisioner-runner - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "patch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] - - apiGroups: ["events.k8s.io"] - resources: ["events"] - verbs: ["create", "patch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] ---- -# TODO: remove this, once https://github.com/rook/rook/issues/10141 -# is resolved. -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ceph-nfs-csi-nodeplugin-role - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -subjects: - - kind: ServiceAccount - name: rook-csi-nfs-plugin-sa - namespace: rook-ceph # namespace:operator -roleRef: - kind: ClusterRole - name: ceph-nfs-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ceph-nfs-csi-provisioner-role - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -subjects: - - kind: ServiceAccount - name: rook-csi-nfs-provisioner-sa - namespace: rook-ceph # namespace:operator -roleRef: - kind: ClusterRole - name: ceph-nfs-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- -# Service account for the NFS CSI driver -kind: ServiceAccount -apiVersion: v1 -metadata: - name: rook-csi-nfs-plugin-sa - namespace: rook-ceph # namespace:operator - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -# imagePullSecrets: -# - name: my-registry-secret ---- -# Service account for the NFS CSI provisioner -kind: ServiceAccount -apiVersion: v1 -metadata: - name: rook-csi-nfs-provisioner-sa - namespace: rook-ceph # namespace:operator - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/part-of: rook-ceph-operator -# imagePullSecrets: -# - name: my-registry-secret diff --git a/deploy/examples/images.txt b/deploy/examples/images.txt index 8afa987c4..419e08a8c 100644 --- a/deploy/examples/images.txt +++ b/deploy/examples/images.txt @@ -1,12 +1,12 @@ - docker.io/rook/ceph:master - gcr.io/k8s-staging-sig-storage/objectstorage-sidecar:v0.2.2 - quay.io/ceph/ceph:v20.2.1 - quay.io/ceph/cosi:v0.1.4 - quay.io/cephcsi/ceph-csi-operator:v0.6.0 - quay.io/cephcsi/cephcsi:v3.16.2 - quay.io/csiaddons/k8s-sidecar:v0.14.0 - registry.k8s.io/sig-storage/csi-attacher:v4.11.0 - registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.16.0 - registry.k8s.io/sig-storage/csi-provisioner:v6.1.1 - registry.k8s.io/sig-storage/csi-resizer:v2.1.0 - registry.k8s.io/sig-storage/csi-snapshotter:v8.5.0 +docker.io/rook/ceph:master +gcr.io/k8s-staging-sig-storage/objectstorage-sidecar:v0.2.2 +quay.io/ceph/ceph:v20.2.1 +quay.io/ceph/cosi:v0.1.4 +quay.io/cephcsi/ceph-csi-operator:v0.6.0 +quay.io/cephcsi/cephcsi:v3.16.2 +quay.io/csiaddons/k8s-sidecar:v0.14.0 +registry.k8s.io/sig-storage/csi-attacher:v4.11.0 +registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.16.0 +registry.k8s.io/sig-storage/csi-provisioner:v6.1.1 +registry.k8s.io/sig-storage/csi-resizer:v2.1.0 +registry.k8s.io/sig-storage/csi-snapshotter:v8.5.0 diff --git a/deploy/examples/kustomization.yaml b/deploy/examples/kustomization.yaml index 5bed5c3e8..508ae3a32 100644 --- a/deploy/examples/kustomization.yaml +++ b/deploy/examples/kustomization.yaml @@ -5,4 +5,3 @@ resources: - crds.yaml - common.yaml - operator-openshift.yaml - - csi/nfs/rbac.yaml diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index 80b5d20a5..daa0e321a 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -88,18 +88,9 @@ volumes: - hostPath - projected users: - # A user needs to be added for each rook service account. + # A user needs to be added for each csi service account managed by the ceph-csi-operator. # This assumes running in the default sample "rook-ceph" namespace. # If other namespaces or service accounts are configured, they need to be updated here. - # Service accounts for the cephcsi driver (pre-cephcsi operator) - - system:serviceaccount:rook-ceph:rook-csi-rbd-plugin-sa # serviceaccount:namespace:operator - - system:serviceaccount:rook-ceph:rook-csi-rbd-provisioner-sa # serviceaccount:namespace:operator - - system:serviceaccount:rook-ceph:rook-csi-cephfs-plugin-sa # serviceaccount:namespace:operator - - system:serviceaccount:rook-ceph:rook-csi-cephfs-provisioner-sa # serviceaccount:namespace:operator - - system:serviceaccount:rook-ceph:rook-csi-nfs-plugin-sa # serviceaccount:namespace:operator - - system:serviceaccount:rook-ceph:rook-csi-nfs-provisioner-sa # serviceaccount:namespace:operator - - system:serviceaccount:rook-ceph:nvmeof-csi-provisioner # serviceaccount:namespace:operator - # Service accounts for the csi operator - system:serviceaccount:rook-ceph:ceph-csi-cephfs-ctrlplugin-sa # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:ceph-csi-cephfs-nodeplugin-sa # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:ceph-csi-nfs-ctrlplugin-sa # serviceaccount:namespace:operator @@ -108,6 +99,7 @@ users: - system:serviceaccount:rook-ceph:ceph-csi-nvmeof-nodeplugin-sa # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:ceph-csi-rbd-ctrlplugin-sa # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:ceph-csi-rbd-nodeplugin-sa # serviceaccount:namespace:operator + - system:serviceaccount:rook-ceph:nvmeof-csi-provisioner # serviceaccount:namespace:operator --- # Rook Ceph Operator Config # Use this ConfigMap to override operator configurations @@ -126,430 +118,6 @@ data: # Allow using loop devices for osds in test clusters. ROOK_CEPH_ALLOW_LOOP_DEVICES: "false" - # Enable CSI Operator - ROOK_USE_CSI_OPERATOR: "true" - - # Enable the CSI driver. - # To run the non-default version of the CSI driver, see the override-able image properties in operator.yaml - ROOK_CSI_ENABLE_CEPHFS: "true" - # Enable the default version of the CSI RBD driver. To start another version of the CSI driver, see image properties below. - ROOK_CSI_ENABLE_RBD: "true" - # Enable the CSI NFS driver. To start another version of the CSI driver, see image properties below. - ROOK_CSI_ENABLE_NFS: "false" - # Disable the CSI driver. - ROOK_CSI_DISABLE_DRIVER: "false" - - # Set to true to enable Ceph CSI pvc encryption support. - CSI_ENABLE_ENCRYPTION: "false" - - # set to true to enable adding volume metadata on the CephFS subvolume and RBD images. - # CSI_ENABLE_METADATA: "true" - - # cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful in cases - # like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster. - # CSI_CLUSTER_NAME: "my-prod-cluster" - - # Set to true to enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary - # in some network configurations where the SDN does not provide access to an external cluster or - # there is significant drop in read/write performance. - # CSI_ENABLE_HOST_NETWORK: "true" - - # Set logging level for cephCSI containers maintained by the cephCSI. - # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. - # CSI_LOG_LEVEL: "0" - - # Set logging level for Kubernetes-csi sidecar containers. - # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. - # CSI_SIDECAR_LOG_LEVEL: "0" - - # Set replicas for csi provisioner deployment. - CSI_PROVISIONER_REPLICAS: "2" - - # OMAP generator generates the omap mapping between the PV name and the RBD image - # which helps CSI to identify the rbd images for CSI operations. - # CSI_ENABLE_OMAP_GENERATOR need to be enabled when we are using rbd mirroring feature. - # By default OMAP generator is disabled and when enabled it willbe deployed as a - # sidecar with CSI provisioner pod, to enable set it to true. - # CSI_ENABLE_OMAP_GENERATOR: "true" - - # set to false to disable deployment of snapshotter container in CephFS provisioner pod. - CSI_ENABLE_CEPHFS_SNAPSHOTTER: "true" - - # set to false to disable deployment of snapshotter container in NFS provisioner pod. - CSI_ENABLE_NFS_SNAPSHOTTER: "true" - - # set to false to disable deployment of snapshotter container in RBD provisioner pod. - CSI_ENABLE_RBD_SNAPSHOTTER: "true" - - # Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs - # If you disable the kernel client, your application may be disrupted during upgrade. - # See the upgrade guide: https://rook.io/docs/rook/latest/ceph-upgrade.html - CSI_FORCE_CEPHFS_KERNEL_CLIENT: "true" - - # (Optional) policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted. - # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html - CSI_RBD_FSGROUPPOLICY: "File" - - # (Optional) policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted. - # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html - CSI_CEPHFS_FSGROUPPOLICY: "File" - - # (Optional) policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted. - # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html - CSI_NFS_FSGROUPPOLICY: "File" - - # (Optional) control the host mount of /etc/selinux for csi plugin pods. - CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT: "false" - - # The default version of CSI supported by Rook will be started. To change the version - # of the CSI driver to something other than what is officially supported, change - # these images to the desired release of the CSI driver. - # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.16.2" - # ROOK_CSI_REGISTRAR_IMAGE: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.16.0" - # ROOK_CSI_RESIZER_IMAGE: "registry.k8s.io/sig-storage/csi-resizer:v2.1.0" - # ROOK_CSI_PROVISIONER_IMAGE: "registry.k8s.io/sig-storage/csi-provisioner:v6.1.1" - # ROOK_CSI_SNAPSHOTTER_IMAGE: "registry.k8s.io/sig-storage/csi-snapshotter:v8.5.0" - # ROOK_CSI_ATTACHER_IMAGE: "registry.k8s.io/sig-storage/csi-attacher:v4.11.0" - - # (Optional) set user created priorityclassName for csi plugin pods. - CSI_PLUGIN_PRIORITY_CLASSNAME: "system-node-critical" - - # (Optional) set user created priorityclassName for csi provisioner pods. - CSI_PROVISIONER_PRIORITY_CLASSNAME: "system-cluster-critical" - - # CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. - # Default value is RollingUpdate. - # CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY: "OnDelete" - # A maxUnavailable parameter of CSI cephFS plugin daemonset update strategy. - # Default value is 1. - # CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE: "1" - # CSI RBD plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. - # Default value is RollingUpdate. - # CSI_RBD_PLUGIN_UPDATE_STRATEGY: "OnDelete" - # A maxUnavailable parameter of CSI RBD plugin daemonset update strategy. - # Default value is 1. - # CSI_RBD_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE: "1" - - # CSI NFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. - # Default value is RollingUpdate. - # CSI_NFS_PLUGIN_UPDATE_STRATEGY: "OnDelete" - - # kubelet directory path, if kubelet configured to use other than /var/lib/kubelet path. - # ROOK_CSI_KUBELET_DIR_PATH: "/var/lib/kubelet" - - # Labels to add to the CSI CephFS Deployments and DaemonSets Pods. - # ROOK_CSI_CEPHFS_POD_LABELS: "key1=value1,key2=value2" - # Labels to add to the CSI RBD Deployments and DaemonSets Pods. - # ROOK_CSI_RBD_POD_LABELS: "key1=value1,key2=value2" - # Labels to add to the CSI NFS Deployments and DaemonSets Pods. - # ROOK_CSI_NFS_POD_LABELS: "key1=value1,key2=value2" - - # (Optional) CephCSI CephFS plugin Volumes - # CSI_CEPHFS_PLUGIN_VOLUME: | - # - name: lib-modules - # hostPath: - # path: /run/current-system/kernel-modules/lib/modules/ - # - name: host-nix - # hostPath: - # path: /nix - - # (Optional) CephCSI CephFS plugin Volume mounts - # CSI_CEPHFS_PLUGIN_VOLUME_MOUNT: | - # - name: host-nix - # mountPath: /nix - # readOnly: true - - # (Optional) CephCSI RBD plugin Volumes - # CSI_RBD_PLUGIN_VOLUME: | - # - name: lib-modules - # hostPath: - # path: /run/current-system/kernel-modules/lib/modules/ - # - name: host-nix - # hostPath: - # path: /nix - - # (Optional) CephCSI RBD plugin Volume mounts - # CSI_RBD_PLUGIN_VOLUME_MOUNT: | - # - name: host-nix - # mountPath: /nix - # readOnly: true - - # (Optional) CephCSI provisioner NodeAffinity(applied to both CephFS and RBD provisioner). - # CSI_PROVISIONER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" - # (Optional) CephCSI provisioner tolerations list(applied to both CephFS and RBD provisioner). - # Put here list of taints you want to tolerate in YAML format. - # CSI provisioner would be best to start on the same nodes as other ceph daemons. - # CSI_PROVISIONER_TOLERATIONS: | - # - effect: NoSchedule - # key: node-role.kubernetes.io/control-plane - # operator: Exists - # - effect: NoExecute - # key: node-role.kubernetes.io/etcd - # operator: Exists - # (Optional) CephCSI plugin NodeAffinity(applied to both CephFS and RBD plugin). - # CSI_PLUGIN_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" - # (Optional) CephCSI plugin tolerations list(applied to both CephFS and RBD plugin). - # Put here list of taints you want to tolerate in YAML format. - # CSI plugins need to be started on all the nodes where the clients need to mount the storage. - # CSI_PLUGIN_TOLERATIONS: | - # - effect: NoSchedule - # key: node-role.kubernetes.io/control-plane - # operator: Exists - # - effect: NoExecute - # key: node-role.kubernetes.io/etcd - # operator: Exists - - # (Optional) CephCSI RBD provisioner NodeAffinity(if specified, overrides CSI_PROVISIONER_NODE_AFFINITY). - # CSI_RBD_PROVISIONER_NODE_AFFINITY: "role=rbd-node" - # (Optional) CephCSI RBD provisioner tolerations list(if specified, overrides CSI_PROVISIONER_TOLERATIONS). - # Put here list of taints you want to tolerate in YAML format. - # CSI provisioner would be best to start on the same nodes as other ceph daemons. - # CSI_RBD_PROVISIONER_TOLERATIONS: | - # - key: node.rook.io/rbd - # operator: Exists - # (Optional) CephCSI RBD plugin NodeAffinity(if specified, overrides CSI_PLUGIN_NODE_AFFINITY). - # CSI_RBD_PLUGIN_NODE_AFFINITY: "role=rbd-node" - # (Optional) CephCSI RBD plugin tolerations list(if specified, overrides CSI_PLUGIN_TOLERATIONS). - # Put here list of taints you want to tolerate in YAML format. - # CSI plugins need to be started on all the nodes where the clients need to mount the storage. - # CSI_RBD_PLUGIN_TOLERATIONS: | - # - key: node.rook.io/rbd - # operator: Exists - - # (Optional) CephCSI CephFS provisioner NodeAffinity(if specified, overrides CSI_PROVISIONER_NODE_AFFINITY). - # CSI_CEPHFS_PROVISIONER_NODE_AFFINITY: "role=cephfs-node" - # (Optional) CephCSI CephFS provisioner tolerations list(if specified, overrides CSI_PROVISIONER_TOLERATIONS). - # Put here list of taints you want to tolerate in YAML format. - # CSI provisioner would be best to start on the same nodes as other ceph daemons. - # CSI_CEPHFS_PROVISIONER_TOLERATIONS: | - # - key: node.rook.io/cephfs - # operator: Exists - # (Optional) CephCSI CephFS plugin NodeAffinity(if specified, overrides CSI_PLUGIN_NODE_AFFINITY). - # CSI_CEPHFS_PLUGIN_NODE_AFFINITY: "role=cephfs-node" - # (Optional) CephCSI CephFS plugin tolerations list(if specified, overrides CSI_PLUGIN_TOLERATIONS). - # Put here list of taints you want to tolerate in YAML format. - # CSI plugins need to be started on all the nodes where the clients need to mount the storage. - # CSI_CEPHFS_PLUGIN_TOLERATIONS: | - # - key: node.rook.io/cephfs - # operator: Exists - - # (Optional) CephCSI NFS provisioner NodeAffinity (overrides CSI_PROVISIONER_NODE_AFFINITY). - # CSI_NFS_PROVISIONER_NODE_AFFINITY: "role=nfs-node" - # (Optional) CephCSI NFS provisioner tolerations list (overrides CSI_PROVISIONER_TOLERATIONS). - # Put here list of taints you want to tolerate in YAML format. - # CSI provisioner would be best to start on the same nodes as other ceph daemons. - # CSI_NFS_PROVISIONER_TOLERATIONS: | - # - key: node.rook.io/nfs - # operator: Exists - # (Optional) CephCSI NFS plugin NodeAffinity (overrides CSI_PLUGIN_NODE_AFFINITY). - # CSI_NFS_PLUGIN_NODE_AFFINITY: "role=nfs-node" - # (Optional) CephCSI NFS plugin tolerations list (overrides CSI_PLUGIN_TOLERATIONS). - # Put here list of taints you want to tolerate in YAML format. - # CSI plugins need to be started on all the nodes where the clients need to mount the storage. - # CSI_NFS_PLUGIN_TOLERATIONS: | - # - key: node.rook.io/nfs - # operator: Exists - - # (Optional) CEPH CSI RBD provisioner resource requirement list, Put here list of resource - # requests and limits you want to apply for provisioner pod - #CSI_RBD_PROVISIONER_RESOURCE: | - # - name : csi-provisioner - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-resizer - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-attacher - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-snapshotter - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-rbdplugin - # resource: - # requests: - # memory: 512Mi - # cpu: 250m - # limits: - # memory: 1Gi - # - name : csi-omap-generator - # resource: - # requests: - # memory: 512Mi - # cpu: 250m - # limits: - # memory: 1Gi - # - name : liveness-prometheus - # resource: - # requests: - # memory: 128Mi - # cpu: 50m - # limits: - # memory: 256Mi - # (Optional) CEPH CSI RBD plugin resource requirement list, Put here list of resource - # requests and limits you want to apply for plugin pod - #CSI_RBD_PLUGIN_RESOURCE: | - # - name : driver-registrar - # resource: - # requests: - # memory: 128Mi - # cpu: 50m - # limits: - # memory: 256Mi - # - name : csi-rbdplugin - # resource: - # requests: - # memory: 512Mi - # cpu: 250m - # limits: - # memory: 1Gi - # - name : liveness-prometheus - # resource: - # requests: - # memory: 128Mi - # cpu: 50m - # limits: - # memory: 256Mi - # (Optional) CEPH CSI CephFS provisioner resource requirement list, Put here list of resource - # requests and limits you want to apply for provisioner pod - #CSI_CEPHFS_PROVISIONER_RESOURCE: | - # - name : csi-provisioner - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-resizer - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-attacher - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-snapshotter - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-cephfsplugin - # resource: - # requests: - # memory: 512Mi - # cpu: 250m - # limits: - # memory: 1Gi - # - name : liveness-prometheus - # resource: - # requests: - # memory: 128Mi - # cpu: 50m - # limits: - # memory: 256Mi - # (Optional) CEPH CSI CephFS plugin resource requirement list, Put here list of resource - # requests and limits you want to apply for plugin pod - #CSI_CEPHFS_PLUGIN_RESOURCE: | - # - name : driver-registrar - # resource: - # requests: - # memory: 128Mi - # cpu: 50m - # limits: - # memory: 256Mi - # - name : csi-cephfsplugin - # resource: - # requests: - # memory: 512Mi - # cpu: 250m - # limits: - # memory: 1Gi - # - name : liveness-prometheus - # resource: - # requests: - # memory: 128Mi - # cpu: 50m - # limits: - # memory: 256Mi - - # (Optional) CEPH CSI NFS provisioner resource requirement list, Put here list of resource - # requests and limits you want to apply for provisioner pod - # CSI_NFS_PROVISIONER_RESOURCE: | - # - name : csi-provisioner - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-nfsplugin - # resource: - # requests: - # memory: 512Mi - # cpu: 250m - # limits: - # memory: 1Gi - # - name : csi-attacher - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # (Optional) CEPH CSI NFS plugin resource requirement list, Put here list of resource - # requests and limits you want to apply for plugin pod - # CSI_NFS_PLUGIN_RESOURCE: | - # - name : driver-registrar - # resource: - # requests: - # memory: 128Mi - # cpu: 50m - # limits: - # memory: 256Mi - # - name : csi-nfsplugin - # resource: - # requests: - # memory: 512Mi - # cpu: 250m - # limits: - # memory: 1Gi - - # Configure CSI CephFS liveness metrics port - # Set to true to enable Ceph CSI liveness container. - CSI_ENABLE_LIVENESS: "false" - # CSI_CEPHFS_LIVENESS_METRICS_PORT: "9081" - # Configure CSI RBD liveness metrics port - # CSI_RBD_LIVENESS_METRICS_PORT: "9080" - # CSIADDONS_PORT: "9070" - - # Set CephFS Kernel mount options to use https://docs.ceph.com/en/latest/man/8/mount.ceph/#options - # Set to "ms_mode=secure" when connections.encrypted is enabled in CephCluster CR - # CSI_CEPHFS_KERNEL_MOUNT_OPTIONS: "ms_mode=secure" - # Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true" @@ -567,53 +135,6 @@ data: ROOK_ENABLE_DISCOVERY_DAEMON: "false" # The timeout value (in seconds) of Ceph commands. It should be >= 1. If this variable is not set or is an invalid value, it's default to 15. ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS: "15" - # Enable the csi addons sidecar. - CSI_ENABLE_CSIADDONS: "false" - # ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.14.0" - # The GCSI RPC timeout value (in seconds). It should be >= 120. If this variable is not set or is an invalid value, it's default to 150. - CSI_GRPC_TIMEOUT_SECONDS: "150" - - # set to false to disable volume group snapshot feature. This feature is - # enabled by default as long as the necessary CRDs are available in the cluster. - CSI_ENABLE_VOLUME_GROUP_SNAPSHOT: "true" - - # Enable topology based provisioning. - CSI_ENABLE_TOPOLOGY: "false" - # Domain labels define which node labels to use as domains - # for CSI nodeplugins to advertise their domains - # NOTE: the value here serves as an example and needs to be - # updated with node labels that define domains of interest - # CSI_TOPOLOGY_DOMAIN_LABELS: "kubernetes.io/hostname,topology.kubernetes.io/zone,topology.rook.io/rack" - - # Enable cross namespace volume data source provisioning. - CSI_ENABLE_CROSS_NAMESPACE_VOLUME_DATA_SOURCE: "false" - - # Whether to skip any attach operation altogether for CephCSI PVCs. - # See more details [here](https://kubernetes-csi.github.io/docs/skip-attach.html#skip-attach-with-csi-driver-object). - # If set to false it skips the volume attachments and makes the creation of pods using the CephCSI PVC fast. - # **WARNING** It's highly discouraged to use this for RWO volumes. for RBD PVC it can cause data corruption, - # csi-addons operations like Reclaimspace and PVC Keyrotation will also not be supported if set to false - # since we'll have no VolumeAttachments to determine which node the PVC is mounted on. - # Refer to this [issue](https://github.com/kubernetes/kubernetes/issues/103305) for more details. - CSI_CEPHFS_ATTACH_REQUIRED: "true" - CSI_RBD_ATTACH_REQUIRED: "true" - CSI_NFS_ATTACH_REQUIRED: "true" - - # (Optional) Duration in seconds that non-leader candidates will wait to force acquire leadership. Default to 137 seconds. - # CSI_LEADER_ELECTION_LEASE_DURATION: "137s" - - # (Optional) Deadline in seconds that the acting leader will retry refreshing leadership before giving up. Defaults to 107 seconds. - # CSI_LEADER_ELECTION_RENEW_DEADLINE: "107s" - - # (Optional) Retry period in seconds the LeaderElector clients should wait between tries of actions. Defaults to 26 seconds. - # CSI_LEADER_ELECTION_RETRY_PERIOD: "26s" - - # csi driver name prefix for cephfs, rbd and nfs. if not specified, default - # will be the namespace name where rook-ceph operator is deployed. - # search for `# csi-provisioner-name` in the storageclass and - # volumesnashotclass and update the name accordingly. - # CSI_DRIVER_NAME_PREFIX: "rook-ceph" - # Rook Discover toleration. Will tolerate all taints with all keys. # (Optional) Rook Discover tolerations list. Put here list of taints you want to tolerate in YAML format. # DISCOVER_TOLERATIONS: | @@ -647,12 +168,6 @@ data: # cpu: 200m # memory: 128Mi - # (Optional) Burst to use while communicating with the kubernetes apiserver. - # CSI_KUBE_API_BURST: "10" - - # (Optional) QPS to use while communicating with the kubernetes apiserver. - # CSI_KUBE_API_QPS: "5.0" - # Whether to create all Rook pods to run on the host network, for example in environments where a CNI is not enabled ROOK_ENFORCE_HOST_NETWORK: "false" diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 747d9b083..3e967b512 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -35,465 +35,6 @@ data: # you need Rook to leave custom CRUSH rules in place. ROOK_DELETE_UNUSED_CRUSH_RULES: "true" - # Enable CSI Operator - ROOK_USE_CSI_OPERATOR: "true" - # Enable the CSI driver. - # To run the non-default version of the CSI driver, see the override-able image properties in operator.yaml - ROOK_CSI_ENABLE_CEPHFS: "true" - # Enable the default version of the CSI RBD driver. To start another version of the CSI driver, see image properties below. - ROOK_CSI_ENABLE_RBD: "true" - # Enable the CSI NFS driver. To start another version of the CSI driver, see image properties below. - ROOK_CSI_ENABLE_NFS: "false" - # Disable the CSI driver. - ROOK_CSI_DISABLE_DRIVER: "false" - - # Set to true to enable Ceph CSI pvc encryption support. - CSI_ENABLE_ENCRYPTION: "false" - - # Set to true to enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary - # in some network configurations where the SDN does not provide access to an external cluster or - # there is significant drop in read/write performance. - # CSI_ENABLE_HOST_NETWORK: "true" - - # Set to true to enable adding volume metadata on the CephFS subvolume and RBD images. - # Not all users might be interested in getting volume/snapshot details as metadata on CephFS subvolume and RBD images. - # Hence enable metadata is false by default. - # CSI_ENABLE_METADATA: "true" - - # cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful in cases - # like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster. - # CSI_CLUSTER_NAME: "my-prod-cluster" - - # Set logging level for cephCSI containers maintained by the cephCSI. - # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. - # CSI_LOG_LEVEL: "0" - - # Set logging level for Kubernetes-csi sidecar containers. - # Supported values from 0 to 5. 0 for general useful logs (the default), 5 for trace level verbosity. - # CSI_SIDECAR_LOG_LEVEL: "0" - - # csi driver name prefix for cephfs, rbd and nfs. if not specified, default - # will be the namespace name where rook-ceph operator is deployed. - # search for `# csi-provisioner-name` in the storageclass and - # volumesnashotclass and update the name accordingly. - # CSI_DRIVER_NAME_PREFIX: "rook-ceph" - - # Set replicas for csi provisioner deployment. - CSI_PROVISIONER_REPLICAS: "2" - - # OMAP generator will generate the omap mapping between the PV name and the RBD image. - # CSI_ENABLE_OMAP_GENERATOR need to be enabled when we are using rbd mirroring feature. - # By default OMAP generator sidecar is deployed with CSI provisioner pod, to disable - # it set it to false. - # CSI_ENABLE_OMAP_GENERATOR: "false" - - # set to false to disable deployment of snapshotter container in CephFS provisioner pod. - CSI_ENABLE_CEPHFS_SNAPSHOTTER: "true" - - # set to false to disable deployment of snapshotter container in NFS provisioner pod. - CSI_ENABLE_NFS_SNAPSHOTTER: "true" - - # set to false to disable deployment of snapshotter container in RBD provisioner pod. - CSI_ENABLE_RBD_SNAPSHOTTER: "true" - - # set to false to disable volume group snapshot feature. This feature is - # enabled by default as long as the necessary CRDs are available in the cluster. - CSI_ENABLE_VOLUME_GROUP_SNAPSHOT: "true" - # Enable cephfs kernel driver instead of ceph-fuse. - # If you disable the kernel client, your application may be disrupted during upgrade. - # See the upgrade guide: https://rook.io/docs/rook/latest/ceph-upgrade.html - # NOTE! cephfs quota is not supported in kernel version < 4.17 - CSI_FORCE_CEPHFS_KERNEL_CLIENT: "true" - - # (Optional) policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted. - # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html - CSI_RBD_FSGROUPPOLICY: "File" - - # (Optional) policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted. - # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html - CSI_CEPHFS_FSGROUPPOLICY: "File" - - # (Optional) policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted. - # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html - CSI_NFS_FSGROUPPOLICY: "File" - - # (Optional) control the host mount of /etc/selinux for csi plugin pods. - CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT: "false" - - # The default version of CSI supported by Rook will be started. To change the version - # of the CSI driver to something other than what is officially supported, change - # these images to the desired release of the CSI driver. - # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.16.2" - # ROOK_CSI_REGISTRAR_IMAGE: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.16.0" - # ROOK_CSI_RESIZER_IMAGE: "registry.k8s.io/sig-storage/csi-resizer:v2.1.0" - # ROOK_CSI_PROVISIONER_IMAGE: "registry.k8s.io/sig-storage/csi-provisioner:v6.1.1" - # ROOK_CSI_SNAPSHOTTER_IMAGE: "registry.k8s.io/sig-storage/csi-snapshotter:v8.5.0" - # ROOK_CSI_ATTACHER_IMAGE: "registry.k8s.io/sig-storage/csi-attacher:v4.11.0" - - # To indicate the image pull policy to be applied to all the containers in the csi driver pods. - # ROOK_CSI_IMAGE_PULL_POLICY: "IfNotPresent" - - # (Optional) set user created priorityclassName for csi plugin pods. - CSI_PLUGIN_PRIORITY_CLASSNAME: "system-node-critical" - - # (Optional) set user created priorityclassName for csi provisioner pods. - CSI_PROVISIONER_PRIORITY_CLASSNAME: "system-cluster-critical" - - # CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. - # Default value is RollingUpdate. - # CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY: "OnDelete" - # A maxUnavailable parameter of CSI cephFS plugin daemonset update strategy. - # Default value is 1. - # CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE: "1" - # CSI RBD plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. - # Default value is RollingUpdate. - # CSI_RBD_PLUGIN_UPDATE_STRATEGY: "OnDelete" - # A maxUnavailable parameter of CSI RBD plugin daemonset update strategy. - # Default value is 1. - # CSI_RBD_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE: "1" - - # CSI NFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. - # Default value is RollingUpdate. - # CSI_NFS_PLUGIN_UPDATE_STRATEGY: "OnDelete" - - # kubelet directory path, if kubelet configured to use other than /var/lib/kubelet path. - # ROOK_CSI_KUBELET_DIR_PATH: "/var/lib/kubelet" - - # Labels to add to the CSI CephFS Deployments and DaemonSets Pods. - # ROOK_CSI_CEPHFS_POD_LABELS: "key1=value1,key2=value2" - # Labels to add to the CSI RBD Deployments and DaemonSets Pods. - # ROOK_CSI_RBD_POD_LABELS: "key1=value1,key2=value2" - # Labels to add to the CSI NFS Deployments and DaemonSets Pods. - # ROOK_CSI_NFS_POD_LABELS: "key1=value1,key2=value2" - - # (Optional) CephCSI CephFS plugin Volumes - # CSI_CEPHFS_PLUGIN_VOLUME: | - # - name: lib-modules - # hostPath: - # path: /run/current-system/kernel-modules/lib/modules/ - # - name: host-nix - # hostPath: - # path: /nix - - # (Optional) CephCSI CephFS plugin Volume mounts - # CSI_CEPHFS_PLUGIN_VOLUME_MOUNT: | - # - name: host-nix - # mountPath: /nix - # readOnly: true - - # (Optional) CephCSI RBD plugin Volumes - # CSI_RBD_PLUGIN_VOLUME: | - # - name: lib-modules - # hostPath: - # path: /run/current-system/kernel-modules/lib/modules/ - # - name: host-nix - # hostPath: - # path: /nix - - # (Optional) CephCSI RBD plugin Volume mounts - # CSI_RBD_PLUGIN_VOLUME_MOUNT: | - # - name: host-nix - # mountPath: /nix - # readOnly: true - - # (Optional) CephCSI provisioner NodeAffinity (applied to both CephFS and RBD provisioner). - # CSI_PROVISIONER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" - # (Optional) CephCSI provisioner tolerations list(applied to both CephFS and RBD provisioner). - # Put here list of taints you want to tolerate in YAML format. - # CSI provisioner would be best to start on the same nodes as other ceph daemons. - # CSI_PROVISIONER_TOLERATIONS: | - # - effect: NoSchedule - # key: node-role.kubernetes.io/control-plane - # operator: Exists - # - effect: NoExecute - # key: node-role.kubernetes.io/etcd - # operator: Exists - # (Optional) CephCSI plugin NodeAffinity (applied to both CephFS and RBD plugin). - # CSI_PLUGIN_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" - # (Optional) CephCSI plugin tolerations list(applied to both CephFS and RBD plugin). - # Put here list of taints you want to tolerate in YAML format. - # CSI plugins need to be started on all the nodes where the clients need to mount the storage. - # CSI_PLUGIN_TOLERATIONS: | - # - effect: NoSchedule - # key: node-role.kubernetes.io/control-plane - # operator: Exists - # - effect: NoExecute - # key: node-role.kubernetes.io/etcd - # operator: Exists - - # (Optional) CephCSI RBD provisioner NodeAffinity (if specified, overrides CSI_PROVISIONER_NODE_AFFINITY). - # CSI_RBD_PROVISIONER_NODE_AFFINITY: "role=rbd-node" - # (Optional) CephCSI RBD provisioner tolerations list(if specified, overrides CSI_PROVISIONER_TOLERATIONS). - # Put here list of taints you want to tolerate in YAML format. - # CSI provisioner would be best to start on the same nodes as other ceph daemons. - # CSI_RBD_PROVISIONER_TOLERATIONS: | - # - key: node.rook.io/rbd - # operator: Exists - # (Optional) CephCSI RBD plugin NodeAffinity (if specified, overrides CSI_PLUGIN_NODE_AFFINITY). - # CSI_RBD_PLUGIN_NODE_AFFINITY: "role=rbd-node" - # (Optional) CephCSI RBD plugin tolerations list(if specified, overrides CSI_PLUGIN_TOLERATIONS). - # Put here list of taints you want to tolerate in YAML format. - # CSI plugins need to be started on all the nodes where the clients need to mount the storage. - # CSI_RBD_PLUGIN_TOLERATIONS: | - # - key: node.rook.io/rbd - # operator: Exists - - # (Optional) CephCSI CephFS provisioner NodeAffinity (if specified, overrides CSI_PROVISIONER_NODE_AFFINITY). - # CSI_CEPHFS_PROVISIONER_NODE_AFFINITY: "role=cephfs-node" - # (Optional) CephCSI CephFS provisioner tolerations list(if specified, overrides CSI_PROVISIONER_TOLERATIONS). - # Put here list of taints you want to tolerate in YAML format. - # CSI provisioner would be best to start on the same nodes as other ceph daemons. - # CSI_CEPHFS_PROVISIONER_TOLERATIONS: | - # - key: node.rook.io/cephfs - # operator: Exists - # (Optional) CephCSI CephFS plugin NodeAffinity (if specified, overrides CSI_PLUGIN_NODE_AFFINITY). - # CSI_CEPHFS_PLUGIN_NODE_AFFINITY: "role=cephfs-node" - # NOTE: Support for defining NodeAffinity for operators other than "In" and "Exists" requires the user to input a - # valid v1.NodeAffinity JSON or YAML string. For example, the following is valid YAML v1.NodeAffinity: - # CSI_CEPHFS_PLUGIN_NODE_AFFINITY: | - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: myKey - # operator: DoesNotExist - # (Optional) CephCSI CephFS plugin tolerations list(if specified, overrides CSI_PLUGIN_TOLERATIONS). - # Put here list of taints you want to tolerate in YAML format. - # CSI plugins need to be started on all the nodes where the clients need to mount the storage. - # CSI_CEPHFS_PLUGIN_TOLERATIONS: | - # - key: node.rook.io/cephfs - # operator: Exists - - # (Optional) CephCSI NFS provisioner NodeAffinity (overrides CSI_PROVISIONER_NODE_AFFINITY). - # CSI_NFS_PROVISIONER_NODE_AFFINITY: "role=nfs-node" - # (Optional) CephCSI NFS provisioner tolerations list (overrides CSI_PROVISIONER_TOLERATIONS). - # Put here list of taints you want to tolerate in YAML format. - # CSI provisioner would be best to start on the same nodes as other ceph daemons. - # CSI_NFS_PROVISIONER_TOLERATIONS: | - # - key: node.rook.io/nfs - # operator: Exists - # (Optional) CephCSI NFS plugin NodeAffinity (overrides CSI_PLUGIN_NODE_AFFINITY). - # CSI_NFS_PLUGIN_NODE_AFFINITY: "role=nfs-node" - # (Optional) CephCSI NFS plugin tolerations list (overrides CSI_PLUGIN_TOLERATIONS). - # Put here list of taints you want to tolerate in YAML format. - # CSI plugins need to be started on all the nodes where the clients need to mount the storage. - # CSI_NFS_PLUGIN_TOLERATIONS: | - # - key: node.rook.io/nfs - # operator: Exists - - # (Optional) CEPH CSI RBD provisioner resource requirement list, Put here list of resource - # requests and limits you want to apply for provisioner pod - #CSI_RBD_PROVISIONER_RESOURCE: | - # - name : csi-provisioner - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-resizer - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-attacher - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-snapshotter - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-rbdplugin - # resource: - # requests: - # memory: 512Mi - # cpu: 250m - # limits: - # memory: 1Gi - # - name : csi-omap-generator - # resource: - # requests: - # memory: 512Mi - # cpu: 250m - # limits: - # memory: 1Gi - # - name : liveness-prometheus - # resource: - # requests: - # memory: 128Mi - # cpu: 50m - # limits: - # memory: 256Mi - # (Optional) CEPH CSI RBD plugin resource requirement list, Put here list of resource - # requests and limits you want to apply for plugin pod - #CSI_RBD_PLUGIN_RESOURCE: | - # - name : driver-registrar - # resource: - # requests: - # memory: 128Mi - # cpu: 50m - # limits: - # memory: 256Mi - # - name : csi-rbdplugin - # resource: - # requests: - # memory: 512Mi - # cpu: 250m - # limits: - # memory: 1Gi - # - name : liveness-prometheus - # resource: - # requests: - # memory: 128Mi - # cpu: 50m - # limits: - # memory: 256Mi - # (Optional) CEPH CSI CephFS provisioner resource requirement list, Put here list of resource - # requests and limits you want to apply for provisioner pod - #CSI_CEPHFS_PROVISIONER_RESOURCE: | - # - name : csi-provisioner - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-resizer - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-attacher - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-snapshotter - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-cephfsplugin - # resource: - # requests: - # memory: 512Mi - # cpu: 250m - # limits: - # memory: 1Gi - # - name : liveness-prometheus - # resource: - # requests: - # memory: 128Mi - # cpu: 50m - # limits: - # memory: 256Mi - # (Optional) CEPH CSI CephFS plugin resource requirement list, Put here list of resource - # requests and limits you want to apply for plugin pod - #CSI_CEPHFS_PLUGIN_RESOURCE: | - # - name : driver-registrar - # resource: - # requests: - # memory: 128Mi - # cpu: 50m - # limits: - # memory: 256Mi - # - name : csi-cephfsplugin - # resource: - # requests: - # memory: 512Mi - # cpu: 250m - # limits: - # memory: 1Gi - # - name : liveness-prometheus - # resource: - # requests: - # memory: 128Mi - # cpu: 50m - # limits: - # memory: 256Mi - - # (Optional) CEPH CSI NFS provisioner resource requirement list, Put here list of resource - # requests and limits you want to apply for provisioner pod - # CSI_NFS_PROVISIONER_RESOURCE: | - # - name : csi-provisioner - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # - name : csi-nfsplugin - # resource: - # requests: - # memory: 512Mi - # cpu: 250m - # limits: - # memory: 1Gi - # - name : csi-attacher - # resource: - # requests: - # memory: 128Mi - # cpu: 100m - # limits: - # memory: 256Mi - # (Optional) CEPH CSI NFS plugin resource requirement list, Put here list of resource - # requests and limits you want to apply for plugin pod - # CSI_NFS_PLUGIN_RESOURCE: | - # - name : driver-registrar - # resource: - # requests: - # memory: 128Mi - # cpu: 50m - # limits: - # memory: 256Mi - # - name : csi-nfsplugin - # resource: - # requests: - # memory: 512Mi - # cpu: 250m - # limits: - # memory: 1Gi - - # Configure CSI CephFS liveness metrics port - # Set to true to enable Ceph CSI liveness container. - CSI_ENABLE_LIVENESS: "false" - # CSI_CEPHFS_LIVENESS_METRICS_PORT: "9081" - # Configure CSI RBD liveness metrics port - # CSI_RBD_LIVENESS_METRICS_PORT: "9080" - - # We can override the ports for each individual component that uses the CSIADDONS sidecar - # This is useful if we're running in hostNetwork, where ports may conflict on the same host - # CSIADDONS_PORT: "9070" - # CSIADDONS_RBD_PROVISIONER_PORT: "9070" - # CSIADDONS_CEPHFS_PROVISIONER_PORT: "9070" - - # Set CephFS Kernel mount options to use https://docs.ceph.com/en/latest/man/8/mount.ceph/#options - # Set to "ms_mode=secure" when connections.encrypted is enabled in CephCluster CR - # CSI_CEPHFS_KERNEL_MOUNT_OPTIONS: "ms_mode=secure" - - # (Optional) Duration in seconds that non-leader candidates will wait to force acquire leadership. Default to 137 seconds. - # CSI_LEADER_ELECTION_LEASE_DURATION: "137s" - - # (Optional) Deadline in seconds that the acting leader will retry refreshing leadership before giving up. Defaults to 107 seconds. - # CSI_LEADER_ELECTION_RENEW_DEADLINE: "107s" - - # (Optional) Retry Period in seconds the LeaderElector clients should wait between tries of actions. Defaults to 26 seconds. - # CSI_LEADER_ELECTION_RETRY_PERIOD: "26s" - # Whether the OBC provisioner should watch on the ceph cluster namespace or not, if not default provisioner value is set ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true" @@ -511,33 +52,6 @@ data: ROOK_ENABLE_DISCOVERY_DAEMON: "false" # The timeout value (in seconds) of Ceph commands. It should be >= 1. If this variable is not set or is an invalid value, it's default to 15. ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS: "15" - # Enable the csi addons sidecar. - CSI_ENABLE_CSIADDONS: "false" - # ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.14.0" - # The CSI GRPC timeout value (in seconds). It should be >= 120. If this variable is not set or is an invalid value, it's default to 150. - CSI_GRPC_TIMEOUT_SECONDS: "150" - - # Enable topology based provisioning. - CSI_ENABLE_TOPOLOGY: "false" - # Domain labels define which node labels to use as domains - # for CSI nodeplugins to advertise their domains - # NOTE: the value here serves as an example and needs to be - # updated with node labels that define domains of interest - # CSI_TOPOLOGY_DOMAIN_LABELS: "kubernetes.io/hostname,topology.kubernetes.io/zone,topology.rook.io/rack" - - # Enable cross namespace volume data source provisioning. - CSI_ENABLE_CROSS_NAMESPACE_VOLUME_DATA_SOURCE: "false" - - # Whether to skip any attach operation altogether for CephCSI PVCs. - # See more details [here](https://kubernetes-csi.github.io/docs/skip-attach.html#skip-attach-with-csi-driver-object). - # If set to false it skips the volume attachments and makes the creation of pods using the CephCSI PVC fast. - # **WARNING** It's highly discouraged to use this for RWO volumes. for RBD PVC it can cause data corruption, - # csi-addons operations like Reclaimspace and PVC Keyrotation will also not be supported if set to false - # since we'll have no VolumeAttachments to determine which node the PVC is mounted on. - # Refer to this [issue](https://github.com/kubernetes/kubernetes/issues/103305) for more details. - CSI_CEPHFS_ATTACH_REQUIRED: "true" - CSI_RBD_ATTACH_REQUIRED: "true" - CSI_NFS_ATTACH_REQUIRED: "true" # Rook Discover toleration. Will tolerate all taints with all keys. # (Optional) Rook Discover tolerations list. Put here list of taints you want to tolerate in YAML format. # DISCOVER_TOLERATIONS: | @@ -571,12 +85,6 @@ data: # cpu: 100m # memory: 128Mi - # (Optional) Burst to use while communicating with the kubernetes apiserver. - # CSI_KUBE_API_BURST: "10" - - # (Optional) QPS to use while communicating with the kubernetes apiserver. - # CSI_KUBE_API_QPS: "5.0" - # Whether to create all Rook pods to run on the host network, for example in environments where a CNI is not enabled ROOK_ENFORCE_HOST_NETWORK: "false" @@ -586,6 +94,79 @@ data: # Custom label to identify node hostname. If not set `kubernetes.io/hostname` will be used ROOK_CUSTOM_HOSTNAME_LABEL: "" --- +# CSI operator resources managed by the admin. +# These CRs define the configuration for CSI drivers deployed by the ceph-csi-operator. +# Customize these as needed for your environment. + +# ImageSet ConfigMap defines the container images used by the CSI drivers. +apiVersion: v1 +kind: ConfigMap +metadata: + name: rook-csi-operator-image-set-configmap + namespace: rook-ceph # namespace:operator +data: + provisioner: "registry.k8s.io/sig-storage/csi-provisioner:v6.1.1" + attacher: "registry.k8s.io/sig-storage/csi-attacher:v4.11.0" + resizer: "registry.k8s.io/sig-storage/csi-resizer:v2.1.0" + snapshotter: "registry.k8s.io/sig-storage/csi-snapshotter:v8.5.0" + registrar: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.16.0" + plugin: "quay.io/cephcsi/cephcsi:v3.16.2" + addons: "quay.io/csiaddons/k8s-sidecar:v0.14.0" +--- +# OperatorConfig defines the default settings for all CSI drivers. +apiVersion: csi.ceph.io/v1 +kind: OperatorConfig +metadata: + name: ceph-csi-operator-config + namespace: rook-ceph # namespace:operator +spec: + driverSpecDefaults: + log: + verbosity: 0 + imageSet: + name: rook-csi-operator-image-set-configmap + enableMetadata: true + generateOMapInfo: false + fsGroupPolicy: File + deployCsiAddons: true + cephFsClientType: kernel + nodePlugin: + priorityClassName: "system-node-critical" + kubeletDirPath: /var/lib/kubelet + enableSeLinuxHostMount: false + controllerPlugin: + priorityClassName: "system-cluster-critical" + replicas: 1 + deploymentStrategy: + type: Recreate +--- +# RBD CSI Driver +apiVersion: csi.ceph.io/v1 +kind: Driver +metadata: + name: rook-ceph.rbd.csi.ceph.com # csi-provisioner-name + namespace: rook-ceph # namespace:operator +spec: + fsGroupPolicy: File + nodePlugin: + updateStrategy: + type: RollingUpdate + controllerPlugin: {} +--- +# CephFS CSI Driver +apiVersion: csi.ceph.io/v1 +kind: Driver +metadata: + name: rook-ceph.cephfs.csi.ceph.com # csi-provisioner-name + namespace: rook-ceph # namespace:operator +spec: + fsGroupPolicy: File + cephFsClientType: kernel + nodePlugin: + updateStrategy: + type: RollingUpdate + controllerPlugin: {} +--- # OLM: BEGIN OPERATOR DEPLOYMENT apiVersion: apps/v1 kind: Deployment diff --git a/images/ceph/Makefile b/images/ceph/Makefile index 67e666c24..dd47a7400 100755 --- a/images/ceph/Makefile +++ b/images/ceph/Makefile @@ -88,12 +88,10 @@ IMAGE_TMP="/tmp/rook-ceph-image-list" list-image: ## Create a list of images for offline installation @echo "producing list of images for offline installation" rm -f $(IMAGE_TMP) - awk '/image:/ {print $2}' $(MANIFESTS_DIR)/operator.yaml $(MANIFESTS_DIR)/cluster.yaml $(MANIFESTS_DIR)/csi-operator.yaml | \ - cut -d: -f2- | tee $(IMAGE_TMP) - awk '/quay.io/ || /registry.k8s.io/ {print $3}' ../../pkg/operator/ceph/csi/spec.go | \ - cut -d= -f2- | tr -d '"' | tee -a $(IMAGE_TMP) - awk '/quay.io/ || /gcr.io/ {print $4}' ../../pkg/operator/ceph/object/cosi/spec.go | \ - cut -d= -f2- | tr -d '"' | tee -a $(IMAGE_TMP) + awk '/image:/ && NF > 1 {print $$2}' $(MANIFESTS_DIR)/operator.yaml $(MANIFESTS_DIR)/cluster.yaml $(MANIFESTS_DIR)/csi-operator.yaml >$(IMAGE_TMP) + grep -E '^[[:space:]]+(provisioner|attacher|resizer|snapshotter|registrar|plugin|addons):' $(MANIFESTS_DIR)/operator.yaml | sed -E 's/.*"([^"]+)".*/\1/' >>$(IMAGE_TMP) + test ! -f ../../pkg/operator/ceph/csi/spec.go || grep -E 'quay\.io|registry\.k8s\.io' ../../pkg/operator/ceph/csi/spec.go | sed -E 's/.*"([^"]+)".*/\1/' >>$(IMAGE_TMP) + grep -E 'quay\.io|gcr\.io' ../../pkg/operator/ceph/object/cosi/spec.go | sed -E 's/.*"([^"]+)".*/\1/' >>$(IMAGE_TMP) rm -f $(MANIFESTS_DIR)/images.txt - cat $(IMAGE_TMP) | sort -h | uniq | grep -v '^[[:space:]]*$$' | tee $(MANIFESTS_DIR)/images.txt + sort -h $(IMAGE_TMP) | uniq | grep -v '^[[:space:]]*$$' >$(MANIFESTS_DIR)/images.txt rm -f $(IMAGE_TMP) diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 0c2480ff7..fcd721703 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -249,11 +249,6 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error { go cluster.reportTelemetry() } - err := csi.SaveCSIDriverOptions(c.context.Clientset, cluster.Namespace, cluster.ClusterInfo) - if err != nil { - return errors.Wrap(err, "failed to save CSI driver options") - } - // Populate ClusterInfo with the last value cluster.mons.ClusterInfo = cluster.ClusterInfo cluster.mons.ClusterInfo.SetName(cluster.namespacedName.Name) diff --git a/pkg/operator/ceph/cluster/cluster_external.go b/pkg/operator/ceph/cluster/cluster_external.go index 925a09282..ece93368b 100644 --- a/pkg/operator/ceph/cluster/cluster_external.go +++ b/pkg/operator/ceph/cluster/cluster_external.go @@ -20,7 +20,6 @@ package cluster import ( "context" - cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes" "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" @@ -115,20 +114,6 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error } } - // Save CSI configmap - monEndpoints := csi.MonEndpoints(cluster.ClusterInfo.InternalMonitors, cluster.Spec.RequireMsgr2()) - csiConfigEntry := &csi.CSIClusterConfigEntry{ - Namespace: cluster.ClusterInfo.Namespace, - ClusterInfo: cephcsi.ClusterInfo{ - Monitors: monEndpoints, - }, - } - - clusterId := cluster.namespacedName.Namespace // cluster id is same as cluster namespace for CephClusters - err = csi.SaveClusterConfig(c.context.Clientset, clusterId, cluster.namespacedName.Namespace, cluster.ClusterInfo, csiConfigEntry) - if err != nil { - return errors.Wrap(err, "failed to update csi cluster config") - } log.NamespacedInfo(cluster.Namespace, logger, "successfully updated csi config map") // Create Crash Collector Secret @@ -169,16 +154,14 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error } } - if csi.EnableCSIOperator() { - log.NamespacedInfo(cluster.Namespace, logger, "create cephConnection and defaultClientProfile for external mode") - err = csi.CreateUpdateCephConnection(c.context.Client, cluster.ClusterInfo, *cluster.Spec) - if err != nil { - return errors.Wrap(err, "failed to create/update cephConnection") - } - err = csi.CreateDefaultClientProfile(c.context.Client, cluster.ClusterInfo) - if err != nil { - return errors.Wrap(err, "failed to create/update default client profile") - } + log.NamespacedInfo(cluster.Namespace, logger, "create cephConnection and defaultClientProfile for external mode") + err = csi.CreateUpdateCephConnection(c.context.Client, cluster.ClusterInfo, *cluster.Spec) + if err != nil { + return errors.Wrap(err, "failed to create/update cephConnection") + } + err = csi.CreateDefaultClientProfile(c.context.Client, cluster.ClusterInfo) + if err != nil { + return errors.Wrap(err, "failed to create/update default client profile") } opcontroller.UpdateCondition(c.OpManagerCtx, c.context, cluster.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionConnected, v1.ConditionTrue, cephv1.ClusterConnectedReason, "Cluster connected successfully") diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 233782308..6d0a6098a 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -547,13 +547,7 @@ func (c *ClusterController) requestClusterDelete(clusterObj *cephv1.CephCluster) } func (c *ClusterController) checkIfVolumesExist(cluster *cephv1.CephCluster) error { - if csi.CSIEnabled() { - err := c.csiVolumesAllowForDeletion(cluster) - if err != nil { - return err - } - } - return nil + return c.csiVolumesAllowForDeletion(cluster) } func (c *ClusterController) csiVolumesAllowForDeletion(cluster *cephv1.CephCluster) error { diff --git a/pkg/operator/ceph/cluster/mon/health_test.go b/pkg/operator/ceph/cluster/mon/health_test.go index 06e67317b..0212bc37e 100644 --- a/pkg/operator/ceph/cluster/mon/health_test.go +++ b/pkg/operator/ceph/cluster/mon/health_test.go @@ -24,6 +24,7 @@ import ( "testing" "time" + csiopv1 "github.com/ceph/ceph-csi-operator/api/v1" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/client/clientset/versioned/scheme" "github.com/rook/rook/pkg/clusterd" @@ -62,7 +63,7 @@ func TestCheckHealth(t *testing.T) { objects := []runtime.Object{ getCephCluster("default", "default"), } - client := getClient(objects...) + client := getClient(t, objects...) clientset := test.New(t, 1) configDir := t.TempDir() context := &clusterd.Context{ @@ -451,7 +452,7 @@ func TestCheckHealthNotFound(t *testing.T) { objects := []runtime.Object{ getCephCluster("default", "default"), } - client := getClient(objects...) + client := getClient(t, objects...) clientset := test.New(t, 1) configDir := t.TempDir() context := &clusterd.Context{ @@ -524,7 +525,7 @@ func TestAddRemoveMons(t *testing.T) { objects := []runtime.Object{ getCephCluster("default", "default"), } - client := getClient(objects...) + client := getClient(t, objects...) clientset := test.New(t, 1) configDir := t.TempDir() context := &clusterd.Context{ @@ -1015,7 +1016,7 @@ func TestExternalMons_inSpec_notInQuorum(t *testing.T) { objects := []runtime.Object{ getCephCluster("default", "default"), } - client := getClient(objects...) + client := getClient(t, objects...) clientset := test.New(t, 1) configDir := t.TempDir() context := &clusterd.Context{ @@ -1150,7 +1151,7 @@ func TestExternalMons_inSpec_inQuorum(t *testing.T) { objects := []runtime.Object{ getCephCluster("default", "default"), } - client := getClient(objects...) + client := getClient(t, objects...) clientset := test.New(t, 1) configDir := t.TempDir() context := &clusterd.Context{ @@ -1297,11 +1298,12 @@ func TestExternalMons_inSpec_inQuorum(t *testing.T) { } } -func getClient(obj ...runtime.Object) client.Client { +func getClient(t *testing.T, obj ...runtime.Object) client.Client { // Register operator types with the runtime scheme. - scheme := scheme.Scheme - scheme.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}) - client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(obj...).Build() + s := scheme.Scheme + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}) + assert.NoError(t, csiopv1.AddToScheme(s)) + client := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(obj...).Build() return client } diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index fbd45e574..fda592f80 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -55,8 +55,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/retry" "k8s.io/utils/ptr" - - cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes" ) const ( @@ -1240,20 +1238,7 @@ func (c *Cluster) saveMonConfig() error { return errors.Wrap(err, "failed to write connection config for new mons") } - monEndpoints := csi.MonEndpoints(c.ClusterInfo.AllMonitors(), c.spec.RequireMsgr2()) - csiConfigEntry := &csi.CSIClusterConfigEntry{ - Namespace: c.ClusterInfo.Namespace, - ClusterInfo: cephcsi.ClusterInfo{ - Monitors: monEndpoints, - }, - } - - clusterId := c.Namespace // cluster id is same as cluster namespace for CephClusters - if err := csi.SaveClusterConfig(c.context.Clientset, clusterId, c.Namespace, c.ClusterInfo, csiConfigEntry); err != nil { - return errors.Wrap(err, "failed to update csi cluster config") - } - - if csi.EnableCSIOperator() && len(c.ClusterInfo.AllMonitors()) > 0 { + if len(c.ClusterInfo.AllMonitors()) > 0 { err := csi.CreateUpdateCephConnection(c.context.Client, c.ClusterInfo, c.spec) if err != nil { return errors.Wrap(err, "failed to create/update cephConnection") diff --git a/pkg/operator/ceph/cluster/mon/mon_test.go b/pkg/operator/ceph/cluster/mon/mon_test.go index fa40c923c..12110ab49 100644 --- a/pkg/operator/ceph/cluster/mon/mon_test.go +++ b/pkg/operator/ceph/cluster/mon/mon_test.go @@ -27,6 +27,7 @@ import ( "testing" "time" + csiopv1 "github.com/ceph/ceph-csi-operator/api/v1" "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -101,10 +102,14 @@ func newTestStartClusterWithQuorumResponse(t *testing.T, namespace string, monRe } }, } + s := scheme.Scheme + assert.NoError(t, csiopv1.AddToScheme(s)) + cl := fake.NewClientBuilder().WithScheme(s).Build() return &clusterd.Context{ Clientset: clientset, Executor: executor, ConfigDir: configDir, + Client: cl, }, nil } @@ -444,7 +449,10 @@ func TestSaveMonEndpoints(t *testing.T) { clientset := test.New(t, 1) configDir := t.TempDir() ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() - c := New(ctx, &clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", cephv1.ClusterSpec{}, ownerInfo) + s := scheme.Scheme + assert.NoError(t, csiopv1.AddToScheme(s)) + cl := fake.NewClientBuilder().WithScheme(s).Build() + c := New(ctx, &clusterd.Context{Clientset: clientset, ConfigDir: configDir, Client: cl}, "ns", cephv1.ClusterSpec{}, ownerInfo) setCommonMonProperties(c, 1, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") // create the initial config map @@ -491,7 +499,10 @@ func TestMaxMonID(t *testing.T) { clientset := test.New(t, 1) configDir := t.TempDir() ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() - c := New(context.TODO(), &clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", cephv1.ClusterSpec{}, ownerInfo) + s := scheme.Scheme + assert.NoError(t, csiopv1.AddToScheme(s)) + cl := fake.NewClientBuilder().WithScheme(s).Build() + c := New(context.TODO(), &clusterd.Context{Clientset: clientset, ConfigDir: configDir, Client: cl}, "ns", cephv1.ClusterSpec{}, ownerInfo) c.ClusterInfo = clienttest.CreateTestClusterInfo(1) // when the configmap is not found, the maxMonID is -1 @@ -1259,6 +1270,7 @@ func TestRotateMonCephxKeys(t *testing.T) { s := scheme.Scheme s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}) s.AddKnownTypes(v1.SchemeGroupVersion, &v1.Secret{}) + assert.NoError(t, csiopv1.AddToScheme(s)) object := []runtime.Object{ cluster, diff --git a/pkg/operator/ceph/cluster/predicate.go b/pkg/operator/ceph/cluster/predicate.go index 019c98def..bb89a2713 100644 --- a/pkg/operator/ceph/cluster/predicate.go +++ b/pkg/operator/ceph/cluster/predicate.go @@ -145,31 +145,12 @@ func predicateForOperatorConfigCMWatcher[T *corev1.ConfigMap]() predicate.TypedF return predicate.TypedFuncs[T]{ UpdateFunc: func(e event.TypedUpdateEvent[T]) bool { objNew := (*corev1.ConfigMap)(e.ObjectNew) - objOld := (*corev1.ConfigMap)(e.ObjectOld) - - if objNew.Name != opcontroller.OperatorSettingConfigMapName { - return false - } - - if objOld.Data["ROOK_USE_CSI_OPERATOR"] != objNew.Data["ROOK_USE_CSI_OPERATOR"] { - log.NamespacedInfo(objNew.Namespace, logger, "ROOK_USE_CSI_OPERATOR changed from %q to %q", objOld.Data["ROOK_USE_CSI_OPERATOR"], objNew.Data["ROOK_USE_CSI_OPERATOR"]) - return true - } - - return false + return objNew.Name == opcontroller.OperatorSettingConfigMapName }, CreateFunc: func(e event.TypedCreateEvent[T]) bool { objNew := (*corev1.ConfigMap)(e.Object) - if objNew.Name != opcontroller.OperatorSettingConfigMapName { - return false - } - - if objNew.Data["ROOK_USE_CSI_OPERATOR"] == "true" { - return true - } - - return false + return objNew.Name == opcontroller.OperatorSettingConfigMapName }, DeleteFunc: func(e event.TypedDeleteEvent[T]) bool { diff --git a/pkg/operator/ceph/csi/ceph_config.go b/pkg/operator/ceph/csi/ceph_config.go deleted file mode 100644 index 8d85d2bab..000000000 --- a/pkg/operator/ceph/csi/ceph_config.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2022 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "context" - - "github.com/pkg/errors" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" -) - -const ( - configName = "csi-ceph-conf-override" -) - -// checkCsiCephConfigMapExists checks if the csi-ceph-conf-override configmap -// exists in the given namespace. -func checkCsiCephConfigMapExists(ctx context.Context, clientset kubernetes.Interface, namespace string) (bool, error) { - _, err := clientset.CoreV1().ConfigMaps(namespace).Get(ctx, configName, metav1.GetOptions{}) - if err != nil { - if !k8serrors.IsNotFound(err) { - return false, errors.Wrapf(err, "failed to get csi ceph.conf configmap %q (in %q)", configName, namespace) - } - return false, nil - } - - logger.Infof("csi ceph.conf configmap %q exists", configName) - return true, nil -} diff --git a/pkg/operator/ceph/csi/ceph_config_test.go b/pkg/operator/ceph/csi/ceph_config_test.go deleted file mode 100644 index bffc4f893..000000000 --- a/pkg/operator/ceph/csi/ceph_config_test.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2022 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "context" - "testing" - - "github.com/rook/rook/pkg/daemon/ceph/client" - "github.com/rook/rook/pkg/operator/test" - - "github.com/stretchr/testify/assert" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func Test_checkCsiCephConfigMapExists(t *testing.T) { - namespace := "test-namespace" - ctx := context.TODO() - // create mocked client - clientset := test.New(t, 3) - - ok, err := checkCsiCephConfigMapExists(ctx, clientset, namespace) - assert.NoError(t, err) - assert.False(t, ok) - - configMap := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configName, - Namespace: namespace, - }, - } - configMap.Data = map[string]string{ - client.DefaultConfigFile: "", - } - _, err = clientset.CoreV1().ConfigMaps(namespace).Create(ctx, configMap, metav1.CreateOptions{}) - assert.NoError(t, err) - - ok, err = checkCsiCephConfigMapExists(ctx, clientset, namespace) - assert.NoError(t, err) - assert.True(t, ok) -} diff --git a/pkg/operator/ceph/csi/cluster_config.go b/pkg/operator/ceph/csi/cluster_config.go index 855a77d05..23fb3ffa7 100644 --- a/pkg/operator/ceph/csi/cluster_config.go +++ b/pkg/operator/ceph/csi/cluster_config.go @@ -20,13 +20,10 @@ import ( "context" "encoding/json" "fmt" - "os" "strings" - "sync" "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/k8sutil" v1 "k8s.io/api/core/v1" @@ -37,10 +34,7 @@ import ( cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes" ) -var ( - logger = capnslog.NewPackageLogger("github.com/rook/rook", "ceph-csi") - configMutex sync.Mutex -) +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "ceph-csi") type CSIClusterConfigEntry struct { cephcsi.ClusterInfo @@ -103,45 +97,14 @@ func MonEndpoints(mons map[string]*cephclient.MonInfo, requireMsgr2 bool) []stri return endpoints } -// updateNetNamespaceFilePath modify the netNamespaceFilePath for all cluster IDs. -// If holderEnabled is set to true. Otherwise, removes the netNamespaceFilePath value -// for all the clusterIDs. +// updateNetNamespaceFilePath clears the netNamespaceFilePath for all cluster IDs +// in the given namespace. With the ceph-csi-operator, holder pods are no longer used. func updateNetNamespaceFilePath(clusterNamespace string, cc csiClusterConfig) { - var ( - cephFSNetNamespaceFilePath string - rbdNetNamespaceFilePath string - nfsNetNamespaceFilePath string - ) - - if IsHolderEnabled() { - for _, centry := range cc { - if centry.Namespace == clusterNamespace && centry.ClusterID == clusterNamespace { - if centry.CephFS.NetNamespaceFilePath != "" { - cephFSNetNamespaceFilePath = centry.CephFS.NetNamespaceFilePath - } - if centry.RBD.NetNamespaceFilePath != "" { - rbdNetNamespaceFilePath = centry.RBD.NetNamespaceFilePath - } - if centry.NFS.NetNamespaceFilePath != "" { - nfsNetNamespaceFilePath = centry.NFS.NetNamespaceFilePath - } - } - } - - for i, centry := range cc { - if centry.Namespace == clusterNamespace { - cc[i].CephFS.NetNamespaceFilePath = cephFSNetNamespaceFilePath - cc[i].RBD.NetNamespaceFilePath = rbdNetNamespaceFilePath - cc[i].NFS.NetNamespaceFilePath = nfsNetNamespaceFilePath - } - } - } else { - for i := range cc { - if cc[i].Namespace == clusterNamespace { - cc[i].CephFS.NetNamespaceFilePath = "" - cc[i].RBD.NetNamespaceFilePath = "" - cc[i].NFS.NetNamespaceFilePath = "" - } + for i := range cc { + if cc[i].Namespace == clusterNamespace { + cc[i].CephFS.NetNamespaceFilePath = "" + cc[i].RBD.NetNamespaceFilePath = "" + cc[i].NFS.NetNamespaceFilePath = "" } } } @@ -311,137 +274,3 @@ func updateCsiConfigMapOwnerRefs(ctx context.Context, namespace string, clientse return nil } - -// SaveClusterConfig updates the config map used to provide ceph-csi with -// basic cluster configuration. The clusterID, clusterNamespace, and clusterInfo are -// used to determine what "cluster" in the config map will be updated. clusterID should be the same -// as clusterNamespace for CephClusters, but for other resources (e.g., CephBlockPoolRadosNamespace, -// CephFilesystemSubVolumeGroup) or for other supplementary entries, the clusterID should be unique -// and different from the namespace so as not to disrupt CephCluster configurations. -func SaveClusterConfig(clientset kubernetes.Interface, clusterID, clusterNamespace string, clusterInfo *cephclient.ClusterInfo, newCsiClusterConfigEntry *CSIClusterConfigEntry) error { - if EnableCSIOperator() { - logger.Debugf("csi-operator is enabled no need to save/update csi config in configmap %q", configName) - return nil - } - // csi is deployed into the same namespace as the operator - csiNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar) - if csiNamespace == "" { - logger.Warningf("cannot save csi config due to missing env var %q", k8sutil.PodNamespaceEnvVar) - return nil - } - logger.Debugf("using %q for csi configmap namespace", csiNamespace) - - if newCsiClusterConfigEntry != nil { - // set CSIDriverOptions - newCsiClusterConfigEntry.ReadAffinity.Enabled = ReadAffinityEnabled(clusterInfo.CSIDriverSpec.ReadAffinity.Enabled, clusterInfo.CephVersion) - newCsiClusterConfigEntry.ReadAffinity.CrushLocationLabels = clusterInfo.CSIDriverSpec.ReadAffinity.CrushLocationLabels - - newCsiClusterConfigEntry.CephFS.KernelMountOptions = clusterInfo.CSIDriverSpec.CephFS.KernelMountOptions - newCsiClusterConfigEntry.CephFS.FuseMountOptions = clusterInfo.CSIDriverSpec.CephFS.FuseMountOptions - } - - configMutex.Lock() - defer configMutex.Unlock() - - // fetch current ConfigMap contents - configMap, err := clientset.CoreV1().ConfigMaps(csiNamespace).Get(clusterInfo.Context, ConfigName, metav1.GetOptions{}) - if err != nil { - if k8serrors.IsNotFound(err) { - return errors.Wrap(err, "waiting for CSI config map to be created") - } - return errors.Wrap(err, "failed to fetch current csi config map") - } - - // update ConfigMap contents for current cluster - currData := configMap.Data[ConfigKey] - if currData == "" { - currData = "[]" - } - - newData, err := updateCsiClusterConfig(currData, clusterID, clusterInfo, newCsiClusterConfigEntry) - if err != nil { - return errors.Wrap(err, "failed to update csi config map data") - } - configMap.Data[ConfigKey] = newData - - // update ConfigMap with new contents - if _, err := clientset.CoreV1().ConfigMaps(csiNamespace).Update(clusterInfo.Context, configMap, metav1.UpdateOptions{}); err != nil { - return errors.Wrap(err, "failed to update csi config map") - } - - return nil -} - -// updateCSIDriverOptions updates the CSI driver options, including read affinity, kernel mount options -// and fuse mount options, for all entries belonging to the same cluster. -func updateCSIDriverOptions(curr string, clusterInfo *cephclient.ClusterInfo, - csiDriverOptions *cephv1.CSIDriverSpec, -) (string, error) { - cc, err := parseCsiClusterConfig(curr) - if err != nil { - return "", errors.Wrap(err, "failed to parse current csi cluster config") - } - - for i := range cc { - // If the clusterID belongs to the same cluster, update the entry. - if clusterInfo.Namespace == cc[i].Namespace { - cc[i].ReadAffinity.Enabled = ReadAffinityEnabled(csiDriverOptions.ReadAffinity.Enabled, clusterInfo.CephVersion) - cc[i].ReadAffinity.CrushLocationLabels = csiDriverOptions.ReadAffinity.CrushLocationLabels - - cc[i].CephFS.KernelMountOptions = csiDriverOptions.CephFS.KernelMountOptions - cc[i].CephFS.FuseMountOptions = csiDriverOptions.CephFS.FuseMountOptions - } - } - - updateNetNamespaceFilePath(clusterInfo.Namespace, cc) - return formatCsiClusterConfig(cc) -} - -// SaveCSIDriverOptions, similar to SaveClusterConfig, updates the config map used by ceph-csi -// with CSI driver options such as read affinity, kernel mount options and fuse mount options. -func SaveCSIDriverOptions(clientset kubernetes.Interface, clusterNamespace string, clusterInfo *cephclient.ClusterInfo) error { - if EnableCSIOperator() { - logger.Debugf("csi-operator is enabled no need to save/update csi config in configmap %q", configName) - return nil - } - - // csi is deployed into the same namespace as the operator - csiNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar) - if csiNamespace == "" { - logger.Warningf("cannot save csi config due to missing env var %q", k8sutil.PodNamespaceEnvVar) - return nil - } - logger.Debugf("using %q for csi configmap namespace", csiNamespace) - - configMutex.Lock() - defer configMutex.Unlock() - - // fetch current ConfigMap contents - configMap, err := clientset.CoreV1().ConfigMaps(csiNamespace).Get(clusterInfo.Context, ConfigName, metav1.GetOptions{}) - if err != nil { - return errors.Wrap(err, "failed to fetch current csi config map") - } - - // update ConfigMap contents for current cluster - currData := configMap.Data[ConfigKey] - if currData == "" { - currData = "[]" - } - - newData, err := updateCSIDriverOptions(currData, clusterInfo, &clusterInfo.CSIDriverSpec) - if err != nil { - return errors.Wrap(err, "failed to update csi config map data") - } - if currData == newData { - // no change - return nil - } - - // update ConfigMap with new contents - configMap.Data[ConfigKey] = newData - if _, err := clientset.CoreV1().ConfigMaps(csiNamespace).Update(clusterInfo.Context, configMap, metav1.UpdateOptions{}); err != nil { - return errors.Wrap(err, "failed to update csi config map with csi driver options") - } - - return nil -} diff --git a/pkg/operator/ceph/csi/cluster_config_test.go b/pkg/operator/ceph/csi/cluster_config_test.go index 0954a970b..adb2db01a 100644 --- a/pkg/operator/ceph/csi/cluster_config_test.go +++ b/pkg/operator/ceph/csi/cluster_config_test.go @@ -20,12 +20,10 @@ import ( "context" "encoding/json" "fmt" - "reflect" "strings" "testing" cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/cluster/osd/topology" "github.com/rook/rook/pkg/operator/k8sutil" @@ -471,7 +469,6 @@ func TestUpdateCsiClusterConfig(t *testing.T) { }) t.Run("test empty namespace correction and clear netNamespaceFilePath", func(t *testing.T) { - holderEnabled = false currentConfigFormatString := `[` + `{"clusterID":"cluster-id-and-namespace","monitors":["172.30.100.1:3300"],"cephFS":{"netNamespaceFilePath":"%s","subvolumeGroup":"","radosNamespace":"","kernelMountOptions":"","fuseMountOptions":""},"rbd":{"netNamespaceFilePath":"%s","radosNamespace":"","mirrorDaemonCount":0},"nfs":{"netNamespaceFilePath":""},"readAffinity":{"enabled":false,"crushLocationLabels":null},"namespace":"%s"},` + `{"clusterID":"5bb69c306a7d011c3e91c3cec112fb7a","monitors":["172.30.100.1:3300"],"cephFS":{"netNamespaceFilePath":"","subvolumeGroup":"csi","radosNamespace":"","kernelMountOptions":"","fuseMountOptions":""},"rbd":{"netNamespaceFilePath":"","radosNamespace":"","mirrorDaemonCount":0},"nfs":{"netNamespaceFilePath":""},"readAffinity":{"enabled":false,"crushLocationLabels":null},"namespace":"cluster-id-and-namespace"}` + @@ -560,399 +557,51 @@ func verifyEndpointPort(t *testing.T, endpoints []string, expectedPort string) { } } -func TestUpdateCSIDriverOptions(t *testing.T) { - type args struct { - clusterConfig csiClusterConfig - clusterKey string - csiDriverOptions *cephv1.CSIDriverSpec - } - holderEnabled = true - - tests := []struct { - name string - args args - want csiClusterConfig - wantErr bool - }{ - { - name: "empty current config", - args: args{ - clusterConfig: []CSIClusterConfigEntry{}, - clusterKey: "rook-ceph", - csiDriverOptions: &cephv1.CSIDriverSpec{}, - }, - want: []CSIClusterConfigEntry{}, - wantErr: false, - }, - { - name: "single matching current config", - args: args{ - clusterConfig: []CSIClusterConfigEntry{ - { - Namespace: "rook-ceph", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "rook-ceph", - Monitors: []string{"1.1.1.1"}, - CephFS: cephcsi.CephFS{ - NetNamespaceFilePath: "netnamespacefilepath", - }, - }, - }, - }, - clusterKey: "rook-ceph", - csiDriverOptions: &cephv1.CSIDriverSpec{ - ReadAffinity: cephv1.ReadAffinitySpec{ - Enabled: true, - CrushLocationLabels: []string{"topology.rook.io/rack"}, - }, - CephFS: cephv1.CSICephFSSpec{ - KernelMountOptions: "rw,noatime", - FuseMountOptions: "debug", - }, - }, - }, - want: []CSIClusterConfigEntry{ - { - Namespace: "rook-ceph", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "rook-ceph", - Monitors: []string{"1.1.1.1"}, - ReadAffinity: cephcsi.ReadAffinity{ - Enabled: true, - CrushLocationLabels: []string{"topology.rook.io/rack"}, - }, - CephFS: cephcsi.CephFS{ - KernelMountOptions: "rw,noatime", - FuseMountOptions: "debug", - NetNamespaceFilePath: "netnamespacefilepath", - }, - }, - }, - }, - wantErr: false, - }, - { - name: "multiple matching current config", - args: args{ - clusterConfig: []CSIClusterConfigEntry{ - { - Namespace: "rook-ceph", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "rook-ceph", - Monitors: []string{"1.1.1.1"}, - }, - }, - { - Namespace: "rook-ceph", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "rook-ceph-2", - Monitors: []string{"1.1.1.1"}, - }, - }, - { - Namespace: "rook-ceph-1", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "rook-ceph-3", - Monitors: []string{"1.1.1.1"}, - }, - }, - }, - clusterKey: "rook-ceph", - csiDriverOptions: &cephv1.CSIDriverSpec{ - ReadAffinity: cephv1.ReadAffinitySpec{ - Enabled: true, - CrushLocationLabels: []string{"topology.rook.io/rack"}, - }, - CephFS: cephv1.CSICephFSSpec{ - KernelMountOptions: "rw,noatime", - FuseMountOptions: "debug", - }, - }, - }, - want: []CSIClusterConfigEntry{ - { - Namespace: "rook-ceph", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "rook-ceph", - Monitors: []string{"1.1.1.1"}, - ReadAffinity: cephcsi.ReadAffinity{ - Enabled: true, - CrushLocationLabels: []string{"topology.rook.io/rack"}, - }, - CephFS: cephcsi.CephFS{ - KernelMountOptions: "rw,noatime", - FuseMountOptions: "debug", - }, - }, - }, - { - Namespace: "rook-ceph", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "rook-ceph-2", - Monitors: []string{"1.1.1.1"}, - ReadAffinity: cephcsi.ReadAffinity{ - Enabled: true, - CrushLocationLabels: []string{"topology.rook.io/rack"}, - }, - CephFS: cephcsi.CephFS{ - KernelMountOptions: "rw,noatime", - FuseMountOptions: "debug", - }, - }, - }, - { - Namespace: "rook-ceph-1", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "rook-ceph-3", - Monitors: []string{"1.1.1.1"}, - }, - }, - }, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - dataString, err := formatCsiClusterConfig(tt.args.clusterConfig) - assert.NoError(t, err) - got, err := updateCSIDriverOptions(dataString, &cephclient.ClusterInfo{Namespace: tt.args.clusterKey}, tt.args.csiDriverOptions) - if (err != nil) != tt.wantErr { - t.Errorf("updateCSIDriverOptions() error = %v, wantErr %v", err, tt.wantErr) - return - } - expectedString, err := formatCsiClusterConfig(tt.want) - assert.NoError(t, err) - assert.Equal(t, expectedString, got) - }) - } -} - func TestUpdateNetNamespaceFilePath(t *testing.T) { - type args struct { - clusterConfig csiClusterConfig - clusterKey string - holderEnabled bool - } + t.Run("empty config", func(t *testing.T) { + cc := csiClusterConfig{} + updateNetNamespaceFilePath("rook-ceph", cc) + assert.Empty(t, cc) + }) - tests := []struct { - name string - args args - want csiClusterConfig - }{ - { - name: "empty", - args: args{ - clusterKey: "rook-ceph", - holderEnabled: false, - clusterConfig: []CSIClusterConfigEntry{}, - }, - want: []CSIClusterConfigEntry{}, - }, - { - name: "holder enabled", - args: args{ - clusterKey: "rook-ceph", - holderEnabled: true, - clusterConfig: []CSIClusterConfigEntry{ - { - Namespace: "rook-ceph", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "rook-ceph", - Monitors: []string{"1.1.1.1"}, - CephFS: cephcsi.CephFS{ - NetNamespaceFilePath: "cephfs.net.ns", - }, - RBD: cephcsi.RBD{ - NetNamespaceFilePath: "rbd.net.ns", - }, - NFS: cephcsi.NFS{ - NetNamespaceFilePath: "nfs.net.ns", - }, - }, + t.Run("clears netNamespaceFilePath for matching namespace", func(t *testing.T) { + cc := csiClusterConfig{ + { + Namespace: "rook-ceph", + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "rook-ceph", + Monitors: []string{"1.1.1.1"}, + CephFS: cephcsi.CephFS{ + NetNamespaceFilePath: "cephfs.net.ns", }, - { - Namespace: "rook-ceph", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "cluster-1", - Monitors: []string{"1.1.1.1"}, - CephFS: cephcsi.CephFS{ - SubvolumeGroup: "csi", - }, - }, + RBD: cephcsi.RBD{ + NetNamespaceFilePath: "rbd.net.ns", + }, + NFS: cephcsi.NFS{ + NetNamespaceFilePath: "nfs.net.ns", }, }, }, - want: []CSIClusterConfigEntry{ - { - Namespace: "rook-ceph", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "rook-ceph", - Monitors: []string{"1.1.1.1"}, - CephFS: cephcsi.CephFS{ - NetNamespaceFilePath: "cephfs.net.ns", - }, - RBD: cephcsi.RBD{ - NetNamespaceFilePath: "rbd.net.ns", - }, - NFS: cephcsi.NFS{ - NetNamespaceFilePath: "nfs.net.ns", - }, - }, - }, - { - Namespace: "rook-ceph", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "cluster-1", - Monitors: []string{"1.1.1.1"}, - CephFS: cephcsi.CephFS{ - SubvolumeGroup: "csi", - NetNamespaceFilePath: "cephfs.net.ns", - }, - RBD: cephcsi.RBD{ - NetNamespaceFilePath: "rbd.net.ns", - }, - NFS: cephcsi.NFS{ - NetNamespaceFilePath: "nfs.net.ns", - }, + { + Namespace: "rook-ceph", + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "cluster-1", + Monitors: []string{"1.1.1.1"}, + RBD: cephcsi.RBD{ + RadosNamespace: "group-1", }, }, }, - }, - { - name: "holder disabled", - args: args{ - clusterKey: "rook-ceph", - holderEnabled: false, - clusterConfig: []CSIClusterConfigEntry{ - { - Namespace: "rook-ceph", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "rook-ceph", - Monitors: []string{"1.1.1.1"}, - CephFS: cephcsi.CephFS{ - NetNamespaceFilePath: "cephfs.net.ns", - }, - RBD: cephcsi.RBD{ - NetNamespaceFilePath: "rbd.net.ns", - }, - NFS: cephcsi.NFS{ - NetNamespaceFilePath: "nfs.net.ns", - }, - }, - }, - { - Namespace: "rook-ceph", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "cluster-1", - Monitors: []string{"1.1.1.1"}, - RBD: cephcsi.RBD{ - RadosNamespace: "group-1", - }, - }, - }, - }, - }, - want: []CSIClusterConfigEntry{ - { - Namespace: "rook-ceph", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "rook-ceph", - Monitors: []string{"1.1.1.1"}, - CephFS: cephcsi.CephFS{ - NetNamespaceFilePath: "", - }, - RBD: cephcsi.RBD{ - NetNamespaceFilePath: "", - }, - NFS: cephcsi.NFS{ - NetNamespaceFilePath: "", - }, - }, - }, - { - Namespace: "rook-ceph", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "cluster-1", - Monitors: []string{"1.1.1.1"}, - CephFS: cephcsi.CephFS{ - NetNamespaceFilePath: "", - }, - RBD: cephcsi.RBD{ - NetNamespaceFilePath: "", - RadosNamespace: "group-1", - }, - NFS: cephcsi.NFS{ - NetNamespaceFilePath: "", - }, - }, - }, - }, - }, - } - - cephfsNsFilePath := "cephfs.net.ns" - rbdNsFilePath := "rbd.net.ns" - nfsNsFilePath := "nfs.net.ns" - - csiConfigMap := csiClusterConfig{ - { - Namespace: "rook-ceph", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "rook-ceph", - CephFS: cephcsi.CephFS{ - NetNamespaceFilePath: cephfsNsFilePath, - }, - RBD: cephcsi.RBD{ - NetNamespaceFilePath: rbdNsFilePath, - }, - NFS: cephcsi.NFS{ - NetNamespaceFilePath: nfsNsFilePath, - }, - }, - }, - { - Namespace: "rook-ceph", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "svg", - }, - }, - { - Namespace: "default", - ClusterInfo: cephcsi.ClusterInfo{ - ClusterID: "rook-ceph", - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - holderEnabled = tt.args.holderEnabled - updateNetNamespaceFilePath("rook-ceph", tt.args.clusterConfig) - assert.True(t, reflect.DeepEqual(tt.args.clusterConfig, tt.want)) - }) - } - - t.Run("Holder enabled and disabled later", func(t *testing.T) { - holderEnabled = true - updateNetNamespaceFilePath("rook-ceph", csiConfigMap) - for _, c := range csiConfigMap { - if c.Namespace == "rook-ceph" { - assert.Equal(t, cephfsNsFilePath, c.CephFS.NetNamespaceFilePath) - assert.Equal(t, rbdNsFilePath, c.RBD.NetNamespaceFilePath) - assert.Equal(t, nfsNsFilePath, c.NFS.NetNamespaceFilePath) - } } - holderEnabled = false - updateNetNamespaceFilePath("rook-ceph", csiConfigMap) - for _, c := range csiConfigMap { - if c.Namespace == "rook-ceph" { - assert.Equal(t, "", c.CephFS.NetNamespaceFilePath) - assert.Equal(t, "", c.RBD.NetNamespaceFilePath) - assert.Equal(t, "", c.NFS.NetNamespaceFilePath) + updateNetNamespaceFilePath("rook-ceph", cc) - } + for _, c := range cc { + assert.Equal(t, "", c.CephFS.NetNamespaceFilePath) + assert.Equal(t, "", c.RBD.NetNamespaceFilePath) + assert.Equal(t, "", c.NFS.NetNamespaceFilePath) } + assert.Equal(t, "group-1", cc[1].RBD.RadosNamespace) }) } diff --git a/pkg/operator/ceph/csi/config.go b/pkg/operator/ceph/csi/config.go index 88c44da1b..c2659d20e 100644 --- a/pkg/operator/ceph/csi/config.go +++ b/pkg/operator/ceph/csi/config.go @@ -34,6 +34,25 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +var ( + CephFSDriverName string + NFSDriverName string + RBDDriverName string + + ConfigName = "rook-ceph-csi-config" + ConfigKey = "csi-cluster-config-json" +) + +const ( + CsiRBDPlugin = "csi-rbdplugin" + CsiCephFSPlugin = "csi-cephfsplugin" + CsiNFSPlugin = "csi-nfsplugin" + + rbdDriverSuffix = "rbd.csi.ceph.com" + cephFSDriverSuffix = "cephfs.csi.ceph.com" + nfsDriverSuffix = "nfs.csi.ceph.com" +) + func CreateUpdateClientProfileRadosNamespace(ctx context.Context, c client.Client, clusterInfo *cephclient.ClusterInfo, cephBlockPoolRadosNamespaceName, clusterID string) error { logger.Info("creating ceph-csi clientProfile CR for rados namespace") diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go index d348e664d..b875fc8e9 100644 --- a/pkg/operator/ceph/csi/controller.go +++ b/pkg/operator/ceph/csi/controller.go @@ -18,12 +18,11 @@ package csi import ( "context" + "fmt" "os" - "strconv" csiopv1 "github.com/ceph/ceph-csi-operator/api/v1" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -138,9 +137,6 @@ func (r *ReconcileCSI) Reconcile(context context.Context, request reconcile.Requ return reconcileResponse, err } -// allow overriding for unit tests -var reconcileSaveCSIDriverOptions = SaveCSIDriverOptions - func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, error) { // reconcileResult is used to communicate the result of the reconciliation back to the caller var reconcileResult reconcile.Result @@ -155,8 +151,6 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e } ownerInfo := k8sutil.NewOwnerInfoWithOwnerRef(ownerRef, r.opConfig.OperatorNamespace) - // create an empty config map. config map will be filled with data - // later when clusters have mons err = CreateCsiConfigMap(r.opManagerContext, r.opConfig.OperatorNamespace, r.context.Clientset, ownerInfo) if err != nil { return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed creating csi config map") @@ -166,18 +160,11 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to apply operator settings configmap") } - enableCSIOperator, err = strconv.ParseBool(k8sutil.GetOperatorSetting("ROOK_USE_CSI_OPERATOR", "true")) - if err != nil { - return reconcileResult, errors.Wrap(err, "unable to parse value for 'ROOK_USE_CSI_OPERATOR'") - } - - // do not recocnile if csi driver is disabled - disableCSI, err := strconv.ParseBool(k8sutil.GetOperatorSetting("ROOK_CSI_DISABLE_DRIVER", "false")) - if err != nil { - return reconcile.Result{}, errors.Wrap(err, "unable to parse value for 'ROOK_CSI_DISABLE_DRIVER") - } else if disableCSI { - logger.Info("ceph csi driver is disabled") - } + // Set driver names based on operator namespace + driverPrefix := fmt.Sprintf("%s.", r.opConfig.OperatorNamespace) + CephFSDriverName = driverPrefix + cephFSDriverSuffix + RBDDriverName = driverPrefix + rbdDriverSuffix + NFSDriverName = driverPrefix + nfsDriverSuffix // See if there is a CephCluster cephClusters := &cephv1.CephClusterList{} @@ -185,45 +172,21 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e if err != nil { if kerrors.IsNotFound(err) { logger.Debug("no ceph cluster found not deploying ceph csi driver") - EnableRBD, EnableCephFS, EnableNFS = false, false, false - err = r.stopDrivers() - if err != nil { - return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to stop Drivers") - } - return reconcile.Result{}, nil } - // Error reading the object - requeue the request. return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to list ceph clusters") } - // Do nothing if no ceph cluster is present if len(cephClusters.Items) == 0 { logger.Debug("no ceph cluster found not deploying ceph csi driver") - EnableRBD, EnableCephFS, EnableNFS = false, false, false - err = r.stopDrivers() - if err != nil { - return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to stop Drivers") - } - return reconcile.Result{}, nil } - // if at least one cephcluster is present update the csi lograte sidecar - // with the first listed ceph cluster specs with logrotate enabled - r.setCSILogrotateParams(cephClusters.Items) - err = peermap.CreateOrUpdateConfig(r.opManagerContext, r.context, &peermap.PeerIDMappings{}) if err != nil { return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to create pool ID mapping config map") } - exists, err := checkCsiCephConfigMapExists(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace) - if err != nil { - return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get csi ceph.conf configmap") - } - CustomCSICephConfigExists = exists - for i, cluster := range cephClusters.Items { if !cluster.DeletionTimestamp.IsZero() { logger.Debugf("ceph cluster %q is being deleting, no need to reconcile the csi driver", request.NamespacedName) @@ -239,11 +202,8 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e r.firstCephCluster = &cephClusters.Items[i].Spec } - // Load cluster info for later use in updating the ceph-csi configmap clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cluster.Namespace, &cephClusters.Items[i].Spec) if err != nil { - // This avoids a requeue with exponential backoff and allows the controller to reconcile - // more quickly when the cluster is ready. if errors.Is(err, opcontroller.ClusterInfoNoClusterNoSecret) { logger.Infof("cluster info for cluster %q is not ready yet, will retry in %s, proceeding with ready clusters", cluster.Name, opcontroller.WaitForRequeueIfCephClusterNotReady.RequeueAfter.String()) reconcileResult = opcontroller.WaitForRequeueIfCephClusterNotReady @@ -252,88 +212,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to load cluster info for cluster %q", cluster.Name) } clusterInfo.OwnerInfo = k8sutil.NewOwnerInfo(&cephClusters.Items[i], r.scheme) - - // ensure any remaining holder-related configs are cleared - holderEnabled = false - err = reconcileSaveCSIDriverOptions(r.context.Clientset, cluster.Namespace, clusterInfo) - if err != nil { - return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to update CSI driver options for cluster %q", cluster.Name) - } - - // disable Rook-managed CSI drivers if CSI operator is enabled - if EnableCSIOperator() { - logger.Info("disabling csi-driver since EnableCSIOperator is true") - err := r.stopDrivers() - if err != nil { - return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to stop csi Drivers") - } - err = r.deleteRookCSICMIfExists() - if err != nil { - return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to delete rook-ceph-csi-config configmap") - } - err = r.reconcileOperatorConfig(cluster, clusterInfo) - if err != nil { - return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to reconcile csi-op config CR") - } - return reconcileResult, nil - } - } - - if !disableCSI && !EnableCSIOperator() { - // delete CSI operator resources if CSI operator is disabled but CSI driver is enabled - err := r.deleteCSIOperatorResources() - if err != nil { - logger.Errorf("failed to delete CSI operator resources: %v", err) - } - - err = r.validateAndConfigureDrivers(ownerInfo) - if err != nil { - return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to configure ceph csi") - } } return reconcileResult, nil } - -func (r *ReconcileCSI) reconcileOperatorConfig(cluster cephv1.CephCluster, clusterInfo *cephclient.ClusterInfo) error { - if err := r.setParams(); err != nil { - return errors.Wrapf(err, "failed to configure CSI parameters") - } - - if err := validateCSIParam(); err != nil { - return errors.Wrapf(err, "failed to validate CSI parameters") - } - - err := r.createOrUpdateOperatorConfig(cluster) - if err != nil { - return errors.Wrap(err, "failed to configure csi operator operator config cr") - } - - err = r.createOrUpdateDriverResources(cluster, clusterInfo) - if err != nil { - return errors.Wrap(err, "failed to configure ceph-CSI operator drivers cr") - } - return nil -} - -func (r *ReconcileCSI) setCSILogrotateParams(cephClustersItems []cephv1.CephCluster) { - logger.Debug("set logrotate values in csi param") - spec := cephClustersItems[0].Spec - for _, cluster := range cephClustersItems { - if cluster.Spec.LogCollector.Enabled { - spec = cluster.Spec - break - } - } - csiRootPath = spec.DataDirHostPath - if spec.DataDirHostPath == "" { - csiRootPath = k8sutil.DataDir - } - - CSIParam.CSILogRotation = spec.LogCollector.Enabled - if spec.LogCollector.Enabled { - maxSize, period := opcontroller.GetLogRotateConfig(spec) - CSIParam.CSILogRotationMaxSize = maxSize.String() - CSIParam.CSILogRotationPeriod = period - } -} diff --git a/pkg/operator/ceph/csi/controller_test.go b/pkg/operator/ceph/csi/controller_test.go index 45870c972..59814da19 100644 --- a/pkg/operator/ceph/csi/controller_test.go +++ b/pkg/operator/ceph/csi/controller_test.go @@ -27,48 +27,29 @@ import ( rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" "github.com/rook/rook/pkg/client/clientset/versioned/scheme" "github.com/rook/rook/pkg/clusterd" - "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" - apifake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) func TestCephCSIController(t *testing.T) { - oldReconcileSaveCSIDriverOptions := reconcileSaveCSIDriverOptions - defer func() { reconcileSaveCSIDriverOptions = oldReconcileSaveCSIDriverOptions }() - saveCSIDriverOptionsCalledForClusterNS := []string{} - reconcileSaveCSIDriverOptions = func(clientset kubernetes.Interface, clusterNamespace string, clusterInfo *client.ClusterInfo) error { - saveCSIDriverOptionsCalledForClusterNS = append(saveCSIDriverOptionsCalledForClusterNS, clusterNamespace) - return nil - } - ctx := context.TODO() var ( name = "rook-ceph" namespace = "rook-ceph" ) - // Set DEBUG logging capnslog.SetGlobalLogLevel(capnslog.DEBUG) os.Setenv("ROOK_LOG_LEVEL", "DEBUG") t.Setenv(k8sutil.PodNameEnvVar, "rook-ceph-operator") t.Setenv(k8sutil.PodNamespaceEnvVar, namespace) - CSIParam = Param{ - CSIPluginImage: "image", - RegistrarImage: "image", - ProvisionerImage: "image", - AttacherImage: "image", - SnapshotterImage: "image", - } req := reconcile.Request{ NamespacedName: types.NamespacedName{ Name: name, @@ -88,16 +69,13 @@ func TestCephCSIController(t *testing.T) { assert.NoError(t, err) _, err = c.Clientset.AppsV1().ReplicaSets(namespace).Create(context.TODO(), test.FakeReplicaSet(namespace), metav1.CreateOptions{}) assert.NoError(t, err) - // Register operator types with the runtime scheme. s := scheme.Scheme s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}, &v1.ConfigMapList{}, &cephv1.CephClusterList{}, &csiopv1.CephConnection{}, &csiopv1.ClientProfile{}, &csiopv1.OperatorConfig{}, &csiopv1.Driver{}) - // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithScheme(s).Build() c.Client = cl - // Create a ReconcileCSI object with the scheme and fake client. r := &ReconcileCSI{ client: cl, context: c, @@ -117,9 +95,8 @@ func TestCephCSIController(t *testing.T) { fakeClientSet := test.New(t, 1) test.SetFakeKubernetesVersion(fakeClientSet, "v1.21.0") c := &clusterd.Context{ - Clientset: fakeClientSet, - RookClientset: rookclient.NewSimpleClientset(), - ApiExtensionsClient: apifake.NewClientset(), + Clientset: fakeClientSet, + RookClientset: rookclient.NewSimpleClientset(), } _, err := c.Clientset.CoreV1().Pods(namespace).Create(ctx, test.FakeOperatorPod(namespace), metav1.CreateOptions{}) assert.NoError(t, err) @@ -140,7 +117,6 @@ func TestCephCSIController(t *testing.T) { }, }, } - // Mock clusterInfo secrets := map[string][]byte{ "fsid": []byte(name), "mon-secret": []byte("monsecret"), @@ -158,15 +134,11 @@ func TestCephCSIController(t *testing.T) { assert.NoError(t, err) s := scheme.Scheme s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}, &v1.ConfigMap{}) - saveCSIDriverOptionsCalledForClusterNS = []string{} - object := []runtime.Object{ cephCluster, } - // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() c.Client = cl - // // Create a ReconcileCSI object with the scheme and fake client. r := &ReconcileCSI{ client: cl, context: c, @@ -180,85 +152,5 @@ func TestCephCSIController(t *testing.T) { res, err := r.Reconcile(ctx, req) assert.NoError(t, err) assert.False(t, res.Requeue) - - assert.Equal(t, []string{namespace}, saveCSIDriverOptionsCalledForClusterNS) - }) - - t.Run("success ceph csi deployment with multus", func(t *testing.T) { - fakeClientSet := test.New(t, 1) - test.SetFakeKubernetesVersion(fakeClientSet, "v1.21.0") - c := &clusterd.Context{ - Clientset: fakeClientSet, - RookClientset: rookclient.NewSimpleClientset(), - ApiExtensionsClient: apifake.NewClientset(), - } - _, err := c.Clientset.CoreV1().Pods(namespace).Create(ctx, test.FakeOperatorPod(namespace), metav1.CreateOptions{}) - assert.NoError(t, err) - _, err = c.Clientset.AppsV1().ReplicaSets(namespace).Create(context.TODO(), test.FakeReplicaSet(namespace), metav1.CreateOptions{}) - assert.NoError(t, err) - cephCluster := &cephv1.CephCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - Namespace: namespace, - }, - Spec: cephv1.ClusterSpec{ - Network: cephv1.NetworkSpec{ - Provider: cephv1.NetworkProviderMultus, - Selectors: map[cephv1.CephNetworkType]string{"public": "public-net", "cluster": "cluster-net"}, - }, - }, - Status: cephv1.ClusterStatus{ - Phase: "", - CephVersion: &cephv1.ClusterVersion{ - Version: "14.2.9-0", - }, - CephStatus: &cephv1.CephStatus{ - FSID: "89d00089-d6e6-4445-b596-96898ce31793", - }, - }, - } - // Mock clusterInfo - secrets := map[string][]byte{ - "fsid": []byte(name), - "mon-secret": []byte("monsecret"), - "admin-secret": []byte("adminsecret"), - } - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "rook-ceph-mon", - Namespace: namespace, - }, - Data: secrets, - Type: k8sutil.RookType, - } - _, err = c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) - assert.NoError(t, err) - s := scheme.Scheme - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}, &v1.ConfigMap{}) - saveCSIDriverOptionsCalledForClusterNS = []string{} - - object := []runtime.Object{ - cephCluster, - } - // Create a fake client to mock API calls. - cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() - c.Client = cl - // Create a ReconcileCSI object with the scheme and fake client. - r := &ReconcileCSI{ - scheme: s, - client: cl, - context: c, - opConfig: controller.OperatorConfig{ - OperatorNamespace: namespace, - Image: "rook", - ServiceAccount: "foo", - }, - } - - res, err := r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.False(t, res.Requeue) - - assert.Equal(t, []string{namespace}, saveCSIDriverOptionsCalledForClusterNS) }) } diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go deleted file mode 100644 index 0b21d6674..000000000 --- a/pkg/operator/ceph/csi/csi.go +++ /dev/null @@ -1,394 +0,0 @@ -/* -Copyright 2020 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "context" - "fmt" - "strconv" - "strings" - "time" - - "github.com/rook/rook/pkg/operator/ceph/controller" - "github.com/rook/rook/pkg/operator/k8sutil" - - "github.com/pkg/errors" - kerrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func (r *ReconcileCSI) validateAndConfigureDrivers(ownerInfo *k8sutil.OwnerInfo) error { - var err error - - if err = r.setParams(); err != nil { - return errors.Wrapf(err, "failed to configure CSI parameters") - } - - if err = validateCSIParam(); err != nil { - return errors.Wrapf(err, "failed to validate CSI parameters") - } - - if CSIEnabled() { - if err = r.startDrivers(ownerInfo); err != nil { - return errors.Wrap(err, "failed to start ceph csi drivers") - } - } - - // Check whether RBD or CephFS needs to be disabled - return r.stopDrivers() -} - -func (r *ReconcileCSI) setParams() error { - var err error - - if EnableRBD, err = strconv.ParseBool(k8sutil.GetOperatorSetting("ROOK_CSI_ENABLE_RBD", "true")); err != nil { - return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_RBD'") - } - - if EnableCephFS, err = strconv.ParseBool(k8sutil.GetOperatorSetting("ROOK_CSI_ENABLE_CEPHFS", "true")); err != nil { - return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_CEPHFS'") - } - - if EnableNFS, err = strconv.ParseBool(k8sutil.GetOperatorSetting("ROOK_CSI_ENABLE_NFS", "false")); err != nil { - return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_NFS'") - } - - if CSIParam.EnableCSIHostNetwork, err = strconv.ParseBool(k8sutil.GetOperatorSetting("CSI_ENABLE_HOST_NETWORK", "true")); err != nil { - return errors.Wrap(err, "failed to parse value for 'CSI_ENABLE_HOST_NETWORK'") - } - - // If not set or set to anything but "false", the kernel client will be enabled - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_FORCE_CEPHFS_KERNEL_CLIENT", "true"), "false") { - CSIParam.ForceCephFSKernelClient = "false" - } else { - CSIParam.ForceCephFSKernelClient = "true" - } - - // parse RPC timeout - timeout := k8sutil.GetOperatorSetting(grpcTimeout, "150") - timeoutSeconds, err := strconv.Atoi(timeout) - if err != nil { - logger.Errorf("failed to parse %q. Defaulting to %d. %v", grpcTimeout, defaultGRPCTimeout, err) - timeoutSeconds = defaultGRPCTimeout - } - if timeoutSeconds < 120 { - logger.Warningf("%s is %q but it should be >= 120, setting the default value %d", grpcTimeout, timeout, defaultGRPCTimeout) - timeoutSeconds = defaultGRPCTimeout - } - CSIParam.GRPCTimeout = time.Duration(timeoutSeconds) * time.Second - - // parse Liveness port - CSIParam.CephFSLivenessMetricsPort, err = getPortFromConfig("CSI_CEPHFS_LIVENESS_METRICS_PORT", DefaultCephFSLivenessMerticsPort) - if err != nil { - return errors.Wrap(err, "error getting CSI CephFS liveness metrics port.") - } - - CSIParam.CSIAddonsPort, err = getPortFromConfig("CSIADDONS_PORT", DefaultCSIAddonsPort) - if err != nil { - return errors.Wrap(err, "failed to get CSI Addons port") - } - - CSIParam.CSIAddonsRBDProvisionerPort, err = getPortFromConfig("CSIADDONS_RBD_PROVISIONER_PORT", DefaultCSIAddonsRBDProvisionerPort) - if err != nil { - return errors.Wrap(err, "failed to get CSI Addons port for RBD provisioner") - } - - CSIParam.CSIAddonsCephFSProvisionerPort, err = getPortFromConfig("CSIADDONS_CEPHFS_PROVISIONER_PORT", DefaultCSIAddonsCephFSProvisionerPort) - if err != nil { - return errors.Wrap(err, "failed to get CSI Addons port for Ceph FS provisioner") - } - - CSIParam.RBDLivenessMetricsPort, err = getPortFromConfig("CSI_RBD_LIVENESS_METRICS_PORT", DefaultRBDLivenessMerticsPort) - if err != nil { - return errors.Wrap(err, "error getting CSI RBD liveness metrics port.") - } - - CSIParam.EnableLiveness, err = strconv.ParseBool(k8sutil.GetOperatorSetting("CSI_ENABLE_LIVENESS", "false")) - if err != nil { - return errors.Wrap(err, "failed to parse value for 'CSI_ENABLE_LIVENESS'") - } - - CSIParam.Privileged = controller.HostPathRequiresPrivileged() - - // default value `system-node-critical` is the highest available priority - CSIParam.PluginPriorityClassName = k8sutil.GetOperatorSetting("CSI_PLUGIN_PRIORITY_CLASSNAME", "") - - // default value `system-cluster-critical` is applied for some - // critical pods in cluster but less priority than plugin pods - CSIParam.ProvisionerPriorityClassName = k8sutil.GetOperatorSetting("CSI_PROVISIONER_PRIORITY_CLASSNAME", "") - - CSIParam.EnableOMAPGenerator = false - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_OMAP_GENERATOR", "false"), "true") { - CSIParam.EnableOMAPGenerator = true - } - - CSIParam.EnableRBDSnapshotter = true - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_RBD_SNAPSHOTTER", "true"), "false") { - CSIParam.EnableRBDSnapshotter = false - } - - CSIParam.EnableCephFSSnapshotter = true - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_CEPHFS_SNAPSHOTTER", "true"), "false") { - CSIParam.EnableCephFSSnapshotter = false - } - - CSIParam.EnableNFSSnapshotter = true - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_NFS_SNAPSHOTTER", "true"), "false") { - CSIParam.EnableNFSSnapshotter = false - } - - CSIParam.EnableCSIAddonsSideCar = false - _, err = r.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(r.opManagerContext, "csiaddonsnodes.csiaddons.openshift.io", metav1.GetOptions{}) - if err == nil { - CSIParam.EnableCSIAddonsSideCar = true - } - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_CSIADDONS", ""), "false") { - CSIParam.EnableCSIAddonsSideCar = false - } - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_CSIADDONS", ""), "true") { - CSIParam.EnableCSIAddonsSideCar = true - } - - CSIParam.EnableCrossNamespaceVolumeDataSource = false - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_CROSS_NAMESPACE_VOLUME_DATA_SOURCE", "false"), "true") { - CSIParam.EnableCrossNamespaceVolumeDataSource = true - } - - CSIParam.EnableCSITopology = false - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_TOPOLOGY", "false"), "true") { - CSIParam.EnableCSITopology = true - } - - CSIParam.EnableCSIEncryption = false - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_ENCRYPTION", "false"), "true") { - CSIParam.EnableCSIEncryption = true - } - - CSIParam.CSIEnableMetadata = true - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_METADATA", "true"), "false") { - CSIParam.CSIEnableMetadata = false - } - - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { - CSIParam.CephFSPluginUpdateStrategy = onDelete - } else { - CSIParam.CephFSPluginUpdateStrategy = rollingUpdate - CSIParam.CephFSPluginUpdateStrategyMaxUnavailable = k8sutil.GetOperatorSetting("CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE", "1") - } - - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_NFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { - CSIParam.NFSPluginUpdateStrategy = onDelete - } else { - CSIParam.NFSPluginUpdateStrategy = rollingUpdate - } - - // Default values are based on Kubernetes official documentation. - // https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_RBD_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { - CSIParam.RBDPluginUpdateStrategy = onDelete - } else { - CSIParam.RBDPluginUpdateStrategy = rollingUpdate - CSIParam.RBDPluginUpdateStrategyMaxUnavailable = k8sutil.GetOperatorSetting("CSI_RBD_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE", "1") - } - - CSIParam.EnablePluginSelinuxHostMount = false - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT", "false"), "true") { - CSIParam.EnablePluginSelinuxHostMount = true - } - - logLevel := k8sutil.GetOperatorSetting("CSI_LOG_LEVEL", "") - CSIParam.LogLevel = defaultLogLevel - if logLevel != "" { - l, err := strconv.ParseUint(logLevel, 10, 8) - if err != nil { - logger.Errorf("failed to parse CSI_LOG_LEVEL. Defaulting to %d. %v", defaultLogLevel, err) - } else { - CSIParam.LogLevel = uint8(l) - } - } - - sidecarLogLevel := k8sutil.GetOperatorSetting("CSI_SIDECAR_LOG_LEVEL", "") - CSIParam.SidecarLogLevel = defaultSidecarLogLevel - if sidecarLogLevel != "" { - l, err := strconv.ParseUint(sidecarLogLevel, 10, 8) - if err != nil { - logger.Errorf("failed to parse CSI_SIDECAR_LOG_LEVEL. Defaulting to %d. %v", defaultSidecarLogLevel, err) - } else { - CSIParam.SidecarLogLevel = uint8(l) - } - } - - leaderElectionLeaseDuration := k8sutil.GetOperatorSetting("CSI_LEADER_ELECTION_LEASE_DURATION", "") - CSIParam.LeaderElectionLeaseDuration = defaultLeaderElectionLeaseDuration - if leaderElectionLeaseDuration != "" { - d, err := time.ParseDuration(leaderElectionLeaseDuration) - if err != nil { - logger.Errorf("failed to parse CSI_LEADER_ELECTION_LEASE_DURATION. Defaulting to %s. %v", defaultLeaderElectionLeaseDuration, err) - } else { - CSIParam.LeaderElectionLeaseDuration = d - } - } - - leaderElectionRenewDeadline := k8sutil.GetOperatorSetting("CSI_LEADER_ELECTION_RENEW_DEADLINE", "") - CSIParam.LeaderElectionRenewDeadline = defaultLeaderElectionRenewDeadline - if leaderElectionRenewDeadline != "" { - d, err := time.ParseDuration(leaderElectionRenewDeadline) - if err != nil { - logger.Errorf("failed to parse CSI_LEADER_ELECTION_RENEW_DEADLINE. Defaulting to %s. %v", defaultLeaderElectionRenewDeadline, err) - } else { - CSIParam.LeaderElectionRenewDeadline = d - } - } - - leaderElectionRetryPeriod := k8sutil.GetOperatorSetting("CSI_LEADER_ELECTION_RETRY_PERIOD", "") - CSIParam.LeaderElectionRetryPeriod = defaultLeaderElectionRetryPeriod - if leaderElectionRetryPeriod != "" { - d, err := time.ParseDuration(leaderElectionRetryPeriod) - if err != nil { - logger.Errorf("failed to parse CSI_LEADER_ELECTION_RETRY_PERIOD. Defaulting to %s. %v", defaultLeaderElectionRetryPeriod, err) - } else { - CSIParam.LeaderElectionRetryPeriod = d - } - } - - CSIParam.ProvisionerReplicas = defaultProvisionerReplicas - nodes, err := r.context.Clientset.CoreV1().Nodes().List(r.opManagerContext, metav1.ListOptions{}) - if err == nil { - if len(nodes.Items) == 1 { - CSIParam.ProvisionerReplicas = 1 - } else { - replicaStr := k8sutil.GetOperatorSetting("CSI_PROVISIONER_REPLICAS", "2") - replicas, err := strconv.ParseInt(replicaStr, 10, 32) - if err != nil { - logger.Errorf("failed to parse CSI_PROVISIONER_REPLICAS. Defaulting to %d. %v", defaultProvisionerReplicas, err) - } else { - CSIParam.ProvisionerReplicas = int32(replicas) - } - } - } else { - logger.Errorf("failed to get nodes. Defaulting the number of replicas of provisioner pods to %d. %v", CSIParam.ProvisionerReplicas, err) - } - - CSIParam.CSIPluginImage = getImage("ROOK_CSI_CEPH_IMAGE", DefaultCSIPluginImage) - CSIParam.RegistrarImage = getImage("ROOK_CSI_REGISTRAR_IMAGE", DefaultRegistrarImage) - CSIParam.ProvisionerImage = getImage("ROOK_CSI_PROVISIONER_IMAGE", DefaultProvisionerImage) - CSIParam.AttacherImage = getImage("ROOK_CSI_ATTACHER_IMAGE", DefaultAttacherImage) - CSIParam.SnapshotterImage = getImage("ROOK_CSI_SNAPSHOTTER_IMAGE", DefaultSnapshotterImage) - CSIParam.ResizerImage = getImage("ROOK_CSI_RESIZER_IMAGE", DefaultResizerImage) - CSIParam.KubeletDirPath = k8sutil.GetOperatorSetting("ROOK_CSI_KUBELET_DIR_PATH", DefaultKubeletDirPath) - CSIParam.CSIAddonsImage = getImage("ROOK_CSIADDONS_IMAGE", DefaultCSIAddonsImage) - CSIParam.CSIDomainLabels = k8sutil.GetOperatorSetting("CSI_TOPOLOGY_DOMAIN_LABELS", "") - csiCephFSPodLabels := k8sutil.GetOperatorSetting("ROOK_CSI_CEPHFS_POD_LABELS", "") - CSIParam.CSICephFSPodLabels = k8sutil.ParseStringToLabels(csiCephFSPodLabels) - csiNFSPodLabels := k8sutil.GetOperatorSetting("ROOK_CSI_NFS_POD_LABELS", "") - CSIParam.CSINFSPodLabels = k8sutil.ParseStringToLabels(csiNFSPodLabels) - csiRBDPodLabels := k8sutil.GetOperatorSetting("ROOK_CSI_RBD_POD_LABELS", "") - CSIParam.CSIRBDPodLabels = k8sutil.ParseStringToLabels(csiRBDPodLabels) - CSIParam.CSIClusterName = k8sutil.GetOperatorSetting("CSI_CLUSTER_NAME", "") - CSIParam.ImagePullPolicy = k8sutil.GetOperatorSetting("ROOK_CSI_IMAGE_PULL_POLICY", DefaultCSIImagePullPolicy) - CSIParam.CephFSKernelMountOptions = k8sutil.GetOperatorSetting("CSI_CEPHFS_KERNEL_MOUNT_OPTIONS", "") - - CSIParam.CephFSAttachRequired = true - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_CEPHFS_ATTACH_REQUIRED", "true"), "false") { - CSIParam.CephFSAttachRequired = false - } - CSIParam.RBDAttachRequired = true - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_RBD_ATTACH_REQUIRED", "true"), "false") { - CSIParam.RBDAttachRequired = false - } - CSIParam.NFSAttachRequired = true - if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_NFS_ATTACH_REQUIRED", "true"), "false") { - CSIParam.NFSAttachRequired = false - } - - CSIParam.DriverNamePrefix = k8sutil.GetOperatorSetting("CSI_DRIVER_NAME_PREFIX", r.opConfig.OperatorNamespace) - - crd, err := r.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), "volumegroupsnapshotclasses.groupsnapshot.storage.k8s.io", metav1.GetOptions{}) - if err != nil && !kerrors.IsNotFound(err) { - return errors.Wrapf(err, "failed to get volumegroupsnapshotclasses.groupsnapshot.storage.k8s.io CRD") - } - CSIParam.VolumeGroupSnapshotSupported = (err == nil) - - if err == nil && len(crd.Spec.Versions) > 0 { - ver := crd.Spec.Versions[0] - // Determine if VolumeGroupSnapshot feature should be disabled - disableVGS := strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_VOLUME_GROUP_SNAPSHOT", "true"), "false") - const ( - enableVolumeGroupSnapshotFlag = "--enable-volume-group-snapshots=" - featureGateFlag = "--feature-gates=CSIVolumeGroupSnapshot=" - ) - // Check for "v1alpha1" version to determine the appropriate CLI flag - // In the "v1alpha1" version, we use the '--enable-volume-group-snapshots' flag. - // In later versions (e.g., "v1beta1"), we use the '--feature-gates=CSIVolumeGroupSnapshot' flag. - if ver.Name == "v1alpha1" { - CSIParam.VolumeGroupSnapshotCLIFlag = enableVolumeGroupSnapshotFlag + "true" - } else { - CSIParam.VolumeGroupSnapshotCLIFlag = featureGateFlag + "true" - } - if disableVGS { - if ver.Name == "v1alpha1" { - CSIParam.VolumeGroupSnapshotCLIFlag = enableVolumeGroupSnapshotFlag + "false" - } else { - CSIParam.VolumeGroupSnapshotCLIFlag = featureGateFlag + "false" - } - } - } - - kubeApiBurst := k8sutil.GetOperatorSetting("CSI_KUBE_API_BURST", "") - CSIParam.KubeApiBurst = 0 - if kubeApiBurst != "" { - k, err := strconv.ParseUint(kubeApiBurst, 10, 16) - if err != nil { - logger.Errorf("failed to parse CSI_KUBE_API_BURST. %v", err) - } else { - CSIParam.KubeApiBurst = uint16(k) - } - } - - kubeApiQPS := k8sutil.GetOperatorSetting("CSI_KUBE_API_QPS", "") - CSIParam.KubeApiQPS = 0 - if kubeApiQPS != "" { - k, err := strconv.ParseFloat(kubeApiQPS, 32) - if err != nil { - logger.Errorf("failed to parse CSI_KUBE_API_QPS. %v", err) - } else { - CSIParam.KubeApiQPS = float32(k) - } - } - - if strings.HasSuffix(CSIParam.DriverNamePrefix, ".") { - // As operator is adding a dot at the end of the prefix, we should not - // allow the user to add a dot at the end of the prefix. as it will - // result in two dots at the end of the prefix. which cases the csi - // driver name creation failure - return errors.Errorf("driver name prefix %q should not end with a dot", CSIParam.DriverNamePrefix) - } - - err = validateCSIDriverNamePrefix(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace, CSIParam.DriverNamePrefix) - if err != nil { - return err - } - // Add a dot at the end of the prefix for having the driver name prefix - // with format . - CSIParam.DriverNamePrefix = fmt.Sprintf("%s.", CSIParam.DriverNamePrefix) - - // Set the driver names - CephFSDriverName = CSIParam.DriverNamePrefix + cephFSDriverSuffix - RBDDriverName = CSIParam.DriverNamePrefix + rbdDriverSuffix - NFSDriverName = CSIParam.DriverNamePrefix + nfsDriverSuffix - return nil -} diff --git a/pkg/operator/ceph/csi/csidriver.go b/pkg/operator/ceph/csi/csidriver.go deleted file mode 100644 index 7948dddfd..000000000 --- a/pkg/operator/ceph/csi/csidriver.go +++ /dev/null @@ -1,118 +0,0 @@ -/* -Copyright 2021 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "context" - - "github.com/pkg/errors" - v1k8scsi "k8s.io/api/storage/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - v1 "k8s.io/client-go/kubernetes/typed/storage/v1" -) - -type v1CsiDriver struct { - csiDriver *v1k8scsi.CSIDriver - csiClient v1.CSIDriverInterface -} - -// createCSIDriverInfo Registers CSI driver by creating a CSIDriver object -func (d v1CsiDriver) createCSIDriverInfo( - ctx context.Context, - clientset kubernetes.Interface, - name, fsGroupPolicy string, - attachRequired bool, -) error { - mountInfo := false - // Create CSIDriver object - csiDriver := &v1k8scsi.CSIDriver{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: v1k8scsi.CSIDriverSpec{ - AttachRequired: &attachRequired, - PodInfoOnMount: &mountInfo, - }, - } - - selinuxMount := true - csiDriver.Spec.SELinuxMount = &selinuxMount - - if fsGroupPolicy != "" { - policy := v1k8scsi.FSGroupPolicy(fsGroupPolicy) - csiDriver.Spec.FSGroupPolicy = &policy - } - csidrivers := clientset.StorageV1().CSIDrivers() - driver, err := csidrivers.Get(ctx, name, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - _, err = csidrivers.Create(ctx, csiDriver, metav1.CreateOptions{}) - if err != nil { - return err - } - logger.Infof("CSIDriver object created for driver %q", name) - } - return err - } - - // As FSGroupPolicy and AttachRequired fields are immutable, should be set only during create time. - // if the request is to change the FSGroupPolicy or AttachRequired, we are deleting the CSIDriver object and creating it. - if (driver.Spec.FSGroupPolicy != nil && csiDriver.Spec.FSGroupPolicy != nil && *driver.Spec.FSGroupPolicy != *csiDriver.Spec.FSGroupPolicy) || *driver.Spec.AttachRequired != *csiDriver.Spec.AttachRequired { - d.csiClient = csidrivers - d.csiDriver = csiDriver - return d.reCreateCSIDriverInfo(ctx) - } - - // For csidriver we need to provide the resourceVersion when updating the object. - // From the docs (https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata) - // > "This value MUST be treated as opaque by clients and passed unmodified back to the server" - csiDriver.ObjectMeta.ResourceVersion = driver.ObjectMeta.ResourceVersion - _, err = csidrivers.Update(ctx, csiDriver, metav1.UpdateOptions{}) - if err != nil { - return err - } - logger.Infof("CSIDriver object updated for driver %q", name) - return nil -} - -func (d v1CsiDriver) reCreateCSIDriverInfo(ctx context.Context) error { - err := d.csiClient.Delete(ctx, d.csiDriver.Name, metav1.DeleteOptions{}) - if err != nil { - return errors.Wrapf(err, "failed to delete CSIDriver object for driver %q", d.csiDriver.Name) - } - logger.Infof("CSIDriver object deleted for driver %q", d.csiDriver.Name) - _, err = d.csiClient.Create(ctx, d.csiDriver, metav1.CreateOptions{}) - if err != nil { - return errors.Wrapf(err, "failed to recreate CSIDriver object for driver %q", d.csiDriver.Name) - } - logger.Infof("CSIDriver object recreated for driver %q", d.csiDriver.Name) - return nil -} - -// deleteCSIDriverInfo deletes CSIDriverInfo and returns the error if any -func (d v1CsiDriver) deleteCSIDriverInfo(ctx context.Context, clientset kubernetes.Interface, name string) error { - err := clientset.StorageV1().CSIDrivers().Delete(ctx, name, metav1.DeleteOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - logger.Debugf("%s CSIDriver not found; skipping deletion.", name) - return nil - } - } - return err -} diff --git a/pkg/operator/ceph/csi/operator_config.go b/pkg/operator/ceph/csi/operator_config.go deleted file mode 100644 index c8c86679c..000000000 --- a/pkg/operator/ceph/csi/operator_config.go +++ /dev/null @@ -1,181 +0,0 @@ -/* -Copyright 2024 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" - - csiopv1 "github.com/ceph/ceph-csi-operator/api/v1" - "github.com/pkg/errors" - appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - k8scsiv1 "k8s.io/api/storage/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" -) - -const ( - opConfigCRName = "ceph-csi-operator-config" -) - -func (r *ReconcileCSI) createOrUpdateOperatorConfig(cluster cephv1.CephCluster) error { - logger.Info("Creating ceph-CSI operator config CR") - - opConfig := &csiopv1.OperatorConfig{} - opConfig.Name = opConfigCRName - opConfig.Namespace = r.opConfig.OperatorNamespace - - imageSetCmName, err := r.createImageSetConfigmap() - if err != nil { - return errors.Wrapf(err, "failed to create ceph-CSI operator config ImageSetConfigmap for CR %s", opConfigCRName) - } - - spec := r.generateCSIOpConfigSpec(cluster, opConfig, imageSetCmName) - - err = r.client.Get(r.opManagerContext, types.NamespacedName{Name: opConfigCRName, Namespace: r.opConfig.OperatorNamespace}, opConfig) - if err != nil { - if kerrors.IsNotFound(err) { - opConfig.Spec = spec - err = r.client.Create(r.opManagerContext, opConfig) - if err != nil { - return errors.Wrapf(err, "failed to create ceph-CSI operator operator config CR %q", opConfig.Name) - } - - logger.Infof("Successfully created ceph-CSI operator config CR %q", opConfig.Name) - return nil - } - return errors.Wrapf(err, "failed to get ceph-CSI operator operator config CR %q", opConfigCRName) - } - - opConfig.Spec = spec - err = r.client.Update(r.opManagerContext, opConfig) - if err != nil { - return errors.Wrapf(err, "failed to update ceph-CSI operator operator config CR %q", opConfig.Name) - } - logger.Infof("Successfully updated ceph-CSI operator config CR %q", opConfig.Name) - - return nil -} - -func (r *ReconcileCSI) generateCSIOpConfigSpec(cluster cephv1.CephCluster, opConfig *csiopv1.OperatorConfig, imageSetCmName string) csiopv1.OperatorConfigSpec { - controllerPluginHostNetwork := opcontroller.EnforceHostNetwork() || CSIParam.EnableCSIHostNetwork - - cephfsClientType := csiopv1.KernelCephFsClient - if CSIParam.ForceCephFSKernelClient == "false" { - cephfsClientType = csiopv1.AutoDetectCephFsClient - } - - opConfig.Spec = csiopv1.OperatorConfigSpec{ - DriverSpecDefaults: &csiopv1.DriverSpec{ - Log: &csiopv1.LogSpec{ - Verbosity: int(CSIParam.LogLevel), - }, - ImageSet: &v1.LocalObjectReference{ - Name: imageSetCmName, - }, - ClusterName: &cluster.Name, - EnableMetadata: &CSIParam.CSIEnableMetadata, - GenerateOMapInfo: &CSIParam.EnableOMAPGenerator, - FsGroupPolicy: k8scsiv1.FileFSGroupPolicy, - NodePlugin: &csiopv1.NodePluginSpec{ - PodCommonSpec: csiopv1.PodCommonSpec{ - PrioritylClassName: &CSIParam.PluginPriorityClassName, - Affinity: &v1.Affinity{ - NodeAffinity: getNodeAffinity(pluginNodeAffinityEnv, &v1.NodeAffinity{}), - }, - Tolerations: getToleration(pluginTolerationsEnv, []v1.Toleration{}), - ImagePullPolicy: v1.PullPolicy(CSIParam.ImagePullPolicy), - }, - Resources: csiopv1.NodePluginResourcesSpec{}, - KubeletDirPath: CSIParam.KubeletDirPath, - EnableSeLinuxHostMount: &CSIParam.EnablePluginSelinuxHostMount, - }, - ControllerPlugin: &csiopv1.ControllerPluginSpec{ - DeploymentStrategy: &appsv1.DeploymentStrategy{ - Type: appsv1.RecreateDeploymentStrategyType, - }, - HostNetwork: &controllerPluginHostNetwork, - PodCommonSpec: csiopv1.PodCommonSpec{ - PrioritylClassName: &CSIParam.ProvisionerPriorityClassName, - Affinity: &v1.Affinity{ - NodeAffinity: getNodeAffinity(provisionerNodeAffinityEnv, &v1.NodeAffinity{}), - }, - Tolerations: getToleration(provisionerTolerationsEnv, []v1.Toleration{}), - ImagePullPolicy: v1.PullPolicy(CSIParam.ImagePullPolicy), - }, - Replicas: &CSIParam.ProvisionerReplicas, - Resources: csiopv1.ControllerPluginResourcesSpec{}, - }, - DeployCsiAddons: &CSIParam.EnableCSIAddonsSideCar, - CephFsClientType: cephfsClientType, - }, - } - if CSIParam.EnableCSIEncryption { - opConfig.Spec.DriverSpecDefaults.Encryption = &csiopv1.EncryptionSpec{ - ConfigMapRef: v1.LocalObjectReference{ - Name: "rook-ceph-csi-kms-config", - }, - } - } - - return opConfig.Spec -} - -func (r *ReconcileCSI) createImageSetConfigmap() (string, error) { - data := map[string]string{ - "provisioner": CSIParam.ProvisionerImage, - "attacher": CSIParam.AttacherImage, - "resizer": CSIParam.ResizerImage, - "snapshotter": CSIParam.SnapshotterImage, - "registrar": CSIParam.RegistrarImage, - "plugin": CSIParam.CSIPluginImage, - "addons": CSIParam.CSIAddonsImage, - } - - cm := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "rook-csi-operator-image-set-configmap", - Namespace: r.opConfig.OperatorNamespace, - }, - Data: data, - } - - err := r.client.Get(r.opManagerContext, types.NamespacedName{Name: cm.Name, Namespace: r.opConfig.OperatorNamespace}, cm) - if err != nil { - if kerrors.IsNotFound(err) { - err = r.client.Create(r.opManagerContext, cm) - if err != nil { - return "", errors.Wrapf(err, "failed to create imageSet cm %q for ceph-CSI operator-config CR %q", cm.Name, opConfigCRName) - } - - logger.Infof("Successfully create imageSet cm %s for ceph-CSI operator-config CR %q", cm.Name, opConfigCRName) - return cm.Name, nil - } - return "", errors.Wrapf(err, "failed to get imageSet cm %q for ceph-CSI operator-config CR %q", cm.Name, opConfigCRName) - } - - cm.Data = data - err = r.client.Update(r.opManagerContext, cm) - if err != nil { - return "", errors.Wrapf(err, "failed to updated imageSet cm %q ceph-CSI operator-config CR %q", cm.Name, opConfigCRName) - } - logger.Infof("Successfully updated imageSet cm %s for ceph-CSI operator-config CR %q", cm.Name, opConfigCRName) - - return cm.Name, nil -} diff --git a/pkg/operator/ceph/csi/operator_config_test.go b/pkg/operator/ceph/csi/operator_config_test.go deleted file mode 100644 index 6f38ba8ef..000000000 --- a/pkg/operator/ceph/csi/operator_config_test.go +++ /dev/null @@ -1,99 +0,0 @@ -/* -Copyright 2024 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "context" - "testing" - - csiopv1 "github.com/ceph/ceph-csi-operator/api/v1" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" - "github.com/rook/rook/pkg/client/clientset/versioned/scheme" - "github.com/rook/rook/pkg/clusterd" - opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" - testop "github.com/rook/rook/pkg/operator/test" - "github.com/stretchr/testify/assert" - v1 "k8s.io/api/core/v1" - apifake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func TestReconcileCSI_createOrUpdateOperatorConfig(t *testing.T) { - ns := "test" - cephv1.SetEnforceHostNetwork(true) - defer cephv1.SetEnforceHostNetwork(false) - r := &ReconcileCSI{ - context: &clusterd.Context{ - Clientset: testop.New(t, 1), - RookClientset: rookclient.NewSimpleClientset(), - ApiExtensionsClient: apifake.NewClientset(), - }, - opManagerContext: context.TODO(), - opConfig: opcontroller.OperatorConfig{ - OperatorNamespace: "test", - }, - } - cluster := &cephv1.CephCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testCluster", - Namespace: ns, - }, - Spec: cephv1.ClusterSpec{ - CSI: cephv1.CSIDriverSpec{ - ReadAffinity: cephv1.ReadAffinitySpec{ - Enabled: true, - CrushLocationLabels: []string{"kubernetes.io/hostname"}, - }, - CephFS: cephv1.CSICephFSSpec{ - KernelMountOptions: "ms_mode=crc", - }, - }, - Network: cephv1.NetworkSpec{ - Connections: &cephv1.ConnectionsSpec{ - Encryption: &cephv1.EncryptionSpec{ - Enabled: true, - }, - }, - }, - }, - } - opConfig := &csiopv1.OperatorConfig{} - - // Register operator types with the runtime scheme. - s := scheme.Scheme - s.AddKnownTypes(cephv1.SchemeGroupVersion, opConfig, &cephv1.CephCluster{}, &v1.ConfigMap{}) - object := []runtime.Object{ - cluster, - } - cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() - r.client = cl - - err := r.setParams() - assert.NoError(t, err) - - err = r.createOrUpdateOperatorConfig(*cluster) - assert.NoError(t, err) - - err = cl.Get(context.TODO(), types.NamespacedName{Name: opConfigCRName, Namespace: r.opConfig.OperatorNamespace}, opConfig) - assert.NoError(t, err) - assert.True(t, *opConfig.Spec.DriverSpecDefaults.EnableMetadata) - assert.True(t, *opConfig.Spec.DriverSpecDefaults.ControllerPlugin.HostNetwork) -} diff --git a/pkg/operator/ceph/csi/operator_driver.go b/pkg/operator/ceph/csi/operator_driver.go deleted file mode 100644 index ae84cfdb4..000000000 --- a/pkg/operator/ceph/csi/operator_driver.go +++ /dev/null @@ -1,485 +0,0 @@ -/* -Copyright 2024 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "context" - "encoding/json" - "fmt" - "reflect" - "strings" - - csiopv1 "github.com/ceph/ceph-csi-operator/api/v1" - "github.com/pkg/errors" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - cephclient "github.com/rook/rook/pkg/daemon/ceph/client" - "github.com/rook/rook/pkg/operator/k8sutil" - v1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - k8scsiv1 "k8s.io/api/storage/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -func (r *ReconcileCSI) createOrUpdateDriverResources(cluster cephv1.CephCluster, clusterInfo *cephclient.ClusterInfo) error { - if EnableRBD { - logger.Info("Creating RBD driver resources") - err := r.transferCSIDriverOwner(r.opManagerContext, RBDDriverName) - if err != nil { - return errors.Wrap(err, "failed to create update RBD driver for csi-operator driver CR ") - } - err = r.createOrUpdateRBDDriverResource(cluster, clusterInfo) - if err != nil { - return errors.Wrapf(err, "failed to create or update RBD driver resource in the namespace %q", r.opConfig.OperatorNamespace) - } - } - if EnableCephFS { - logger.Info("Creating CephFS driver resources") - err := r.transferCSIDriverOwner(r.opManagerContext, CephFSDriverName) - if err != nil { - return errors.Wrap(err, "failed to create update CephFS driver for csi-operator driver CR ") - } - err = r.createOrUpdateCephFSDriverResource(cluster, clusterInfo) - if err != nil { - return errors.Wrapf(err, "failed to create or update cephFS driver resource in the namespace %q", r.opConfig.OperatorNamespace) - } - } - if EnableNFS { - logger.Info("Creating NFS driver resources") - err := r.transferCSIDriverOwner(r.opManagerContext, NFSDriverName) - if err != nil { - return errors.Wrap(err, "failed to create update NFS driver for csi-operator driver CR ") - } - err = r.createOrUpdateNFSDriverResource(cluster, clusterInfo) - if err != nil { - return errors.Wrapf(err, "failed to create or update NFS driver resource in the namespace %q", r.opConfig.OperatorNamespace) - } - } - - return nil -} - -func (r *ReconcileCSI) createOrUpdateRBDDriverResource(cluster cephv1.CephCluster, clusterInfo *cephclient.ClusterInfo) error { - resourceName := fmt.Sprintf("%s.rbd.csi.ceph.com", r.opConfig.OperatorNamespace) - spec, err := r.generateDriverSpec(cluster) - if err != nil { - return err - } - - spec.NodePlugin.Labels = CSIParam.CSIRBDPodLabels - spec.ControllerPlugin.Labels = CSIParam.CSIRBDPodLabels - rbdDriver := &csiopv1.Driver{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - Namespace: r.opConfig.OperatorNamespace, - }, - Spec: spec, - } - - rbdDriver.Spec.ControllerPlugin.Resources = createDriverControllerPluginResources(rbdProvisionerResource) - rbdDriver.Spec.NodePlugin.Resources = createDriverNodePluginResouces(rbdPluginResource) - rbdDriver.Spec.NodePlugin.UpdateStrategy = &v1.DaemonSetUpdateStrategy{ - Type: v1.RollingUpdateDaemonSetStrategyType, - } - - if CSIParam.CSIDomainLabels != "" { - domainLabels := strings.Split(CSIParam.CSIDomainLabels, ",") - rbdDriver.Spec.NodePlugin.Topology = &csiopv1.TopologySpec{ - DomainLabels: domainLabels, - } - } - - if CSIParam.RBDPluginUpdateStrategy == "OnDelete" { - rbdDriver.Spec.NodePlugin.UpdateStrategy = &v1.DaemonSetUpdateStrategy{ - Type: v1.OnDeleteDaemonSetStrategyType, - } - } - - podVolumes := getPodVolumes(rbdPluginVolume, rbdPluginVolumeMount) - rbdDriver.Spec.NodePlugin.PodCommonSpec.Volumes = podVolumes - - err = r.createOrUpdateDriverResource(clusterInfo, rbdDriver) - if err != nil { - return errors.Wrapf(err, "failed to create or update RBD driver resource %q", rbdDriver.Name) - } - - return nil -} - -func (r *ReconcileCSI) createOrUpdateCephFSDriverResource(cluster cephv1.CephCluster, clusterInfo *cephclient.ClusterInfo) error { - resourceName := fmt.Sprintf("%s.cephfs.csi.ceph.com", r.opConfig.OperatorNamespace) - spec, err := r.generateDriverSpec(cluster) - if err != nil { - return err - } - - spec.NodePlugin.Labels = CSIParam.CSICephFSPodLabels - spec.ControllerPlugin.Labels = CSIParam.CSICephFSPodLabels - - cephFsDriver := &csiopv1.Driver{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - Namespace: r.opConfig.OperatorNamespace, - }, - Spec: spec, - } - - if CSIParam.VolumeGroupSnapshotCLIFlag != "" { - cephFsDriver.Spec.SnapshotPolicy = csiopv1.VolumeGroupSnapshotPolicy - } - - cephFsDriver.Spec.ControllerPlugin.Resources = createDriverControllerPluginResources(cephFSProvisionerResource) - - cephFsDriver.Spec.NodePlugin.Resources = createDriverNodePluginResouces(cephFSPluginResource) - cephFsDriver.Spec.NodePlugin.UpdateStrategy = &v1.DaemonSetUpdateStrategy{ - Type: v1.RollingUpdateDaemonSetStrategyType, - } - - if CSIParam.CSIDomainLabels != "" { - domainLabels := strings.Split(CSIParam.CSIDomainLabels, ",") - cephFsDriver.Spec.NodePlugin.Topology = &csiopv1.TopologySpec{ - DomainLabels: domainLabels, - } - } - - if CSIParam.RBDPluginUpdateStrategy == "OnDelete" { - cephFsDriver.Spec.NodePlugin.UpdateStrategy = &v1.DaemonSetUpdateStrategy{ - Type: v1.OnDeleteDaemonSetStrategyType, - } - } - - podVolumes := getPodVolumes(cephFSPluginVolume, cephFSPluginVolumeMount) - cephFsDriver.Spec.NodePlugin.PodCommonSpec.Volumes = podVolumes - - err = r.createOrUpdateDriverResource(clusterInfo, cephFsDriver) - if err != nil { - return errors.Wrapf(err, "failed to create or update cephFS driver resource %q", cephFsDriver.Name) - } - - return nil -} - -func (r *ReconcileCSI) createOrUpdateNFSDriverResource(cluster cephv1.CephCluster, clusterInfo *cephclient.ClusterInfo) error { - resourceName := fmt.Sprintf("%s.nfs.csi.ceph.com", r.opConfig.OperatorNamespace) - spec, err := r.generateDriverSpec(cluster) - if err != nil { - return err - } - spec.NodePlugin.Labels = CSIParam.CSINFSPodLabels - spec.ControllerPlugin.Labels = CSIParam.CSINFSPodLabels - - NFSDriver := &csiopv1.Driver{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - Namespace: r.opConfig.OperatorNamespace, - }, - Spec: spec, - } - - NFSDriver.Spec.ControllerPlugin.Resources = createDriverControllerPluginResources(nfsProvisionerResource) - - NFSDriver.Spec.NodePlugin.Resources = createDriverNodePluginResouces(nfsPluginResource) - NFSDriver.Spec.NodePlugin.UpdateStrategy = &v1.DaemonSetUpdateStrategy{ - Type: v1.RollingUpdateDaemonSetStrategyType, - } - - if CSIParam.CSIDomainLabels != "" { - domainLabels := strings.Split(CSIParam.CSIDomainLabels, ",") - NFSDriver.Spec.NodePlugin.Topology = &csiopv1.TopologySpec{ - DomainLabels: domainLabels, - } - } - - if CSIParam.RBDPluginUpdateStrategy == "OnDelete" { - NFSDriver.Spec.NodePlugin.UpdateStrategy = &v1.DaemonSetUpdateStrategy{ - Type: v1.OnDeleteDaemonSetStrategyType, - } - } - - err = r.createOrUpdateDriverResource(clusterInfo, NFSDriver) - if err != nil { - return errors.Wrapf(err, "failed to create or update NFS driver resource %q", NFSDriver.Name) - } - - return nil -} - -func (r ReconcileCSI) createOrUpdateDriverResource(clusterInfo *cephclient.ClusterInfo, driverResource *csiopv1.Driver) error { - spec := driverResource.Spec - - err := r.client.Get(r.opManagerContext, types.NamespacedName{Name: driverResource.Name, Namespace: r.opConfig.OperatorNamespace}, driverResource) - if err != nil { - if apierrors.IsNotFound(err) { - err = r.client.Create(r.opManagerContext, driverResource) - if err != nil { - return errors.Wrapf(err, "failed to create CSI-operator driver CR %q", driverResource.Name) - } - - logger.Infof("successfully created CSI driver cr %q", driverResource.Name) - return nil - } - return errors.Wrapf(err, "failed to get CSI-operator driver CR %q", opConfigCRName) - } - - driverResource.Spec = spec - err = r.client.Update(r.opManagerContext, driverResource) - if err != nil { - return errors.Wrapf(err, "failed to update CSI-operator driver CR %q", driverResource.Name) - } - - logger.Infof("successfully updated CSI-operator driver resource %q", driverResource.Name) - return nil -} - -func (r *ReconcileCSI) generateDriverSpec(cluster cephv1.CephCluster) (csiopv1.DriverSpec, error) { - cephfsClientType := csiopv1.KernelCephFsClient - if CSIParam.ForceCephFSKernelClient == "false" { - cephfsClientType = csiopv1.AutoDetectCephFsClient - } - imageSetCmName, err := r.createImageSetConfigmap() - if err != nil { - return csiopv1.DriverSpec{}, errors.Wrapf(err, "failed to create ceph-CSI operator config ImageSetConfigmap for CR %s", opConfigCRName) - } - - driverSpec := csiopv1.DriverSpec{ - Log: &csiopv1.LogSpec{ - Verbosity: int(CSIParam.LogLevel), - }, - ImageSet: &corev1.LocalObjectReference{ - Name: imageSetCmName, - }, - ClusterName: &cluster.Name, - EnableMetadata: &CSIParam.CSIEnableMetadata, - GenerateOMapInfo: &CSIParam.EnableOMAPGenerator, - FsGroupPolicy: k8scsiv1.FileFSGroupPolicy, - NodePlugin: &csiopv1.NodePluginSpec{ - PodCommonSpec: csiopv1.PodCommonSpec{ - PrioritylClassName: &CSIParam.PluginPriorityClassName, - Affinity: &corev1.Affinity{ - NodeAffinity: getNodeAffinity(pluginNodeAffinityEnv, &corev1.NodeAffinity{}), - }, - Tolerations: getToleration(pluginTolerationsEnv, []corev1.Toleration{}), - }, - Resources: csiopv1.NodePluginResourcesSpec{}, - KubeletDirPath: CSIParam.KubeletDirPath, - EnableSeLinuxHostMount: &CSIParam.EnablePluginSelinuxHostMount, - }, - ControllerPlugin: &csiopv1.ControllerPluginSpec{ - PodCommonSpec: csiopv1.PodCommonSpec{ - PrioritylClassName: &CSIParam.ProvisionerPriorityClassName, - Affinity: &corev1.Affinity{ - NodeAffinity: getNodeAffinity(provisionerNodeAffinityEnv, &corev1.NodeAffinity{}), - }, - Tolerations: getToleration(provisionerTolerationsEnv, []corev1.Toleration{}), - }, - Replicas: &CSIParam.ProvisionerReplicas, - Resources: csiopv1.ControllerPluginResourcesSpec{}, - }, - DeployCsiAddons: &CSIParam.EnableCSIAddonsSideCar, - CephFsClientType: cephfsClientType, - } - - // Apply Multus annotations if the cluster uses Multus networking - if cluster.Spec.Network.IsMultus() { - // Apply Multus to ControllerPlugin only - if driverSpec.ControllerPlugin.PodCommonSpec.Annotations == nil { - driverSpec.ControllerPlugin.PodCommonSpec.Annotations = make(map[string]string) - } - controllerObjectMeta := metav1.ObjectMeta{ - Annotations: driverSpec.ControllerPlugin.PodCommonSpec.Annotations, - } - err = r.applyMultusToObjectMeta(cluster.Namespace, &cluster.Spec.Network, &controllerObjectMeta) - if err != nil { - return csiopv1.DriverSpec{}, errors.Wrap(err, "failed to apply multus configuration to ControllerPlugin") - } - driverSpec.ControllerPlugin.PodCommonSpec.Annotations = controllerObjectMeta.Annotations - } - - return driverSpec, nil -} - -// applyMultusToObjectMeta applies Multus network annotations to the given ObjectMeta -func (r *ReconcileCSI) applyMultusToObjectMeta(clusterNamespace string, netSpec *cephv1.NetworkSpec, objectMeta *metav1.ObjectMeta) error { - return k8sutil.ApplyMultus(clusterNamespace, netSpec, objectMeta) -} - -func getPodVolumes(volumeEnvVar, volumeMountEnvVar string) []csiopv1.VolumeSpec { - volumes := extractVolumes(volumeEnvVar) - volumeMounts := extractVolumeMounts(volumeMountEnvVar) - return append(volumes, volumeMounts...) -} - -func extractVolumes(volumeEnvVar string) []csiopv1.VolumeSpec { - volumesRaw := k8sutil.GetOperatorSetting(volumeEnvVar, "") - if volumesRaw == "" { - return nil - } - volumes, err := k8sutil.YamlToVolumes(volumesRaw) - if err != nil { - logger.Errorf("failed to parse %q for %q. %v", volumesRaw, volumeEnvVar, err) - return nil - } - // Convert the volumes to csiopv1.VolumeSpec, where the mounts are empty. - // The CSI volume does not enforce the volume and mount to be defined together, - // so we can just have volumes defined here without mounts. - csiVolumes := []csiopv1.VolumeSpec{} - for i := range volumes { - csiVolumes = append(csiVolumes, csiopv1.VolumeSpec{ - Volume: volumes[i], - }) - } - return csiVolumes -} - -func extractVolumeMounts(volumeMountsEnvVar string) []csiopv1.VolumeSpec { - volumeMountsRaw := k8sutil.GetOperatorSetting(volumeMountsEnvVar, "") - if volumeMountsRaw == "" { - return nil - } - volumeMounts, err := k8sutil.YamlToVolumeMounts(volumeMountsRaw) - if err != nil { - logger.Errorf("failed to parse %q for %q. %v", volumeMountsRaw, configName, err) - return nil - } - // Convert the volume mounts to csiopv1.VolumeSpec, where the volumes are empty. - // The CSI volume does not enforce the volume and mount to be defined together, - // so we can just have mounts defined here without volumes. - csiVolumes := []csiopv1.VolumeSpec{} - for i := range volumeMounts { - csiVolumes = append(csiVolumes, csiopv1.VolumeSpec{ - Mount: volumeMounts[i], - }) - } - return csiVolumes -} - -func createDriverControllerPluginResources(key string) csiopv1.ControllerPluginResourcesSpec { - controllerPluginResources := csiopv1.ControllerPluginResourcesSpec{} - resource := getComputeResource(key) - - for _, r := range resource { - if !reflect.DeepEqual(r.Resource, corev1.ResourceRequirements{}) { - switch { - case strings.Contains(r.Name, "provisioner"): - controllerPluginResources.Provisioner = &corev1.ResourceRequirements{ - Limits: r.Resource.Limits, - Requests: r.Resource.Requests, - } - case strings.Contains(r.Name, "resizer"): - controllerPluginResources.Resizer = &corev1.ResourceRequirements{ - Limits: r.Resource.Limits, - Requests: r.Resource.Requests, - } - case strings.Contains(r.Name, "snapshotter"): - controllerPluginResources.Snapshotter = &corev1.ResourceRequirements{ - Limits: r.Resource.Limits, - Requests: r.Resource.Requests, - } - case strings.Contains(r.Name, "attacher"): - controllerPluginResources.Attacher = &corev1.ResourceRequirements{ - Limits: r.Resource.Limits, - Requests: r.Resource.Requests, - } - case strings.Contains(r.Name, "plugin"): - controllerPluginResources.Plugin = &corev1.ResourceRequirements{ - Limits: r.Resource.Limits, - Requests: r.Resource.Requests, - } - case strings.Contains(r.Name, "omap-generator"): - controllerPluginResources.OMapGenerator = &corev1.ResourceRequirements{ - Limits: r.Resource.Limits, - Requests: r.Resource.Requests, - } - case strings.Contains(r.Name, "addons"): - controllerPluginResources.Addons = &corev1.ResourceRequirements{ - Limits: r.Resource.Limits, - Requests: r.Resource.Requests, - } - } - } - } - return controllerPluginResources -} - -func createDriverNodePluginResouces(key string) csiopv1.NodePluginResourcesSpec { - nodePluginResources := csiopv1.NodePluginResourcesSpec{} - resource := getComputeResource(key) - - for _, r := range resource { - if !reflect.DeepEqual(r.Resource, corev1.ResourceRequirements{}) { - if strings.Contains(r.Name, "registrar") { - nodePluginResources.Registrar = &corev1.ResourceRequirements{ - Limits: r.Resource.Limits, - Requests: r.Resource.Requests, - } - } else if strings.Contains(r.Name, "plugin") { - nodePluginResources.Plugin = &corev1.ResourceRequirements{ - Limits: r.Resource.Limits, - Requests: r.Resource.Requests, - } - } else if strings.Contains(r.Name, "addons") { - nodePluginResources.Addons = &corev1.ResourceRequirements{ - Limits: r.Resource.Limits, - Requests: r.Resource.Requests, - } - } - } - } - return nodePluginResources -} - -// transferCSIDriverOwner update CSIDriver and returns the error if any -func (r *ReconcileCSI) transferCSIDriverOwner(ctx context.Context, name string) error { - logger.Info("adding annotation to CSIDriver resource for csi-operator to own it") - csiDriver, err := r.context.Clientset.StorageV1().CSIDrivers().Get(ctx, name, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - logger.Debugf("%s CSIDriver not found; skipping ownership transfer.", name) - return nil - } - } - - key := "csi.ceph.io/ownerref" - ownerObjKey := client.ObjectKeyFromObject(csiDriver) - ownerObjKey.Namespace = r.opConfig.OperatorNamespace - val, err := json.Marshal(ownerObjKey) - if err != nil { - return errors.Wrapf(err, "failed to marshal owner object key %q", ownerObjKey.Name) - } - - annotations := csiDriver.GetAnnotations() - if annotations == nil { - annotations = map[string]string{} - csiDriver.SetAnnotations(annotations) - } - if oldValue, exist := annotations[key]; !exist || oldValue != string(val) { - annotations[key] = string(val) - } else { - return nil - } - _, err = r.context.Clientset.StorageV1().CSIDrivers().Update(ctx, csiDriver, metav1.UpdateOptions{}) - if err != nil { - return errors.Wrapf(err, "failed to update CSIDriver %s", name) - } - - return nil -} diff --git a/pkg/operator/ceph/csi/operator_driver_test.go b/pkg/operator/ceph/csi/operator_driver_test.go deleted file mode 100644 index 02bc77dcd..000000000 --- a/pkg/operator/ceph/csi/operator_driver_test.go +++ /dev/null @@ -1,233 +0,0 @@ -/* -Copyright 2024 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "context" - "fmt" - "testing" - - csiopv1 "github.com/ceph/ceph-csi-operator/api/v1" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" - "github.com/rook/rook/pkg/client/clientset/versioned/scheme" - "github.com/rook/rook/pkg/clusterd" - clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test" - opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" - testop "github.com/rook/rook/pkg/operator/test" - "github.com/stretchr/testify/assert" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func TestReconcileCSI_createOrUpdateDriverResources(t *testing.T) { - tests := []struct { - name string - networkSpec cephv1.NetworkSpec - expectMultusAnnots bool - }{ - { - name: "without multus", - networkSpec: cephv1.NetworkSpec{ - Connections: &cephv1.ConnectionsSpec{ - Encryption: &cephv1.EncryptionSpec{ - Enabled: true, - }, - }, - }, - expectMultusAnnots: false, - }, - { - name: "with multus", - networkSpec: cephv1.NetworkSpec{ - Provider: "multus", - Selectors: map[cephv1.CephNetworkType]string{ - "public": "rook-ceph/rook-ceph-public-network", - "cluster": "rook-ceph/rook-ceph-cluster-network", - }, - }, - expectMultusAnnots: true, - }, - } - - CSIParam.CSIRBDPodLabels = map[string]string{"rbd-label": "rbd-value"} - CSIParam.CSICephFSPodLabels = map[string]string{"cephfs-label": "cephfs-value"} - CSIParam.CSINFSPodLabels = map[string]string{"nfs-label": "nfs-value"} - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ns := "test" - r := &ReconcileCSI{ - context: &clusterd.Context{ - Clientset: testop.New(t, 1), - RookClientset: rookclient.NewSimpleClientset(), - }, - opManagerContext: context.TODO(), - opConfig: opcontroller.OperatorConfig{ - OperatorNamespace: "test", - }, - } - cluster := &cephv1.CephCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testCluster", - Namespace: ns, - }, - Spec: cephv1.ClusterSpec{ - CSI: cephv1.CSIDriverSpec{ - ReadAffinity: cephv1.ReadAffinitySpec{ - Enabled: true, - CrushLocationLabels: []string{"kubernetes.io/hostname"}, - }, - CephFS: cephv1.CSICephFSSpec{ - KernelMountOptions: "ms_mode=crc", - }, - }, - Network: tt.networkSpec, - }, - } - driver := &csiopv1.Driver{} - - // Register operator types with the runtime scheme. - s := scheme.Scheme - s.AddKnownTypes(cephv1.SchemeGroupVersion, driver, &cephv1.CephCluster{}, &v1.ConfigMap{}) - object := []runtime.Object{ - cluster, - } - cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() - r.client = cl - - EnableRBD = true - EnableCephFS = true - EnableNFS = true - - c := clienttest.CreateTestClusterInfo(3) - c.Namespace = ns - c.SetName("testcluster") - c.NamespacedName() - - err := r.createOrUpdateDriverResources(*cluster, c) - assert.NoError(t, err) - - // Test RBD driver - err = cl.Get(context.TODO(), types.NamespacedName{Name: fmt.Sprintf("%s.rbd.csi.ceph.com", c.Namespace), Namespace: ns}, driver) - assert.NoError(t, err) - if tt.expectMultusAnnots { - // Verify that both NodePlugin and ControllerPlugin have Multus annotations - assert.Nil(t, driver.Spec.NodePlugin.PodCommonSpec.Annotations) - assert.NotContains(t, driver.Spec.NodePlugin.PodCommonSpec.Annotations, "k8s.v1.cni.cncf.io/networks") - - assert.NotNil(t, driver.Spec.ControllerPlugin.PodCommonSpec.Annotations) - assert.Contains(t, driver.Spec.ControllerPlugin.PodCommonSpec.Annotations, "k8s.v1.cni.cncf.io/networks") - - // Verify the annotation value contains the public network only not the cluster network - nodeMultusAnnotation := driver.Spec.NodePlugin.PodCommonSpec.Annotations["k8s.v1.cni.cncf.io/networks"] - assert.NotContains(t, nodeMultusAnnotation, "rook-ceph-public-network") - assert.NotContains(t, nodeMultusAnnotation, "rook-ceph-cluster-network") - - controllerMultusAnnotation := driver.Spec.ControllerPlugin.PodCommonSpec.Annotations["k8s.v1.cni.cncf.io/networks"] - assert.Contains(t, controllerMultusAnnotation, "rook-ceph-public-network") - assert.NotContains(t, controllerMultusAnnotation, "rook-ceph-cluster-network") - } else { - // Verify no Multus annotations when not using Multus - if driver.Spec.NodePlugin.PodCommonSpec.Annotations != nil { - assert.NotContains(t, driver.Spec.NodePlugin.PodCommonSpec.Annotations, "k8s.v1.cni.cncf.io/networks") - } - if driver.Spec.ControllerPlugin.PodCommonSpec.Annotations != nil { - assert.NotContains(t, driver.Spec.ControllerPlugin.PodCommonSpec.Annotations, "k8s.v1.cni.cncf.io/networks") - } - } - - assert.Equal(t, "rbd-value", driver.Spec.ControllerPlugin.PodCommonSpec.Labels["rbd-label"]) - assert.Equal(t, "rbd-value", driver.Spec.NodePlugin.PodCommonSpec.Labels["rbd-label"]) - assert.NotEqualValues(t, "rbd-value", driver.Spec.NodePlugin.PodCommonSpec.Labels["cephfs-label"]) - - // Test CephFS driver - err = cl.Get(context.TODO(), types.NamespacedName{Name: fmt.Sprintf("%s.cephfs.csi.ceph.com", c.Namespace), Namespace: ns}, driver) - assert.NoError(t, err) - - assert.Equal(t, "cephfs-value", driver.Spec.ControllerPlugin.PodCommonSpec.Labels["cephfs-label"]) - assert.Equal(t, "cephfs-value", driver.Spec.NodePlugin.PodCommonSpec.Labels["cephfs-label"]) - assert.NotEqualValues(t, "cephfs-value", driver.Spec.NodePlugin.PodCommonSpec.Labels["rbd-label"]) - - // Test NFS driver - err = cl.Get(context.TODO(), types.NamespacedName{Name: fmt.Sprintf("%s.nfs.csi.ceph.com", c.Namespace), Namespace: ns}, driver) - assert.NoError(t, err) - - assert.Equal(t, "nfs-value", driver.Spec.ControllerPlugin.PodCommonSpec.Labels["nfs-label"]) - assert.Equal(t, "nfs-value", driver.Spec.NodePlugin.PodCommonSpec.Labels["nfs-label"]) - assert.NotEqualValues(t, "nfs-value", driver.Spec.NodePlugin.PodCommonSpec.Labels["rbd-label"]) - - cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() - r.client = cl - - EnableRBD = true - EnableCephFS = true - EnableNFS = true - - // set a valid rbd volume - t.Setenv(rbdPluginVolume, ` - - name: myvola - hostPath: - path: /myvola - - name: myvolb - hostPath: - path: /myvolb`) - t.Setenv(rbdPluginVolumeMount, ` - - name: myvola - mountPath: /myvol - readOnly: true`) - - // set an invalid cephfs volume, but a valid mount - t.Setenv(cephFSPluginVolume, ` - - name: myvol - host`) - t.Setenv(cephFSPluginVolumeMount, ` - - name: myvol - mountPath: /myvol - readOnly: true`) - - err = r.createOrUpdateDriverResources(*cluster, c) - assert.NoError(t, err) - - err = cl.Get(context.TODO(), types.NamespacedName{Name: fmt.Sprintf("%s.rbd.csi.ceph.com", c.Namespace), Namespace: ns}, driver) - assert.NoError(t, err) - assert.Equal(t, 3, len(driver.Spec.NodePlugin.Volumes)) - - err = cl.Get(context.TODO(), types.NamespacedName{Name: fmt.Sprintf("%s.cephfs.csi.ceph.com", c.Namespace), Namespace: ns}, driver) - assert.NoError(t, err) - assert.Equal(t, 1, len(driver.Spec.NodePlugin.Volumes)) - - err = cl.Get(context.TODO(), types.NamespacedName{Name: fmt.Sprintf("%s.nfs.csi.ceph.com", c.Namespace), Namespace: ns}, driver) - assert.NoError(t, err) - - // set a valid cephfs volume - t.Setenv(cephFSPluginVolume, ` - - name: myvol - hostPath: - path: /myvol`) - err = r.createOrUpdateDriverResources(*cluster, c) - assert.NoError(t, err) - - err = cl.Get(context.TODO(), types.NamespacedName{Name: fmt.Sprintf("%s.cephfs.csi.ceph.com", c.Namespace), Namespace: ns}, driver) - assert.NoError(t, err) - assert.Equal(t, 2, len(driver.Spec.NodePlugin.Volumes)) - }) - } -} diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go deleted file mode 100644 index 6c20ed6f4..000000000 --- a/pkg/operator/ceph/csi/spec.go +++ /dev/null @@ -1,970 +0,0 @@ -/* -Copyright 2019 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "context" - _ "embed" - "path" - "strings" - "time" - - csiopv1 "github.com/ceph/ceph-csi-operator/api/v1" - opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" - "github.com/rook/rook/pkg/operator/k8sutil" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/pkg/errors" - apps "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - k8scsi "k8s.io/api/storage/v1beta1" - kerrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" -) - -type Param struct { - CSIPluginImage string - RegistrarImage string - ProvisionerImage string - AttacherImage string - SnapshotterImage string - ResizerImage string - DriverNamePrefix string - KubeletDirPath string - CsiLogRootPath string - ForceCephFSKernelClient string - CephFSKernelMountOptions string - CephFSPluginUpdateStrategy string - CephFSPluginUpdateStrategyMaxUnavailable string - NFSPluginUpdateStrategy string - RBDPluginUpdateStrategy string - RBDPluginUpdateStrategyMaxUnavailable string - PluginPriorityClassName string - ProvisionerPriorityClassName string - VolumeReplicationImage string - CSIAddonsImage string - ImagePullPolicy string - CSIClusterName string - CSIDomainLabels string - GRPCTimeout time.Duration - CSIEnableMetadata bool - EnablePluginSelinuxHostMount bool - EnableCSIHostNetwork bool - EnableOMAPGenerator bool - EnableRBDSnapshotter bool - EnableCephFSSnapshotter bool - EnableNFSSnapshotter bool - EnableCSIAddonsSideCar bool - MountCustomCephConf bool - EnableCSIEncryption bool - EnableCSITopology bool - EnableCrossNamespaceVolumeDataSource bool - EnableLiveness bool - CephFSAttachRequired bool - RBDAttachRequired bool - NFSAttachRequired bool - VolumeGroupSnapshotCLIFlag string - VolumeGroupSnapshotSupported bool - LogLevel uint8 - SidecarLogLevel uint8 - CephFSLivenessMetricsPort uint16 - CSIAddonsPort uint16 - CSIAddonsRBDProvisionerPort uint16 - CSIAddonsCephFSProvisionerPort uint16 - RBDLivenessMetricsPort uint16 - KubeApiBurst uint16 - KubeApiQPS float32 - LeaderElectionLeaseDuration time.Duration - LeaderElectionRenewDeadline time.Duration - LeaderElectionRetryPeriod time.Duration - ProvisionerReplicas int32 - CSICephFSPodLabels map[string]string - CSINFSPodLabels map[string]string - CSIRBDPodLabels map[string]string - CSILogRotation bool - CsiComponentName string - CSILogRotationMaxSize string - CSILogRotationPeriod string - Privileged bool -} - -type templateParam struct { - Param - // non-global template only parameters - Namespace string -} - -var ( - CSIParam Param - - EnableRBD = false - EnableCephFS = false - EnableNFS = false - enableCSIOperator = false - CustomCSICephConfigExists = false - - // driver names - CephFSDriverName string - NFSDriverName string - RBDDriverName string - - // configuration map for csi - ConfigName = "rook-ceph-csi-config" - ConfigKey = "csi-cluster-config-json" -) - -// Specify default images as var instead of const so that they can be overridden with the Go -// linker's -X flag. This allows users to easily build images with a different opinionated set of -// images without having to specify them manually in charts/manifests which can make upgrades more -// manually challenging. -var ( - // image names - DefaultCSIPluginImage = "quay.io/cephcsi/cephcsi:v3.16.2" - DefaultRegistrarImage = "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.16.0" - DefaultProvisionerImage = "registry.k8s.io/sig-storage/csi-provisioner:v6.1.1" - DefaultAttacherImage = "registry.k8s.io/sig-storage/csi-attacher:v4.11.0" - DefaultSnapshotterImage = "registry.k8s.io/sig-storage/csi-snapshotter:v8.5.0" - DefaultResizerImage = "registry.k8s.io/sig-storage/csi-resizer:v2.1.0" - DefaultCSIAddonsImage = "quay.io/csiaddons/k8s-sidecar:v0.14.0" - - // image pull policy - DefaultCSIImagePullPolicy = string(corev1.PullIfNotPresent) - - // Local package template path for RBD - //go:embed template/rbd/csi-rbdplugin.yaml - RBDPluginTemplatePath string - //go:embed template/rbd/csi-rbdplugin-provisioner-dep.yaml - RBDProvisionerDepTemplatePath string - //go:embed template/rbd/csi-rbdplugin-svc.yaml - RBDPluginServiceTemplatePath string - - // Local package template path for CephFS - //go:embed template/cephfs/csi-cephfsplugin.yaml - CephFSPluginTemplatePath string - //go:embed template/cephfs/csi-cephfsplugin-provisioner-dep.yaml - CephFSProvisionerDepTemplatePath string - //go:embed template/cephfs/csi-cephfsplugin-svc.yaml - CephFSPluginServiceTemplatePath string - - // Local package template path for NFS - //go:embed template/nfs/csi-nfsplugin.yaml - NFSPluginTemplatePath string - //go:embed template/nfs/csi-nfsplugin-provisioner-dep.yaml - NFSProvisionerDepTemplatePath string - - //go:embed template/csi-logrotate-sidecar.yaml - LogrotateTemplatePath string - - holderEnabled bool - - csiRootPath string -) - -const ( - // common tolerations and node affinity - provisionerTolerationsEnv = "CSI_PROVISIONER_TOLERATIONS" - provisionerNodeAffinityEnv = "CSI_PROVISIONER_NODE_AFFINITY" - pluginTolerationsEnv = "CSI_PLUGIN_TOLERATIONS" - pluginNodeAffinityEnv = "CSI_PLUGIN_NODE_AFFINITY" - - // CephFS tolerations and node affinity - cephFSProvisionerTolerationsEnv = "CSI_CEPHFS_PROVISIONER_TOLERATIONS" - cephFSProvisionerNodeAffinityEnv = "CSI_CEPHFS_PROVISIONER_NODE_AFFINITY" - cephFSPluginTolerationsEnv = "CSI_CEPHFS_PLUGIN_TOLERATIONS" - cephFSPluginNodeAffinityEnv = "CSI_CEPHFS_PLUGIN_NODE_AFFINITY" - - // NFS tolerations and node affinity - nfsProvisionerTolerationsEnv = "CSI_NFS_PROVISIONER_TOLERATIONS" - nfsProvisionerNodeAffinityEnv = "CSI_NFS_PROVISIONER_NODE_AFFINITY" - nfsPluginTolerationsEnv = "CSI_NFS_PLUGIN_TOLERATIONS" - nfsPluginNodeAffinityEnv = "CSI_NFS_PLUGIN_NODE_AFFINITY" - - // RBD tolerations and node affinity - rbdProvisionerTolerationsEnv = "CSI_RBD_PROVISIONER_TOLERATIONS" - rbdProvisionerNodeAffinityEnv = "CSI_RBD_PROVISIONER_NODE_AFFINITY" - rbdPluginTolerationsEnv = "CSI_RBD_PLUGIN_TOLERATIONS" - rbdPluginNodeAffinityEnv = "CSI_RBD_PLUGIN_NODE_AFFINITY" - - // compute resource for CSI pods - rbdProvisionerResource = "CSI_RBD_PROVISIONER_RESOURCE" - rbdPluginResource = "CSI_RBD_PLUGIN_RESOURCE" - - cephFSProvisionerResource = "CSI_CEPHFS_PROVISIONER_RESOURCE" - cephFSPluginResource = "CSI_CEPHFS_PLUGIN_RESOURCE" - - nfsProvisionerResource = "CSI_NFS_PROVISIONER_RESOURCE" - nfsPluginResource = "CSI_NFS_PLUGIN_RESOURCE" - - cephFSPluginVolume = "CSI_CEPHFS_PLUGIN_VOLUME" - cephFSPluginVolumeMount = "CSI_CEPHFS_PLUGIN_VOLUME_MOUNT" - - rbdPluginVolume = "CSI_RBD_PLUGIN_VOLUME" - rbdPluginVolumeMount = "CSI_RBD_PLUGIN_VOLUME_MOUNT" - - nfsPluginVolume = "CSI_NFS_PLUGIN_VOLUME" - nfsPluginVolumeMount = "CSI_NFS_PLUGIN_VOLUME_MOUNT" - - // kubelet directory path - DefaultKubeletDirPath = "/var/lib/kubelet" - - // gRPC metrics and liveness port for CephFS and RBD - DefaultCephFSGRPCMerticsPort uint16 = 9091 - DefaultCephFSLivenessMerticsPort uint16 = 9081 - DefaultRBDGRPCMerticsPort uint16 = 9090 - DefaultRBDLivenessMerticsPort uint16 = 9080 - DefaultCSIAddonsPort uint16 = 9070 - DefaultCSIAddonsRBDProvisionerPort uint16 = 9070 - DefaultCSIAddonsCephFSProvisionerPort uint16 = 9070 - - // default log level for csi containers - defaultLogLevel uint8 = 0 - defaultSidecarLogLevel uint8 = 0 - - // default leader election flags - defaultLeaderElectionLeaseDuration = 137 * time.Second - defaultLeaderElectionRenewDeadline = 107 * time.Second - defaultLeaderElectionRetryPeriod = 26 * time.Second - - // GRPC timeout. - defaultGRPCTimeout = 150 - grpcTimeout = "CSI_GRPC_TIMEOUT_SECONDS" - // default provisioner replicas - defaultProvisionerReplicas int32 = 2 - - // update strategy - rollingUpdate = "RollingUpdate" - onDelete = "OnDelete" - - // driver daemonset names - CsiRBDPlugin = "csi-rbdplugin" - CsiCephFSPlugin = "csi-cephfsplugin" - CsiNFSPlugin = "csi-nfsplugin" - - // driver deployment names - csiRBDProvisioner = "csi-rbdplugin-provisioner" - csiCephFSProvisioner = "csi-cephfsplugin-provisioner" - csiNFSProvisioner = "csi-nfsplugin-provisioner" - - // cephcsi container names - csiRBDContainerName = "csi-rbdplugin" - csiCephFSContainerName = "csi-cephfsplugin" - csiNFSContainerName = "csi-nfsplugin" - - RBDDriverShortName = "rbd" - CephFSDriverShortName = "cephfs" - NFSDriverShortName = "nfs" - rbdDriverSuffix = "rbd.csi.ceph.com" - cephFSDriverSuffix = "cephfs.csi.ceph.com" - nfsDriverSuffix = "nfs.csi.ceph.com" - nodePlugin = "node-plugin" - controllerPlugin = "controller-plugin" -) - -func CSIEnabled() bool { - return EnableRBD || EnableCephFS || EnableNFS -} - -func IsHolderEnabled() bool { - return holderEnabled -} - -func validateCSIParam() error { - if len(CSIParam.CSIPluginImage) == 0 { - return errors.New("missing csi rbd plugin image") - } - if len(CSIParam.RegistrarImage) == 0 { - return errors.New("missing csi registrar image") - } - if len(CSIParam.ProvisionerImage) == 0 { - return errors.New("missing csi provisioner image") - } - if len(CSIParam.AttacherImage) == 0 { - return errors.New("missing csi attacher image") - } - - return nil -} - -func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { - var ( - err error - rbdPlugin, cephfsPlugin, nfsPlugin *apps.DaemonSet - rbdProvisionerDeployment, cephfsProvisionerDeployment, nfsProvisionerDeployment *apps.Deployment - rbdService, cephfsService *corev1.Service - csiDriverobj v1CsiDriver - ) - - tp := templateParam{ - Param: CSIParam, - Namespace: r.opConfig.OperatorNamespace, - } - - tp.Param.MountCustomCephConf = CustomCSICephConfigExists - - if EnableRBD { - tp.CsiComponentName = nodePlugin - tp.CsiLogRootPath = path.Join(csiRootPath, RBDDriverName) - rbdPlugin, err = templateToDaemonSet("rbdplugin", RBDPluginTemplatePath, tp) - if err != nil { - return errors.Wrap(err, "failed to load rbdplugin template") - } - rbdPlugin.Spec.RevisionHistoryLimit = opcontroller.RevisionHistoryLimit() - if tp.CSILogRotation { - applyLogrotateSidecar(&rbdPlugin.Spec.Template, "csi-rbd-daemonset-log-collector", LogrotateTemplatePath, tp) - } - - tp.CsiComponentName = controllerPlugin - rbdProvisionerDeployment, err = templateToDeployment("rbd-provisioner", RBDProvisionerDepTemplatePath, tp) - if err != nil { - return errors.Wrap(err, "failed to load rbd provisioner deployment template") - } - if tp.CSILogRotation { - applyLogrotateSidecar(&rbdProvisionerDeployment.Spec.Template, "csi-rbd-deployment-log-collector", LogrotateTemplatePath, tp) - } - rbdProvisionerDeployment.Spec.Template.Spec.HostNetwork = opcontroller.EnforceHostNetwork() - rbdProvisionerDeployment.Spec.RevisionHistoryLimit = opcontroller.RevisionHistoryLimit() - - // Create service if either liveness or GRPC metrics are enabled. - if CSIParam.EnableLiveness { - rbdService, err = templateToService("rbd-service", RBDPluginServiceTemplatePath, tp) - if err != nil { - return errors.Wrap(err, "failed to load rbd plugin service template") - } - rbdService.Namespace = r.opConfig.OperatorNamespace - } - } - if EnableCephFS { - tp.CsiComponentName = nodePlugin - tp.CsiLogRootPath = path.Join(csiRootPath, CephFSDriverName) - cephfsPlugin, err = templateToDaemonSet("cephfsplugin", CephFSPluginTemplatePath, tp) - if err != nil { - return errors.Wrap(err, "failed to load CephFS plugin template") - } - cephfsPlugin.Spec.RevisionHistoryLimit = opcontroller.RevisionHistoryLimit() - - if tp.CSILogRotation { - applyLogrotateSidecar(&cephfsPlugin.Spec.Template, "csi-cephfs-daemonset-log-collector", LogrotateTemplatePath, tp) - } - - tp.CsiComponentName = controllerPlugin - cephfsProvisionerDeployment, err = templateToDeployment("cephfs-provisioner", CephFSProvisionerDepTemplatePath, tp) - if err != nil { - return errors.Wrap(err, "failed to load rbd provisioner deployment template") - } - if tp.CSILogRotation { - applyLogrotateSidecar(&cephfsProvisionerDeployment.Spec.Template, "csi-cephfs-deployment-log-collector", LogrotateTemplatePath, tp) - } - cephfsProvisionerDeployment.Spec.Template.Spec.HostNetwork = opcontroller.EnforceHostNetwork() - cephfsProvisionerDeployment.Spec.RevisionHistoryLimit = opcontroller.RevisionHistoryLimit() - - // Create service if either liveness or GRPC metrics are enabled. - if CSIParam.EnableLiveness { - cephfsService, err = templateToService("cephfs-service", CephFSPluginServiceTemplatePath, tp) - if err != nil { - return errors.Wrap(err, "failed to load cephfs plugin service template") - } - cephfsService.Namespace = r.opConfig.OperatorNamespace - } - } - - if EnableNFS { - tp.CsiComponentName = nodePlugin - tp.CsiLogRootPath = path.Join(csiRootPath, NFSDriverName) - nfsPlugin, err = templateToDaemonSet("nfsplugin", NFSPluginTemplatePath, tp) - if err != nil { - return errors.Wrap(err, "failed to load nfs plugin template") - } - nfsPlugin.Spec.RevisionHistoryLimit = opcontroller.RevisionHistoryLimit() - if tp.CSILogRotation { - applyLogrotateSidecar(&nfsPlugin.Spec.Template, "csi-nfs-daemonset-log-collector", LogrotateTemplatePath, tp) - } - - tp.CsiComponentName = controllerPlugin - nfsProvisionerDeployment, err = templateToDeployment("nfs-provisioner", NFSProvisionerDepTemplatePath, tp) - if err != nil { - return errors.Wrap(err, "failed to load nfs provisioner deployment template") - } - if tp.CSILogRotation { - applyLogrotateSidecar(&nfsProvisionerDeployment.Spec.Template, "csi-nfs-deployment-log-collector", LogrotateTemplatePath, tp) - } - nfsProvisionerDeployment.Spec.Template.Spec.HostNetwork = opcontroller.EnforceHostNetwork() - nfsProvisionerDeployment.Spec.RevisionHistoryLimit = opcontroller.RevisionHistoryLimit() - } - - // get common provisioner tolerations and node affinity - provisionerTolerations := getToleration(provisionerTolerationsEnv, []corev1.Toleration{}) - provisionerNodeAffinity := getNodeAffinity(provisionerNodeAffinityEnv, &corev1.NodeAffinity{}) - - // get common plugin tolerations and node affinity - pluginTolerations := getToleration(pluginTolerationsEnv, []corev1.Toleration{}) - pluginNodeAffinity := getNodeAffinity(pluginNodeAffinityEnv, &corev1.NodeAffinity{}) - - if rbdPlugin != nil { - // get RBD plugin tolerations and node affinity, defaults to common tolerations and node affinity if not specified - rbdPluginTolerations := getToleration(rbdPluginTolerationsEnv, pluginTolerations) - rbdPluginNodeAffinity := getNodeAffinity(rbdPluginNodeAffinityEnv, pluginNodeAffinity) - // apply RBD plugin tolerations and node affinity - applyToPodSpec(&rbdPlugin.Spec.Template.Spec, rbdPluginNodeAffinity, rbdPluginTolerations) - // apply resource request and limit to rbdplugin containers - applyResourcesToContainers(rbdPluginResource, &rbdPlugin.Spec.Template.Spec) - // apply custom mounts to volumes - applyVolumeToPodSpec(rbdPluginVolume, &rbdPlugin.Spec.Template.Spec) - // apply custom mounts to volume mounts - applyVolumeMountToContainer(rbdPluginVolumeMount, "csi-rbdplugin", &rbdPlugin.Spec.Template.Spec) - err = ownerInfo.SetControllerReference(rbdPlugin) - if err != nil { - return errors.Wrapf(err, "failed to set owner reference to rbd plugin daemonset %q", rbdPlugin.Name) - } - err = r.applyCephClusterNetworkConfig(r.opManagerContext, &rbdPlugin.Spec.Template.ObjectMeta) - if err != nil { - return errors.Wrapf(err, "failed to apply network config to rbd plugin daemonset %q", rbdPlugin.Name) - } - err = k8sutil.CreateDaemonSet(r.opManagerContext, r.opConfig.OperatorNamespace, r.context.Clientset, rbdPlugin) - if err != nil { - return errors.Wrapf(err, "failed to start rbdplugin daemonset %q", rbdPlugin.Name) - } - k8sutil.AddRookVersionLabelToDaemonSet(rbdPlugin) - } - - if rbdProvisionerDeployment != nil { - // get RBD provisioner tolerations and node affinity, defaults to common tolerations and node affinity if not specified - rbdProvisionerTolerations := getToleration(rbdProvisionerTolerationsEnv, provisionerTolerations) - rbdProvisionerNodeAffinity := getNodeAffinity(rbdProvisionerNodeAffinityEnv, provisionerNodeAffinity) - // apply RBD provisioner tolerations and node affinity - applyToPodSpec(&rbdProvisionerDeployment.Spec.Template.Spec, rbdProvisionerNodeAffinity, rbdProvisionerTolerations) - // apply resource request and limit to rbd provisioner containers - applyResourcesToContainers(rbdProvisionerResource, &rbdProvisionerDeployment.Spec.Template.Spec) - err = ownerInfo.SetControllerReference(rbdProvisionerDeployment) - if err != nil { - return errors.Wrapf(err, "failed to set owner reference to rbd provisioner deployment %q", rbdProvisionerDeployment.Name) - } - antiAffinity := GetPodAntiAffinity("app", csiRBDProvisioner) - rbdProvisionerDeployment.Spec.Template.Spec.Affinity.PodAntiAffinity = &antiAffinity - rbdProvisionerDeployment.Spec.Strategy = apps.DeploymentStrategy{ - Type: apps.RecreateDeploymentStrategyType, - } - - err = r.applyCephClusterNetworkConfig(r.opManagerContext, &rbdProvisionerDeployment.Spec.Template.ObjectMeta) - if err != nil { - return errors.Wrapf(err, "failed to apply network config to rbd plugin provisioner deployment %q", rbdProvisionerDeployment.Name) - } - _, err = k8sutil.CreateOrUpdateDeployment(r.opManagerContext, r.context.Clientset, rbdProvisionerDeployment) - if err != nil { - return errors.Wrapf(err, "failed to start rbd provisioner deployment %q", rbdProvisionerDeployment.Name) - } - k8sutil.AddRookVersionLabelToDeployment(rbdProvisionerDeployment) - logger.Info("successfully started CSI Ceph RBD driver") - } - - if rbdService != nil { - rbdService.Namespace = r.opConfig.OperatorNamespace - err = ownerInfo.SetControllerReference(rbdService) - if err != nil { - return errors.Wrapf(err, "failed to set owner reference to rbd service %q", rbdService) - } - _, err = k8sutil.CreateOrUpdateService(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace, rbdService) - if err != nil { - return errors.Wrapf(err, "failed to create rbd service %q", rbdService.Name) - } - } - - if cephfsPlugin != nil { - // get CephFS plugin tolerations and node affinity, defaults to common tolerations and node affinity if not specified - cephFSPluginTolerations := getToleration(cephFSPluginTolerationsEnv, pluginTolerations) - cephFSPluginNodeAffinity := getNodeAffinity(cephFSPluginNodeAffinityEnv, pluginNodeAffinity) - // apply CephFS plugin tolerations and node affinity - applyToPodSpec(&cephfsPlugin.Spec.Template.Spec, cephFSPluginNodeAffinity, cephFSPluginTolerations) - // apply resource request and limit to cephfs plugin containers - applyResourcesToContainers(cephFSPluginResource, &cephfsPlugin.Spec.Template.Spec) - // apply custom mounts to volumes - applyVolumeToPodSpec(cephFSPluginVolume, &cephfsPlugin.Spec.Template.Spec) - // apply custom mounts to volume mounts - applyVolumeMountToContainer(cephFSPluginVolumeMount, "csi-cephfsplugin", &cephfsPlugin.Spec.Template.Spec) - err = ownerInfo.SetControllerReference(cephfsPlugin) - if err != nil { - return errors.Wrapf(err, "failed to set owner reference to cephfs plugin daemonset %q", cephfsPlugin.Name) - } - err = r.applyCephClusterNetworkConfig(r.opManagerContext, &cephfsPlugin.Spec.Template.ObjectMeta) - if err != nil { - return errors.Wrapf(err, "failed to apply network config to cephfs plugin daemonset %q", cephfsPlugin.Name) - } - - err = k8sutil.CreateDaemonSet(r.opManagerContext, r.opConfig.OperatorNamespace, r.context.Clientset, cephfsPlugin) - if err != nil { - return errors.Wrapf(err, "failed to start cephfs plugin daemonset %q", cephfsPlugin.Name) - } - k8sutil.AddRookVersionLabelToDaemonSet(cephfsPlugin) - } - - if cephfsProvisionerDeployment != nil { - // get CephFS provisioner tolerations and node affinity, defaults to common tolerations and node affinity if not specified - cephFSProvisionerTolerations := getToleration(cephFSProvisionerTolerationsEnv, provisionerTolerations) - cephFSProvisionerNodeAffinity := getNodeAffinity(cephFSProvisionerNodeAffinityEnv, provisionerNodeAffinity) - // apply CephFS provisioner tolerations and node affinity - applyToPodSpec(&cephfsProvisionerDeployment.Spec.Template.Spec, cephFSProvisionerNodeAffinity, cephFSProvisionerTolerations) - // get resource details for cephfs provisioner - // apply resource request and limit to cephfs provisioner containers - applyResourcesToContainers(cephFSProvisionerResource, &cephfsProvisionerDeployment.Spec.Template.Spec) - err = ownerInfo.SetControllerReference(cephfsProvisionerDeployment) - if err != nil { - return errors.Wrapf(err, "failed to set owner reference to cephfs provisioner deployment %q", cephfsProvisionerDeployment.Name) - } - antiAffinity := GetPodAntiAffinity("app", csiCephFSProvisioner) - cephfsProvisionerDeployment.Spec.Template.Spec.Affinity.PodAntiAffinity = &antiAffinity - cephfsProvisionerDeployment.Spec.Strategy = apps.DeploymentStrategy{ - Type: apps.RecreateDeploymentStrategyType, - } - - err = r.applyCephClusterNetworkConfig(r.opManagerContext, &cephfsProvisionerDeployment.Spec.Template.ObjectMeta) - if err != nil { - return errors.Wrapf(err, "failed to apply network config to cephfs plugin provisioner deployment %q", cephfsProvisionerDeployment.Name) - } - - _, err = k8sutil.CreateOrUpdateDeployment(r.opManagerContext, r.context.Clientset, cephfsProvisionerDeployment) - if err != nil { - return errors.Wrapf(err, "failed to start cephfs provisioner deployment %q", cephfsProvisionerDeployment.Name) - } - k8sutil.AddRookVersionLabelToDeployment(cephfsProvisionerDeployment) - logger.Info("successfully started CSI CephFS driver") - } - if cephfsService != nil { - err = ownerInfo.SetControllerReference(cephfsService) - if err != nil { - return errors.Wrapf(err, "failed to set owner reference to cephfs service %q", cephfsService) - } - _, err = k8sutil.CreateOrUpdateService(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace, cephfsService) - if err != nil { - return errors.Wrapf(err, "failed to create cephfs service %q", cephfsService.Name) - } - } - - if nfsPlugin != nil { - // get NFS plugin tolerations and node affinity, defaults to common tolerations and node affinity if not specified - nfsPluginTolerations := getToleration(nfsPluginTolerationsEnv, pluginTolerations) - nfsPluginNodeAffinity := getNodeAffinity(nfsPluginNodeAffinityEnv, pluginNodeAffinity) - // apply NFS plugin tolerations and node affinity - applyToPodSpec(&nfsPlugin.Spec.Template.Spec, nfsPluginNodeAffinity, nfsPluginTolerations) - // apply resource request and limit to nfs plugin containers - applyResourcesToContainers(nfsPluginResource, &nfsPlugin.Spec.Template.Spec) - // apply custom mounts to volumes - applyVolumeToPodSpec(nfsPluginVolume, &nfsPlugin.Spec.Template.Spec) - // apply custom mounts to volume mounts - applyVolumeMountToContainer(nfsPluginVolumeMount, "csi-nfsplugin", &nfsPlugin.Spec.Template.Spec) - err = ownerInfo.SetControllerReference(nfsPlugin) - if err != nil { - return errors.Wrapf(err, "failed to set owner reference to nfs plugin daemonset %q", nfsPlugin.Name) - } - err = r.applyCephClusterNetworkConfig(r.opManagerContext, &nfsPlugin.Spec.Template.ObjectMeta) - if err != nil { - return errors.Wrapf(err, "failed to apply network config to nfs plugin daemonset %q", nfsPlugin.Name) - } - - err = k8sutil.CreateDaemonSet(r.opManagerContext, r.opConfig.OperatorNamespace, r.context.Clientset, nfsPlugin) - if err != nil { - return errors.Wrapf(err, "failed to start nfs plugin daemonset %q", nfsPlugin.Name) - } - k8sutil.AddRookVersionLabelToDaemonSet(nfsPlugin) - } - - if nfsProvisionerDeployment != nil { - // get NFS provisioner tolerations and node affinity, defaults to common tolerations and node affinity if not specified - nfsProvisionerTolerations := getToleration(nfsProvisionerTolerationsEnv, provisionerTolerations) - nfsProvisionerNodeAffinity := getNodeAffinity(nfsProvisionerNodeAffinityEnv, provisionerNodeAffinity) - // apply NFS provisioner tolerations and node affinity - applyToPodSpec(&nfsProvisionerDeployment.Spec.Template.Spec, nfsProvisionerNodeAffinity, nfsProvisionerTolerations) - // get resource details for nfs provisioner - // apply resource request and limit to nfs provisioner containers - applyResourcesToContainers(nfsProvisionerResource, &nfsProvisionerDeployment.Spec.Template.Spec) - err = ownerInfo.SetControllerReference(nfsProvisionerDeployment) - if err != nil { - return errors.Wrapf(err, "failed to set owner reference to nfs provisioner deployment %q", nfsProvisionerDeployment.Name) - } - antiAffinity := GetPodAntiAffinity("app", csiNFSProvisioner) - nfsProvisionerDeployment.Spec.Template.Spec.Affinity.PodAntiAffinity = &antiAffinity - nfsProvisionerDeployment.Spec.Strategy = apps.DeploymentStrategy{ - Type: apps.RecreateDeploymentStrategyType, - } - - err = r.applyCephClusterNetworkConfig(r.opManagerContext, &nfsProvisionerDeployment.Spec.Template.ObjectMeta) - if err != nil { - return errors.Wrapf(err, "failed to apply network config to nfs provisioner deployment %q", nfsProvisionerDeployment.Name) - } - _, err = k8sutil.CreateOrUpdateDeployment(r.opManagerContext, r.context.Clientset, nfsProvisionerDeployment) - if err != nil { - return errors.Wrapf(err, "failed to start nfs provisioner deployment %q", nfsProvisionerDeployment.Name) - } - k8sutil.AddRookVersionLabelToDeployment(nfsProvisionerDeployment) - logger.Info("successfully started CSI NFS driver") - } - - if EnableRBD { - err = csiDriverobj.createCSIDriverInfo( - r.opManagerContext, r.context.Clientset, - RBDDriverName, k8sutil.GetOperatorSetting("CSI_RBD_FSGROUPPOLICY", string(k8scsi.FileFSGroupPolicy)), - tp.Param.RBDAttachRequired) - if err != nil { - return errors.Wrapf(err, "failed to create CSI driver object for %q", RBDDriverName) - } - } - if EnableCephFS { - err = csiDriverobj.createCSIDriverInfo( - r.opManagerContext, r.context.Clientset, - CephFSDriverName, k8sutil.GetOperatorSetting("CSI_CEPHFS_FSGROUPPOLICY", string(k8scsi.FileFSGroupPolicy)), - tp.Param.CephFSAttachRequired) - if err != nil { - return errors.Wrapf(err, "failed to create CSI driver object for %q", CephFSDriverName) - } - } - if EnableNFS { - err = csiDriverobj.createCSIDriverInfo(r.opManagerContext, r.context.Clientset, - NFSDriverName, k8sutil.GetOperatorSetting("CSI_NFS_FSGROUPPOLICY", string(k8scsi.FileFSGroupPolicy)), - tp.Param.NFSAttachRequired) - if err != nil { - return errors.Wrapf(err, "failed to create CSI driver object for %q", NFSDriverName) - } - } - - return nil -} - -func (r *ReconcileCSI) stopDrivers() error { - if !EnableRBD || EnableCSIOperator() { - logger.Debugf("either EnableRBD if `false` or EnableCSIOperator is `true`, `EnableRBD is %t` and `EnableCSIOperator is %t", EnableRBD, EnableCSIOperator()) - err := r.deleteCSIDriverResources(CsiRBDPlugin, csiRBDProvisioner, "csi-rbdplugin-metrics", RBDDriverName) - if err != nil { - return errors.Wrap(err, "failed to remove CSI Ceph RBD driver") - } - logger.Info("successfully removed CSI Ceph RBD driver") - } - - if !EnableCephFS || EnableCSIOperator() { - logger.Debugf("either EnableCephFS if `false` or EnableCSIOperator is `true`, `EnableCephFS is %t` and `EnableCSIOperator is %t", EnableRBD, EnableCSIOperator()) - err := r.deleteCSIDriverResources(CsiCephFSPlugin, csiCephFSProvisioner, "csi-cephfsplugin-metrics", CephFSDriverName) - if err != nil { - return errors.Wrap(err, "failed to remove CSI CephFS driver") - } - logger.Info("successfully removed CSI CephFS driver") - } - - if !EnableNFS || EnableCSIOperator() { - logger.Debugf("either EnableNFS if `false` or EnableCSIOperator is `true`, `EnableNFS is %t` and `EnableCSIOperator is %t", EnableRBD, EnableCSIOperator()) - err := r.deleteCSIDriverResources(CsiNFSPlugin, csiNFSProvisioner, "csi-nfsplugin-metrics", NFSDriverName) - if err != nil { - return errors.Wrap(err, "failed to remove CSI NFS driver") - } - logger.Info("successfully removed CSI NFS driver") - } - - return nil -} - -func (r *ReconcileCSI) deleteCSIDriverResources(daemonset, deployment, service, driverName string) error { - csiDriverobj := v1CsiDriver{} - err := k8sutil.DeleteDaemonset(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace, daemonset) - if err != nil { - return errors.Wrapf(err, "failed to delete the %q", daemonset) - } - - err = k8sutil.DeleteDeployment(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace, deployment) - if err != nil { - return errors.Wrapf(err, "failed to delete the %q", deployment) - } - - err = k8sutil.DeleteService(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace, service) - if err != nil { - return errors.Wrapf(err, "failed to delete the %q", service) - } - - if !EnableCSIOperator() { - err = csiDriverobj.deleteCSIDriverInfo(r.opManagerContext, r.context.Clientset, driverName) - if err != nil { - return errors.Wrapf(err, "failed to delete %q Driver Info", driverName) - } - } - - return nil -} - -func (r *ReconcileCSI) deleteRookCSICMIfExists() error { - logger.Debug("checking for existing rook-ceph-csi-config configmap to delete...") - - cm := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: ConfigName, - Namespace: r.opConfig.OperatorNamespace, - }, - } - - err := r.client.Delete(r.opManagerContext, cm) - if err != nil { - if kerrors.IsNotFound(err) { - logger.Debugf("configmap %q not found. Ignoring since it must already be deleted", ConfigName) - return nil - } - return errors.Wrapf(err, "failed to delete configmap %q in namespace %q", ConfigName, r.opConfig.OperatorNamespace) - } - - logger.Infof("successfully deleted configmap %q in namespace %q", ConfigName, r.opConfig.OperatorNamespace) - return nil -} - -// deleteCSIOperatorResources deletes CSI operator resources when CSI operator is disabled but CSI driver is enabled -func (r *ReconcileCSI) deleteCSIOperatorResources() error { - logger.Debug("cleaning up CSI operator resources since CSI operator is disabled but CSI driver is enabled") - - // Delete OperatorConfigs - err := r.deleteCSIOperatorConfigs() - if err != nil { - logger.Errorf("failed to delete CSI operator configs: %v", err) - } - - // Delete Drivers - err = r.deleteCSIDrivers() - if err != nil { - logger.Errorf("failed to delete CSI drivers: %v", err) - } - - // Delete ClientProfiles - err = r.deleteClientProfiles() - if err != nil { - logger.Errorf("failed to delete CSI client profiles: %v", err) - } - - // Delete CephConnections - err = r.deleteCephConnections() - if err != nil { - logger.Errorf("failed to delete CSI ceph connections: %v", err) - } - - logger.Info("successfully cleaned up CSI operator resources") - return nil -} - -// deleteCSIOperatorConfigsIfExist deletes OperatorConfig resources -func (r *ReconcileCSI) deleteCSIOperatorConfigs() error { - operatorConfigs := &csiopv1.OperatorConfigList{} - err := r.client.List(r.opManagerContext, operatorConfigs, &client.ListOptions{ - Namespace: r.opConfig.OperatorNamespace, - }) - if err != nil { - if _, ok := err.(*meta.NoKindMatchError); ok || kerrors.IsNotFound(err) { - logger.Debug("no CSI operator configs found") - return nil - } - return errors.Wrap(err, "failed to list CSI operator configs") - } - - for _, config := range operatorConfigs.Items { - logger.Debugf("deleting CSI operator config %q in namespace %q", config.Name, config.Namespace) - err := r.client.Delete(r.opManagerContext, &config) - if kerrors.IsNotFound(err) { - return nil - } - if err != nil && !kerrors.IsNotFound(err) { - return errors.Wrapf(err, "failed to delete CSI operator config %q", config.Name) - } - logger.Infof("successfully deleted CSI operator config %q", config.Name) - } - return nil -} - -// deleteCSIDriversIfExist deletes Driver resources -func (r *ReconcileCSI) deleteCSIDrivers() error { - drivers := &csiopv1.DriverList{} - err := r.client.List(r.opManagerContext, drivers, &client.ListOptions{ - Namespace: r.opConfig.OperatorNamespace, - }) - if err != nil { - if _, ok := err.(*meta.NoKindMatchError); ok || kerrors.IsNotFound(err) { - logger.Debug("no CSI drivers found") - return nil - } - return errors.Wrap(err, "failed to list CSI drivers") - } - - for _, driver := range drivers.Items { - logger.Debugf("deleting CSI driver %q in namespace %q", driver.Name, driver.Namespace) - err := r.client.Delete(r.opManagerContext, &driver) - if _, ok := err.(*meta.NoKindMatchError); ok || kerrors.IsNotFound(err) { - return nil - } - if err != nil && !kerrors.IsNotFound(err) { - return errors.Wrapf(err, "failed to delete CSI driver %q", driver.Name) - } - logger.Infof("successfully deleted CSI driver %q", driver.Name) - } - return nil -} - -// deleteClientProfilesIfExist deletes ClientProfile resources -func (r *ReconcileCSI) deleteClientProfiles() error { - clientProfiles := &csiopv1.ClientProfileList{} - err := r.client.List(r.opManagerContext, clientProfiles, &client.ListOptions{ - Namespace: r.opConfig.OperatorNamespace, - }) - if err != nil { - if _, ok := err.(*meta.NoKindMatchError); ok || kerrors.IsNotFound(err) { - logger.Debug("no CSI client profiles found") - return nil - } - return errors.Wrap(err, "failed to list CSI client profiles") - } - - for _, profile := range clientProfiles.Items { - logger.Debugf("deleting CSI client profile %q in namespace %q", profile.Name, profile.Namespace) - err := r.client.Delete(r.opManagerContext, &profile) - if kerrors.IsNotFound(err) { - return nil - } - if err != nil && !kerrors.IsNotFound(err) { - return errors.Wrapf(err, "failed to delete CSI client profile %q", profile.Name) - } - logger.Infof("successfully deleted CSI client profile %q", profile.Name) - } - return nil -} - -// deleteCephConnections deletes CephConnection resources -func (r *ReconcileCSI) deleteCephConnections() error { - cephConnections := &csiopv1.CephConnectionList{} - err := r.client.List(r.opManagerContext, cephConnections, &client.ListOptions{ - Namespace: r.opConfig.OperatorNamespace, - }) - if err != nil { - if _, ok := err.(*meta.NoKindMatchError); ok || kerrors.IsNotFound(err) { - logger.Debug("no CSI ceph connections found") - return nil - } - return errors.Wrap(err, "failed to list CSI ceph connections") - } - - for _, connection := range cephConnections.Items { - logger.Debugf("deleting CSI ceph connection %q in namespace %q", connection.Name, connection.Namespace) - err := r.client.Delete(r.opManagerContext, &connection) - if kerrors.IsNotFound(err) { - return nil - } - if err != nil && !kerrors.IsNotFound(err) { - return errors.Wrapf(err, "failed to delete CSI ceph connection %q", connection.Name) - } - logger.Infof("successfully deleted CSI ceph connection %q", connection.Name) - } - return nil -} - -func (r *ReconcileCSI) applyCephClusterNetworkConfig(ctx context.Context, objectMeta *metav1.ObjectMeta) error { - cephClusters, err := r.context.RookClientset.CephV1().CephClusters(objectMeta.Namespace).List(ctx, metav1.ListOptions{}) - if err != nil { - return errors.Wrap(err, "failed to find CephClusters") - } - for i, cephCluster := range cephClusters.Items { - if cephCluster.Spec.Network.IsMultus() { - err = k8sutil.ApplyMultus(cephCluster.GetNamespace(), &cephClusters.Items[i].Spec.Network, objectMeta) - if err != nil { - return errors.Wrapf(err, "failed to apply multus configuration to CephCluster %q", cephCluster.Name) - } - } - } - - return nil -} - -func validateCSIDriverNamePrefix(ctx context.Context, clientset kubernetes.Interface, namespace, driverNamePrefix string) error { - if EnableRBD { - rbdDriverNamePrefix, err := getCSIDriverNamePrefixFromDeployment(ctx, clientset, namespace, csiRBDProvisioner, "csi-rbdplugin") - if err != nil { - return err - } - if rbdDriverNamePrefix != "" { - if rbdDriverNamePrefix != driverNamePrefix { - return errors.Errorf("rbd driver already exists with prefix %q, cannot use prefix %q", rbdDriverNamePrefix, driverNamePrefix) - } - return nil - } - } - - if EnableCephFS { - cephFSDriverNamePrefix, err := getCSIDriverNamePrefixFromDeployment(ctx, clientset, namespace, csiCephFSProvisioner, "csi-cephfsplugin") - if err != nil { - return err - } - if cephFSDriverNamePrefix != "" { - if cephFSDriverNamePrefix != driverNamePrefix { - return errors.Errorf("cephFS driver already exists with prefix %q, cannot use prefix %q", cephFSDriverNamePrefix, driverNamePrefix) - } - return nil - - } - } - - if EnableNFS { - nfsDriverNamePrefix, err := getCSIDriverNamePrefixFromDeployment(ctx, clientset, namespace, csiNFSProvisioner, "csi-nfsplugin") - if err != nil { - return err - } - if nfsDriverNamePrefix != "" { - if nfsDriverNamePrefix != driverNamePrefix { - return errors.Errorf("nfs driver already exists with prefix %q, cannot use prefix %q", nfsDriverNamePrefix, driverNamePrefix) - } - return nil - } - } - - return nil -} - -func getCSIDriverNamePrefixFromDeployment(ctx context.Context, clientset kubernetes.Interface, namespace, deploymentName, containerName string) (string, error) { - deployment, err := clientset.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) - if kerrors.IsNotFound(err) { - return "", nil - } - if err != nil { - return "", errors.Wrapf(err, "failed to get deployment %q", deploymentName) - } - - for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name == containerName { - for _, arg := range container.Args { - if prefix, ok := getPrefixFromArg(arg); ok { - return prefix, nil - } - } - } - } - - return "", errors.Errorf("failed to get CSI driver name from deployment %q", deploymentName) -} - -func getPrefixFromArg(arg string) (string, bool) { - if strings.Contains(arg, "--drivername=") { - driverName := strings.Split(arg, "=")[1] - - for _, suffix := range []string{rbdDriverSuffix, cephFSDriverSuffix, nfsDriverSuffix} { - // Add a dot as we are adding it to the Prefix - if prefix, ok := strings.CutSuffix(driverName, "."+suffix); ok { - return prefix, true - } - } - } - return "", false -} - -func EnableCSIOperator() bool { - return enableCSIOperator -} diff --git a/pkg/operator/ceph/csi/spec_test.go b/pkg/operator/ceph/csi/spec_test.go deleted file mode 100644 index bd28f0c6b..000000000 --- a/pkg/operator/ceph/csi/spec_test.go +++ /dev/null @@ -1,117 +0,0 @@ -/* -Copyright 2022 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "context" - _ "embed" - "testing" - - "github.com/stretchr/testify/assert" - apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kfake "k8s.io/client-go/kubernetes/fake" -) - -func Test_getCSIDriverNamePrefixFromDeployment(t *testing.T) { - namespace := "test" - deployment := func(name, containerName, drivernameSuffix string) *apps.Deployment { - return &apps.Deployment{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, - Spec: apps.DeploymentSpec{ - Template: v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: containerName, - Args: []string{ - "--drivername=test-prefix." + drivernameSuffix, - }, - }, - }, - }, - }, - }, - } - } - clientset := kfake.NewClientset() - - ctx := context.TODO() - csidrivers := []struct { - testCaseName string - deploymentName string - containerName string - driverNameSuffix string - expectedPrefix string - }{ - { - "get csi driver name prefix for rbd when deployment exists", - csiRBDProvisioner, - csiRBDContainerName, - rbdDriverSuffix, - "test-prefix", - }, - { - "get csi driver name prefix for rbd when deployment does not exist", - "", - "csi-rbdplugin", - "", - "", - }, - { - "get csi driver name prefix for cephfs when deployment exists", - csiCephFSProvisioner, - csiCephFSContainerName, - cephFSDriverSuffix, - "test-prefix", - }, - { - "get csi driver name prefix for cephfs when deployment does not exist", - "", - "csi-cephfsplugin", - "", - "", - }, - { - "get csi driver name prefix for nfs when deployment exists", - csiNFSProvisioner, - csiNFSContainerName, - nfsDriverSuffix, - "test-prefix", - }, - { - "get csi driver name prefix for nfs when deployment does not exist", - "", - "csi-nfsplugin", - "", - "", - }, - } - - for _, c := range csidrivers { - t.Run(c.testCaseName, func(t *testing.T) { - if c.deploymentName != "" { - _, err := clientset.AppsV1().Deployments(namespace).Create(ctx, deployment(c.deploymentName, c.containerName, c.driverNameSuffix), metav1.CreateOptions{}) - assert.NoError(t, err) - } - prefix, err := getCSIDriverNamePrefixFromDeployment(ctx, clientset, namespace, c.deploymentName, c.containerName) - assert.NoError(t, err) - assert.Equal(t, c.expectedPrefix, prefix) - }) - } -} diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml deleted file mode 100644 index ebcb060f7..000000000 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ /dev/null @@ -1,341 +0,0 @@ -kind: Deployment -apiVersion: apps/v1 -metadata: - name: csi-cephfsplugin-provisioner - namespace: {{ .Namespace }} -spec: - replicas: {{ .ProvisionerReplicas }} - selector: - matchLabels: - app: csi-cephfsplugin-provisioner - template: - metadata: - labels: - app: csi-cephfsplugin-provisioner - contains: csi-cephfsplugin-metrics - {{ range $key, $value := .CSICephFSPodLabels }} - {{ $key }}: "{{ $value }}" - {{ end }} - spec: - securityContext: {} - serviceAccountName: rook-csi-cephfs-provisioner-sa - {{ if .ProvisionerPriorityClassName }} - priorityClassName: {{ .ProvisionerPriorityClassName }} - {{ end }} - containers: - {{ if .CephFSAttachRequired }} - - name: csi-attacher - image: {{ .AttacherImage }} - args: - - "--v={{ .SidecarLogLevel }}" - - "--csi-address=$(ADDRESS)" - - "--leader-election=true" - - "--timeout={{ .GRPCTimeout }}" - - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" - - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" - - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - {{ if .KubeApiBurst }} - - "--kube-api-burst={{ .KubeApiBurst }}" - {{ end }} - {{ if .KubeApiQPS }} - - "--kube-api-qps={{ .KubeApiQPS }}" - {{ end }} - env: - - name: ADDRESS - value: /csi/csi-provisioner.sock - imagePullPolicy: {{ .ImagePullPolicy }} - volumeMounts: - - name: socket-dir - mountPath: /csi - {{ end }} - {{ if .EnableCephFSSnapshotter }} - - name: csi-snapshotter - image: {{ .SnapshotterImage }} - args: - - "--csi-address=$(ADDRESS)" - - "--v={{ .SidecarLogLevel }}" - - "--timeout={{ .GRPCTimeout }}" - - "--leader-election=true" - - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" - - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" - - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - - "--extra-create-metadata=true" - {{ if .VolumeGroupSnapshotSupported }} - - "{{ .VolumeGroupSnapshotCLIFlag }}" - {{ end }} - {{ if .KubeApiBurst }} - - "--kube-api-burst={{ .KubeApiBurst }}" - {{ end }} - {{ if .KubeApiQPS }} - - "--kube-api-qps={{ .KubeApiQPS }}" - {{ end }} - env: - - name: ADDRESS - value: unix:///csi/csi-provisioner.sock - imagePullPolicy: {{ .ImagePullPolicy }} - volumeMounts: - - name: socket-dir - mountPath: /csi - {{ end }} - - name: csi-resizer - image: {{ .ResizerImage }} - args: - - "--csi-address=$(ADDRESS)" - - "--v={{ .SidecarLogLevel }}" - - "--timeout={{ .GRPCTimeout }}" - - "--leader-election=true" - - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" - - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" - - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - - "--handle-volume-inuse-error=false" - - "--feature-gates=RecoverVolumeExpansionFailure=true" - {{ if .KubeApiBurst }} - - "--kube-api-burst={{ .KubeApiBurst }}" - {{ end }} - {{ if .KubeApiQPS }} - - "--kube-api-qps={{ .KubeApiQPS }}" - {{ end }} - env: - - name: ADDRESS - value: unix:///csi/csi-provisioner.sock - imagePullPolicy: {{ .ImagePullPolicy }} - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: csi-provisioner - image: {{ .ProvisionerImage }} - args: - - "--csi-address=$(ADDRESS)" - - "--v={{ .SidecarLogLevel }}" - - "--timeout={{ .GRPCTimeout }}" - - "--retry-interval-start=500ms" - - "--leader-election=true" - - "--leader-election-namespace={{ .Namespace }}" - - "--feature-gates=HonorPVReclaimPolicy=true" - - "--prevent-volume-mode-conversion=true" - - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" - - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" - - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - - "--extra-create-metadata=true" - - "--feature-gates=CrossNamespaceVolumeDataSource={{ .EnableCrossNamespaceVolumeDataSource }}" - {{ if .KubeApiBurst }} - - "--kube-api-burst={{ .KubeApiBurst }}" - {{ end }} - {{ if .KubeApiQPS }} - - "--kube-api-qps={{ .KubeApiQPS }}" - {{ end }} - env: - - name: ADDRESS - value: unix:///csi/csi-provisioner.sock - imagePullPolicy: {{ .ImagePullPolicy }} - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: csi-cephfsplugin - image: {{ .CSIPluginImage }} - args: - - "--nodeid=$(NODE_ID)" - - "--type=cephfs" - - "--endpoint=$(CSI_ENDPOINT)" - - "--v={{ .LogLevel }}" - - "--controllerserver=true" - - "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com" - - "--pidlimit=-1" - - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}" - {{ if .CSILogRotation }} - - "--logtostderr=false" - - "--alsologtostderr=true" - - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-cephfsplugin.log" - {{ end }} - {{ if .CSIEnableMetadata }} - - "--setmetadata={{ .CSIEnableMetadata }}" - {{ end }} - {{ if .CSIClusterName }} - - "--clustername={{ .CSIClusterName }}" - {{ end }} - {{ if .EnableCSIAddonsSideCar }} - - "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)" - {{ end }} - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CSI_ENDPOINT - value: unix:///csi/csi-provisioner.sock - {{ if .EnableCSIAddonsSideCar }} - - name: CSIADDONS_ENDPOINT - value: unix:///csi/csi-addons.sock - {{ end }} - imagePullPolicy: {{ .ImagePullPolicy }} - {{ if and .Privileged .CSILogRotation }} - securityContext: - privileged: true - {{ end }} - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: host-sys - mountPath: /sys - - name: lib-modules - mountPath: /lib/modules - readOnly: true - - name: host-dev - mountPath: /dev - - name: ceph-csi-config - mountPath: /etc/ceph-csi-config/ - - name: keys-tmp-dir - mountPath: /tmp/csi/keys - {{ if .CSILogRotation }} - - mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }} - name: csi-log - {{ end }} - {{ if .MountCustomCephConf }} - - name: ceph-config - mountPath: /etc/ceph/ceph.conf - subPath: ceph.conf - {{ end }} - {{ if .EnableCSIEncryption }} - - name: rook-ceph-csi-kms-config - mountPath: /etc/ceph-csi-encryption-kms-config/ - {{ end }} - {{ if .EnableCSIAddonsSideCar }} - - name: csi-addons - image: {{ .CSIAddonsImage }} - args : - - "--node-id=$(NODE_ID)" - - "--v={{ .LogLevel }}" - - "--csi-addons-address=$(CSIADDONS_ENDPOINT)" - - "--controller-port={{ .CSIAddonsCephFSProvisionerPort }}" - - "--pod=$(POD_NAME)" - - "--namespace=$(POD_NAMESPACE)" - - "--pod-uid=$(POD_UID)" - - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/" - - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" - - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" - - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - {{ if .CSILogRotation }} - - "--logtostderr=false" - - "--alsologtostderr=true" - - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-addons.log" - {{ end }} - ports: - - containerPort: {{ .CSIAddonsCephFSProvisionerPort }} - env: - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CSIADDONS_ENDPOINT - value: unix:///csi/csi-addons.sock - imagePullPolicy: {{ .ImagePullPolicy }} - {{ if and .Privileged .CSILogRotation }} - securityContext: - privileged: true - capabilities: - add: [] - drop: ["ALL"] - {{ end }} - volumeMounts: - - name: socket-dir - mountPath: /csi - {{ if .CSILogRotation }} - - mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }} - name: csi-log - {{ end }} - {{ end }} - {{ if .EnableLiveness }} - - name: liveness-prometheus - image: {{ .CSIPluginImage }} - args: - - "--type=liveness" - - "--endpoint=$(CSI_ENDPOINT)" - - "--metricsport={{ .CephFSLivenessMetricsPort }}" - - "--metricspath=/metrics" - - "--polltime=60s" - - "--timeout=3s" - env: - - name: CSI_ENDPOINT - value: unix:///csi/csi-provisioner.sock - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - volumeMounts: - - name: socket-dir - mountPath: /csi - imagePullPolicy: {{ .ImagePullPolicy }} - {{ end }} - volumes: - - name: socket-dir - emptyDir: { - medium: "Memory" - } - - name: host-sys - hostPath: - path: /sys - - name: lib-modules - hostPath: - path: /lib/modules - - name: host-dev - hostPath: - path: /dev - - name: ceph-csi-config - configMap: - name: rook-ceph-csi-config - items: - - key: csi-cluster-config-json - path: config.json - - name: keys-tmp-dir - emptyDir: { - medium: "Memory" - } - {{ if .CSILogRotation }} - - name: csi-log - hostPath: - path: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }} - type: DirectoryOrCreate - - name: csi-logs-logrotate - emptyDir: - type: DirectoryOrCreate - {{ end }} - {{ if .MountCustomCephConf }} - - name: ceph-config - configMap: - name: csi-ceph-conf-override - items: - - key: ceph.conf - path: ceph.conf - {{ end }} - {{ if .EnableCSIEncryption }} - - name: rook-ceph-csi-kms-config - configMap: - name: rook-ceph-csi-kms-config - items: - - key: config.json - path: config.json - {{ end }} diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml deleted file mode 100644 index 890b68614..000000000 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -# This is a service to expose the liveness and grpc metrics -apiVersion: v1 -kind: Service -metadata: - name: csi-cephfsplugin-metrics - labels: - app: csi-metrics -spec: - ports: - - name: csi-http-metrics - port: 8080 - protocol: TCP - targetPort: {{ .CephFSLivenessMetricsPort }} - selector: - contains: csi-cephfsplugin-metrics diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml deleted file mode 100644 index 6d92de1ab..000000000 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ /dev/null @@ -1,242 +0,0 @@ -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: csi-cephfsplugin - namespace: {{ .Namespace }} -spec: - selector: - matchLabels: - app: csi-cephfsplugin - updateStrategy: - type: {{ .CephFSPluginUpdateStrategy }} - {{ if eq .CephFSPluginUpdateStrategy "RollingUpdate" }} - rollingUpdate: - maxUnavailable: {{ .CephFSPluginUpdateStrategyMaxUnavailable }} - {{ end }} - template: - metadata: - labels: - app: csi-cephfsplugin - contains: csi-cephfsplugin-metrics - {{ range $key, $value := .CSICephFSPodLabels }} - {{ $key }}: "{{ $value }}" - {{ end }} - spec: - securityContext: {} - serviceAccountName: rook-csi-cephfs-plugin-sa - hostNetwork: {{ .EnableCSIHostNetwork }} - {{ if .PluginPriorityClassName }} - priorityClassName: {{ .PluginPriorityClassName }} - {{ end }} - # to use e.g. Rook orchestrated cluster, and mons' FQDN is - # resolved through k8s service, set dns policy to cluster first - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: driver-registrar - # This is necessary only for systems with SELinux, where - # non-privileged sidecar containers cannot access unix domain socket - # created by privileged CSI driver container. - securityContext: - privileged: true - capabilities: - add: [] - drop: ["ALL"] - image: {{ .RegistrarImage }} - imagePullPolicy: {{ .ImagePullPolicy }} - args: - - "--v={{ .LogLevel }}" - - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path={{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com/csi.sock" - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - - name: csi-cephfsplugin - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - drop: ["ALL"] - allowPrivilegeEscalation: true - image: {{ .CSIPluginImage }} - args: - - "--nodeid=$(NODE_ID)" - - "--type=cephfs" - - "--endpoint=$(CSI_ENDPOINT)" - - "--v={{ .LogLevel }}" - - "--nodeserver=true" - - "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com" - - "--pidlimit=-1" - - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}" - {{ if .CSILogRotation }} - - "--logtostderr=false" - - "--alsologtostderr=true" - - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-cephfsplugin.log" - {{ end }} - {{ if .CephFSKernelMountOptions }} - - "--kernelmountoptions={{ .CephFSKernelMountOptions }}" - {{ end }} - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - imagePullPolicy: {{ .ImagePullPolicy }} - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: ceph-csi-mountinfo - mountPath: /csi/mountinfo - - name: csi-plugins-dir - mountPath: "{{ .KubeletDirPath }}/plugins" - mountPropagation: "Bidirectional" - - name: pods-mount-dir - mountPath: "{{ .KubeletDirPath }}/pods" - mountPropagation: "Bidirectional" - - name: host-sys - mountPath: /sys - - name: lib-modules - mountPath: /lib/modules - readOnly: true - - name: host-dev - mountPath: /dev - - name: ceph-csi-config - mountPath: /etc/ceph-csi-config/ - - name: keys-tmp-dir - mountPath: /tmp/csi/keys - - name: host-run-mount - mountPath: /run/mount - {{ if .CSILogRotation }} - - mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }} - name: csi-log - {{ end }} - {{ if .EnablePluginSelinuxHostMount }} - - name: etc-selinux - mountPath: /etc/selinux - readOnly: true - {{ end }} - {{ if .MountCustomCephConf }} - - name: ceph-config - mountPath: /etc/ceph/ceph.conf - subPath: ceph.conf - {{ end }} - {{ if .EnableCSIEncryption }} - - name: rook-ceph-csi-kms-config - mountPath: /etc/ceph-csi-encryption-kms-config/ - {{ end }} - {{ if .EnableLiveness }} - - name: liveness-prometheus - securityContext: - privileged: true - capabilities: - add: [] - drop: ["ALL"] - image: {{ .CSIPluginImage }} - args: - - "--type=liveness" - - "--endpoint=$(CSI_ENDPOINT)" - - "--metricsport={{ .CephFSLivenessMetricsPort }}" - - "--metricspath=/metrics" - - "--polltime=60s" - - "--timeout=3s" - env: - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - volumeMounts: - - name: plugin-dir - mountPath: /csi - imagePullPolicy: {{ .ImagePullPolicy }} - {{ end }} - volumes: - - name: ceph-csi-mountinfo - hostPath: - path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com/mountinfo" - type: DirectoryOrCreate - - name: plugin-dir - hostPath: - path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com/" - type: DirectoryOrCreate - - name: csi-plugins-dir - hostPath: - path: "{{ .KubeletDirPath }}/plugins" - type: Directory - - name: registration-dir - hostPath: - path: "{{ .KubeletDirPath }}/plugins_registry/" - type: Directory - - name: pods-mount-dir - hostPath: - path: "{{ .KubeletDirPath }}/pods" - type: Directory - - name: host-sys - hostPath: - path: /sys - - name: lib-modules - hostPath: - path: /lib/modules - - name: host-dev - hostPath: - path: /dev - - name: ceph-csi-config - configMap: - name: rook-ceph-csi-config - items: - - key: csi-cluster-config-json - path: config.json - - name: keys-tmp-dir - emptyDir: { - medium: "Memory" - } - - name: host-run-mount - hostPath: - path: /run/mount - {{ if .CSILogRotation }} - - name: csi-log - hostPath: - path: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }} - type: DirectoryOrCreate - - name: csi-logs-logrotate - emptyDir: - type: DirectoryOrCreate - {{ end }} - {{ if .EnablePluginSelinuxHostMount }} - - name: etc-selinux - hostPath: - path: /etc/selinux - {{ end }} - {{ if .MountCustomCephConf }} - - name: ceph-config - configMap: - name: csi-ceph-conf-override - items: - - key: ceph.conf - path: ceph.conf - {{ end }} - {{ if .EnableCSIEncryption }} - - name: rook-ceph-csi-kms-config - configMap: - name: rook-ceph-csi-kms-config - items: - - key: config.json - path: config.json - {{ end }} diff --git a/pkg/operator/ceph/csi/template/csi-logrotate-sidecar.yaml b/pkg/operator/ceph/csi/template/csi-logrotate-sidecar.yaml deleted file mode 100644 index ea74b11da..000000000 --- a/pkg/operator/ceph/csi/template/csi-logrotate-sidecar.yaml +++ /dev/null @@ -1,41 +0,0 @@ -args: - - | - echo "Starting the csi-logrotate-sidecar" - mkdir -p {{ .CsiLogRootPath }}/logrotate-config/{{ .CsiComponentName }} - echo '{{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/*.log { - {{ .CSILogRotationPeriod }} - missingok - rotate 7 - compress - copytruncate - notifempty - }' > {{ .CsiLogRootPath }}/logrotate-config/{{ .CsiComponentName }}/csi - echo "File creation container completed" - - LOG_ROTATE_CEPH_CSI_FILE={{ .CsiLogRootPath }}/logrotate-config/{{ .CsiComponentName }}/csi - LOG_MAX_SIZE={{ .CSILogRotationMaxSize }} - if [ "$LOG_MAX_SIZE" != "0" ]; then - sed --in-place "4i \ \ \ \ maxsize $LOG_MAX_SIZE" "$LOG_ROTATE_CEPH_CSI_FILE" - fi - - while true; do - logrotate --verbose "$LOG_ROTATE_CEPH_CSI_FILE" - sleep 15m - done -command: - - /bin/sh - - -c -image: {{ .CSIPluginImage }} -imagePullPolicy: {{ .ImagePullPolicy }} -name: log-collector -{{ if .Privileged }} -securityContext: - privileged: true - capabilities: - drop: ["ALL"] -{{ end }} -volumeMounts: - - mountPath: {{ .CsiLogRootPath }}/logrotate-config/{{ .CsiComponentName }} - name: csi-logs-logrotate - - mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }} - name: csi-log diff --git a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml deleted file mode 100644 index a62473bb1..000000000 --- a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml +++ /dev/null @@ -1,247 +0,0 @@ -kind: Deployment -apiVersion: apps/v1 -metadata: - name: csi-nfsplugin-provisioner - namespace: {{ .Namespace }} -spec: - replicas: {{ .ProvisionerReplicas }} - selector: - matchLabels: - app: csi-nfsplugin-provisioner - template: - metadata: - labels: - app: csi-nfsplugin-provisioner - {{ range $key, $value := .CSINFSPodLabels }} - {{ $key }}: "{{ $value }}" - {{ end }} - spec: - securityContext: {} - serviceAccountName: rook-csi-nfs-provisioner-sa - {{ if .ProvisionerPriorityClassName }} - priorityClassName: {{ .ProvisionerPriorityClassName }} - {{ end }} - containers: - {{ if .NFSAttachRequired }} - - name: csi-attacher - image: {{ .AttacherImage }} - args: - - "--v={{ .SidecarLogLevel }}" - - "--csi-address=$(ADDRESS)" - - "--leader-election=true" - - "--timeout={{ .GRPCTimeout }}" - - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" - - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" - - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - {{ if .KubeApiBurst }} - - "--kube-api-burst={{ .KubeApiBurst }}" - {{ end }} - {{ if .KubeApiQPS }} - - "--kube-api-qps={{ .KubeApiQPS }}" - {{ end }} - env: - - name: ADDRESS - value: /csi/csi-provisioner.sock - imagePullPolicy: {{ .ImagePullPolicy }} - securityContext: - capabilities: - add: [] - drop: ["ALL"] - volumeMounts: - - name: socket-dir - mountPath: /csi - {{ end }} - {{ if .EnableNFSSnapshotter }} - - name: csi-snapshotter - image: {{ .SnapshotterImage }} - args: - - "--csi-address=$(ADDRESS)" - - "--v={{ .SidecarLogLevel }}" - - "--timeout={{ .GRPCTimeout }}" - - "--leader-election=true" - - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" - - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" - - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - - "--extra-create-metadata=true" - {{ if .KubeApiBurst }} - - "--kube-api-burst={{ .KubeApiBurst }}" - {{ end }} - {{ if .KubeApiQPS }} - - "--kube-api-qps={{ .KubeApiQPS }}" - {{ end }} - env: - - name: ADDRESS - value: unix:///csi/csi-provisioner.sock - imagePullPolicy: {{ .ImagePullPolicy }} - securityContext: - capabilities: - add: [] - drop: ["ALL"] - volumeMounts: - - name: socket-dir - mountPath: /csi - {{ end }} - - name: csi-resizer - image: {{ .ResizerImage }} - args: - - "--csi-address=$(ADDRESS)" - - "--v={{ .SidecarLogLevel }}" - - "--timeout={{ .GRPCTimeout }}" - - "--leader-election=true" - - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" - - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" - - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - - "--handle-volume-inuse-error=false" - - "--feature-gates=RecoverVolumeExpansionFailure=true" - {{ if .KubeApiBurst }} - - "--kube-api-burst={{ .KubeApiBurst }}" - {{ end }} - {{ if .KubeApiQPS }} - - "--kube-api-qps={{ .KubeApiQPS }}" - {{ end }} - env: - - name: ADDRESS - value: unix:///csi/csi-provisioner.sock - imagePullPolicy: {{ .ImagePullPolicy }} - securityContext: - capabilities: - add: [] - drop: ["ALL"] - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: csi-provisioner - image: {{ .ProvisionerImage }} - args: - - "--csi-address=$(ADDRESS)" - - "--v={{ .SidecarLogLevel }}" - - "--timeout={{ .GRPCTimeout }}" - - "--retry-interval-start=500ms" - - "--leader-election=true" - - "--leader-election-namespace={{ .Namespace }}" - - "--feature-gates=HonorPVReclaimPolicy=true" - - "--prevent-volume-mode-conversion=true" - - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" - - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" - - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - {{ if .KubeApiBurst }} - - "--kube-api-burst={{ .KubeApiBurst }}" - {{ end }} - {{ if .KubeApiQPS }} - - "--kube-api-qps={{ .KubeApiQPS }}" - {{ end }} - env: - - name: ADDRESS - value: unix:///csi/csi-provisioner.sock - imagePullPolicy: {{ .ImagePullPolicy }} - securityContext: - capabilities: - add: [] - drop: ["ALL"] - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: csi-nfsplugin - image: {{ .CSIPluginImage }} - args: - - "--nodeid=$(NODE_ID)" - - "--type=nfs" - - "--endpoint=$(CSI_ENDPOINT)" - - "--v={{ .LogLevel }}" - - "--controllerserver=true" - - "--drivername={{ .DriverNamePrefix }}nfs.csi.ceph.com" - - "--pidlimit=-1" - {{ if .CSILogRotation }} - - "--logtostderr=false" - - "--alsologtostderr=true" - - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-nfsplugin.log" - {{ end }} - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CSI_ENDPOINT - value: unix:///csi/csi-provisioner.sock - imagePullPolicy: {{ .ImagePullPolicy }} - {{ if and .Privileged .CSILogRotation }} - securityContext: - privileged: true - capabilities: - drop: ["ALL"] - {{ end }} - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: host-sys - mountPath: /sys - - name: lib-modules - mountPath: /lib/modules - readOnly: true - - name: host-dev - mountPath: /dev - - name: ceph-csi-config - mountPath: /etc/ceph-csi-config/ - - name: keys-tmp-dir - mountPath: /tmp/csi/keys - {{ if .CSILogRotation }} - - mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }} - name: csi-log - {{ end }} - {{ if .MountCustomCephConf }} - - name: ceph-config - mountPath: /etc/ceph/ceph.conf - subPath: ceph.conf - {{ end }} - volumes: - - name: socket-dir - emptyDir: { - medium: "Memory" - } - - name: host-sys - hostPath: - path: /sys - - name: lib-modules - hostPath: - path: /lib/modules - - name: host-dev - hostPath: - path: /dev - - name: ceph-csi-config - configMap: - name: rook-ceph-csi-config - items: - - key: csi-cluster-config-json - path: config.json - - name: keys-tmp-dir - emptyDir: { - medium: "Memory" - } - {{ if .CSILogRotation }} - - name: csi-log - hostPath: - path: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }} - type: DirectoryOrCreate - - name: csi-logs-logrotate - emptyDir: - type: DirectoryOrCreate - {{ end }} - {{ if .MountCustomCephConf }} - - name: ceph-config - configMap: - name: csi-ceph-conf-override - items: - - key: ceph.conf - path: ceph.conf - {{ end }} diff --git a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin.yaml b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin.yaml deleted file mode 100644 index d6b7e8d18..000000000 --- a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin.yaml +++ /dev/null @@ -1,182 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: csi-nfsplugin - namespace: {{ .Namespace }} -spec: - selector: - matchLabels: - app: csi-nfsplugin - updateStrategy: - type: {{ .NFSPluginUpdateStrategy }} - template: - metadata: - labels: - app: csi-nfsplugin - {{ range $key, $value := .CSINFSPodLabels }} - {{ $key }}: "{{ $value }}" - {{ end }} - spec: - securityContext: {} - serviceAccountName: rook-csi-nfs-plugin-sa - hostNetwork: {{ .EnableCSIHostNetwork }} - {{ if .PluginPriorityClassName }} - priorityClassName: {{ .PluginPriorityClassName }} - {{ end }} - # to use e.g. Rook orchestrated cluster, and mons' FQDN is - # resolved through k8s service, set dns policy to cluster first - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: driver-registrar - # This is necessary only for systems with SELinux, where - # non-privileged sidecar containers cannot access unix domain socket - # created by privileged CSI driver container. - securityContext: - capabilities: - add: [] - drop: ["ALL"] - privileged: true - image: {{ .RegistrarImage }} - imagePullPolicy: {{ .ImagePullPolicy }} - args: - - "--v={{ .LogLevel }}" - - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path={{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}nfs.csi.ceph.com/csi.sock" - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - - name: csi-nfsplugin - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - SYS_ADMIN - drop: ["ALL"] - privileged: true - args: - - "--nodeid=$(NODE_ID)" - - "--type=nfs" - - "--endpoint=$(CSI_ENDPOINT)" - - "--v={{ .LogLevel }}" - - "--nodeserver=true" - - "--drivername={{ .DriverNamePrefix }}nfs.csi.ceph.com" - - "--pidlimit=-1" - {{ if .CSILogRotation }} - - "--logtostderr=false" - - "--alsologtostderr=true" - - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-nfsplugin.log" - {{ end }} - env: - - name: NODE_ID - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - image: {{ .CSIPluginImage }} - imagePullPolicy: {{ .ImagePullPolicy }} - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: csi-plugins-dir - mountPath: "{{ .KubeletDirPath }}/plugins" - mountPropagation: "Bidirectional" - - name: pods-mount-dir - mountPath: "{{ .KubeletDirPath }}/pods" - mountPropagation: "Bidirectional" - - name: host-sys - mountPath: /sys - - name: lib-modules - mountPath: /lib/modules - readOnly: true - - name: host-dev - mountPath: /dev - - name: ceph-csi-config - mountPath: /etc/ceph-csi-config/ - - name: keys-tmp-dir - mountPath: /tmp/csi/keys - - name: host-run-mount - mountPath: /run/mount - {{ if .CSILogRotation }} - - mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }} - name: csi-log - {{ end }} - {{ if .EnablePluginSelinuxHostMount }} - - name: etc-selinux - mountPath: /etc/selinux - readOnly: true - {{ end }} - {{ if .MountCustomCephConf }} - - name: ceph-config - mountPath: /etc/ceph/ceph.conf - subPath: ceph.conf - {{ end }} - volumes: - - name: plugin-dir - hostPath: - path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}nfs.csi.ceph.com/" - type: DirectoryOrCreate - - name: csi-plugins-dir - hostPath: - path: "{{ .KubeletDirPath }}/plugins" - type: Directory - - name: registration-dir - hostPath: - path: "{{ .KubeletDirPath }}/plugins_registry/" - type: Directory - - name: pods-mount-dir - hostPath: - path: "{{ .KubeletDirPath }}/pods" - type: Directory - - name: host-sys - hostPath: - path: /sys - - name: lib-modules - hostPath: - path: /lib/modules - - name: host-dev - hostPath: - path: /dev - - name: ceph-csi-config - configMap: - name: rook-ceph-csi-config - items: - - key: csi-cluster-config-json - path: config.json - - name: keys-tmp-dir - emptyDir: { - medium: "Memory" - } - - name: host-run-mount - hostPath: - path: /run/mount - {{ if .CSILogRotation }} - - name: csi-log - hostPath: - path: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }} - type: DirectoryOrCreate - - name: csi-logs-logrotate - emptyDir: - type: DirectoryOrCreate - {{ end }} - {{ if .EnablePluginSelinuxHostMount }} - - name: etc-selinux - hostPath: - path: /etc/selinux - {{ end }} - {{ if .MountCustomCephConf }} - - name: ceph-config - configMap: - name: csi-ceph-conf-override - items: - - key: ceph.conf - path: ceph.conf - {{ end }} diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml deleted file mode 100644 index ded285066..000000000 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ /dev/null @@ -1,391 +0,0 @@ -kind: Deployment -apiVersion: apps/v1 -metadata: - name: csi-rbdplugin-provisioner - namespace: {{ .Namespace }} -spec: - replicas: {{ .ProvisionerReplicas }} - selector: - matchLabels: - app: csi-rbdplugin-provisioner - template: - metadata: - labels: - app: csi-rbdplugin-provisioner - contains: csi-rbdplugin-metrics - {{ range $key, $value := .CSIRBDPodLabels }} - {{ $key }}: "{{ $value }}" - {{ end }} - spec: - securityContext: {} - serviceAccountName: rook-csi-rbd-provisioner-sa - {{ if .ProvisionerPriorityClassName }} - priorityClassName: {{ .ProvisionerPriorityClassName }} - {{ end }} - containers: - - name: csi-provisioner - image: {{ .ProvisionerImage }} - args: - - "--csi-address=$(ADDRESS)" - - "--v={{ .SidecarLogLevel }}" - - "--timeout={{ .GRPCTimeout }}" - - "--retry-interval-start=500ms" - - "--leader-election=true" - - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" - - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" - - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - - "--default-fstype=ext4" - - "--extra-create-metadata=true" - - "--prevent-volume-mode-conversion=true" - - "--feature-gates=HonorPVReclaimPolicy=true" - - "--feature-gates=CrossNamespaceVolumeDataSource={{ .EnableCrossNamespaceVolumeDataSource }}" - - "--feature-gates=Topology={{ .EnableCSITopology }}" - {{ if .KubeApiBurst }} - - "--kube-api-burst={{ .KubeApiBurst }}" - {{ end }} - {{ if .KubeApiQPS }} - - "--kube-api-qps={{ .KubeApiQPS }}" - {{ end }} - env: - - name: ADDRESS - value: unix:///csi/csi-provisioner.sock - imagePullPolicy: {{ .ImagePullPolicy }} - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: csi-resizer - image: {{ .ResizerImage }} - args: - - "--csi-address=$(ADDRESS)" - - "--v={{ .SidecarLogLevel }}" - - "--timeout={{ .GRPCTimeout }}" - - "--leader-election=true" - - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" - - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" - - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - - "--handle-volume-inuse-error=false" - - "--feature-gates=RecoverVolumeExpansionFailure=true" - {{ if .KubeApiBurst }} - - "--kube-api-burst={{ .KubeApiBurst }}" - {{ end }} - {{ if .KubeApiQPS }} - - "--kube-api-qps={{ .KubeApiQPS }}" - {{ end }} - env: - - name: ADDRESS - value: unix:///csi/csi-provisioner.sock - imagePullPolicy: {{ .ImagePullPolicy }} - volumeMounts: - - name: socket-dir - mountPath: /csi - {{ if .RBDAttachRequired }} - - name: csi-attacher - image: {{ .AttacherImage }} - args: - - "--v={{ .SidecarLogLevel }}" - - "--timeout={{ .GRPCTimeout }}" - - "--csi-address=$(ADDRESS)" - - "--leader-election=true" - - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" - - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" - - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - - "--default-fstype=ext4" - {{ if .KubeApiBurst }} - - "--kube-api-burst={{ .KubeApiBurst }}" - {{ end }} - {{ if .KubeApiQPS }} - - "--kube-api-qps={{ .KubeApiQPS }}" - {{ end }} - env: - - name: ADDRESS - value: /csi/csi-provisioner.sock - imagePullPolicy: {{ .ImagePullPolicy }} - volumeMounts: - - name: socket-dir - mountPath: /csi - {{ end }} - {{ if .EnableRBDSnapshotter }} - - name: csi-snapshotter - image: {{ .SnapshotterImage }} - args: - - "--csi-address=$(ADDRESS)" - - "--v={{ .SidecarLogLevel }}" - - "--timeout={{ .GRPCTimeout }}" - - "--leader-election=true" - - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" - - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" - - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - - "--extra-create-metadata=true" - {{ if .VolumeGroupSnapshotSupported }} - - "{{ .VolumeGroupSnapshotCLIFlag }}" - {{ end }} - {{ if .KubeApiBurst }} - - "--kube-api-burst={{ .KubeApiBurst }}" - {{ end }} - {{ if .KubeApiQPS }} - - "--kube-api-qps={{ .KubeApiQPS }}" - {{ end }} - env: - - name: ADDRESS - value: unix:///csi/csi-provisioner.sock - imagePullPolicy: {{ .ImagePullPolicy }} - volumeMounts: - - name: socket-dir - mountPath: /csi - {{ end }} - {{ if .EnableOMAPGenerator }} - - name: csi-omap-generator - image: {{ .CSIPluginImage }} - args: - - "--type=controller" - - "--drivernamespace=$(DRIVER_NAMESPACE)" - - "--v={{ .LogLevel }}" - - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com" - {{ if .CSIEnableMetadata }} - - "--setmetadata={{ .CSIEnableMetadata }}" - {{ end }} - {{ if .CSIClusterName }} - - "--clustername={{ .CSIClusterName }}" - {{ end }} - env: - - name: DRIVER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - imagePullPolicy: {{ .ImagePullPolicy }} - volumeMounts: - - name: ceph-csi-configs - mountPath: /etc/ceph-csi-config/ - - name: keys-tmp-dir - mountPath: /tmp/csi/keys - {{ end }} - {{ if .EnableCSIAddonsSideCar }} - - name: csi-addons - image: {{ .CSIAddonsImage }} - args: - - "--node-id=$(NODE_ID)" - - "--v={{ .LogLevel }}" - - "--csi-addons-address=$(CSIADDONS_ENDPOINT)" - - "--controller-port={{ .CSIAddonsRBDProvisionerPort }}" - - "--pod=$(POD_NAME)" - - "--namespace=$(POD_NAMESPACE)" - - "--pod-uid=$(POD_UID)" - - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/" - - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" - - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" - - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - {{ if .CSILogRotation }} - - "--logtostderr=false" - - "--alsologtostderr=true" - - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-addons.log" - {{ end }} - ports: - - containerPort: {{ .CSIAddonsRBDProvisionerPort }} - env: - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CSIADDONS_ENDPOINT - value: unix:///csi/csi-addons.sock - imagePullPolicy: {{ .ImagePullPolicy }} - {{ if and .Privileged .CSILogRotation }} - securityContext: - privileged: true - capabilities: - add: [] - drop: ["ALL"] - {{ end }} - volumeMounts: - - name: socket-dir - mountPath: /csi - {{ if .CSILogRotation }} - - mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }} - name: csi-log - {{ end }} - {{ end }} - - name: csi-rbdplugin - image: {{ .CSIPluginImage }} - args: - - "--nodeid=$(NODE_ID)" - - "--endpoint=$(CSI_ENDPOINT)" - - "--v={{ .LogLevel }}" - - "--type=rbd" - - "--controllerserver=true" - - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com" - - "--pidlimit=-1" - {{ if .CSILogRotation }} - - "--logtostderr=false" - - "--alsologtostderr=true" - - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-rbdplugin.log" - {{ end }} - {{ if .EnableCSIAddonsSideCar }} - - "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)" - {{ end }} - {{ if .CSIEnableMetadata }} - - "--setmetadata={{ .CSIEnableMetadata }}" - {{ end }} - {{ if .CSIClusterName }} - - "--clustername={{ .CSIClusterName }}" - {{ end }} - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CSI_ENDPOINT - value: unix:///csi/csi-provisioner.sock - {{ if .EnableCSIAddonsSideCar }} - - name: CSIADDONS_ENDPOINT - value: unix:///csi/csi-addons.sock - {{ end }} - imagePullPolicy: {{ .ImagePullPolicy }} - {{ if and .Privileged .CSILogRotation }} - securityContext: - privileged: true - capabilities: - add: [] - drop: ["ALL"] - {{ end }} - volumeMounts: - - name: socket-dir - mountPath: /csi - - mountPath: /dev - name: host-dev - {{ if .CSILogRotation }} - - mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }} - name: csi-log - {{ end }} - - mountPath: /sys - name: host-sys - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - name: ceph-csi-configs - mountPath: /etc/ceph-csi-config/ - - name: keys-tmp-dir - mountPath: /tmp/csi/keys - {{ if .MountCustomCephConf }} - - name: ceph-config - mountPath: /etc/ceph/ceph.conf - subPath: ceph.conf - {{ end }} - - name: oidc-token - mountPath: /run/secrets/tokens - readOnly: true - {{ if .EnableCSIEncryption }} - - name: rook-ceph-csi-kms-config - mountPath: /etc/ceph-csi-encryption-kms-config/ - {{ end }} - {{ if .EnableLiveness }} - - name: liveness-prometheus - image: {{ .CSIPluginImage }} - args: - - "--type=liveness" - - "--endpoint=$(CSI_ENDPOINT)" - - "--metricsport={{ .RBDLivenessMetricsPort }}" - - "--metricspath=/metrics" - - "--polltime=60s" - - "--timeout=3s" - env: - - name: CSI_ENDPOINT - value: unix:///csi/csi-provisioner.sock - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - volumeMounts: - - name: socket-dir - mountPath: /csi - imagePullPolicy: {{ .ImagePullPolicy }} - {{ end }} - volumes: - - name: host-dev - hostPath: - path: /dev - {{ if .CSILogRotation }} - - name: csi-log - hostPath: - path: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }} - type: DirectoryOrCreate - - name: csi-logs-logrotate - emptyDir: - type: DirectoryOrCreate - {{ end }} - - name: host-sys - hostPath: - path: /sys - - name: lib-modules - hostPath: - path: /lib/modules - - name: socket-dir - emptyDir: { - medium: "Memory" - } - - name: ceph-csi-configs - projected: - sources: - - name: ceph-csi-config - configMap: - name: rook-ceph-csi-config - items: - - key: csi-cluster-config-json - path: config.json - - name: ceph-csi-mapping-config - configMap: - name: rook-ceph-csi-mapping-config - items: - - key: csi-mapping-config-json - path: cluster-mapping.json - {{ if .MountCustomCephConf }} - - name: ceph-config - configMap: - name: csi-ceph-conf-override - items: - - key: ceph.conf - path: ceph.conf - {{ end }} - - name: keys-tmp-dir - emptyDir: { - medium: "Memory" - } - - name: oidc-token - projected: - sources: - - serviceAccountToken: - path: oidc-token - expirationSeconds: 3600 - audience: ceph-csi-kms - {{ if .EnableCSIEncryption }} - - name: rook-ceph-csi-kms-config - configMap: - name: rook-ceph-csi-kms-config - items: - - key: config.json - path: config.json - {{ end }} diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml deleted file mode 100644 index 0493f06c7..000000000 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -# This is a service to expose the liveness and grpc metrics -apiVersion: v1 -kind: Service -metadata: - name: csi-rbdplugin-metrics - labels: - app: csi-metrics -spec: - ports: - - name: csi-http-metrics - port: 8080 - protocol: TCP - targetPort: {{ .RBDLivenessMetricsPort }} - selector: - contains: csi-rbdplugin-metrics diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml deleted file mode 100644 index 99adaa2c6..000000000 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ /dev/null @@ -1,315 +0,0 @@ -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: csi-rbdplugin - namespace: {{ .Namespace }} -spec: - selector: - matchLabels: - app: csi-rbdplugin - updateStrategy: - type: {{ .RBDPluginUpdateStrategy }} - {{ if eq .RBDPluginUpdateStrategy "RollingUpdate" }} - rollingUpdate: - maxUnavailable: {{ .RBDPluginUpdateStrategyMaxUnavailable }} - {{ end }} - template: - metadata: - labels: - app: csi-rbdplugin - contains: csi-rbdplugin-metrics - {{ range $key, $value := .CSIRBDPodLabels }} - {{ $key }}: "{{ $value }}" - {{ end }} - spec: - securityContext: {} - serviceAccountName: rook-csi-rbd-plugin-sa - {{ if .PluginPriorityClassName }} - priorityClassName: {{ .PluginPriorityClassName }} - {{ end }} - hostNetwork: {{ .EnableCSIHostNetwork }} - hostPID: true - # to use e.g. Rook orchestrated cluster, and mons' FQDN is - # resolved through k8s service, set dns policy to cluster first - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: driver-registrar - # This is necessary only for systems with SELinux, where - # non-privileged sidecar containers cannot access unix domain socket - # created by privileged CSI driver container. - securityContext: - privileged: true - capabilities: - add: [] - drop: ["ALL"] - image: {{ .RegistrarImage }} - imagePullPolicy: {{ .ImagePullPolicy }} - args: - - "--v={{ .LogLevel }}" - - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path={{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com/csi.sock" - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - - name: csi-rbdplugin - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - drop: ["ALL"] - allowPrivilegeEscalation: true - image: {{ .CSIPluginImage }} - args : - - "--nodeid=$(NODE_ID)" - - "--endpoint=$(CSI_ENDPOINT)" - - "--v={{ .LogLevel }}" - - "--type=rbd" - - "--nodeserver=true" - - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com" - - "--pidlimit=-1" - - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/" - {{ if .CSILogRotation }} - - "--logtostderr=false" - - "--alsologtostderr=true" - - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-rbdplugin.log" - {{ end }} - {{ if .EnableCSIAddonsSideCar }} - - "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)" - {{ end }} - {{- if .EnableCSITopology }} - - "--domainlabels={{ .CSIDomainLabels }}" - {{- end }} - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - {{ if .EnableCSIAddonsSideCar }} - - name: CSIADDONS_ENDPOINT - value: unix:///csi/csi-addons.sock - {{ end }} - imagePullPolicy: {{ .ImagePullPolicy }} - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: pods-mount-dir - mountPath: "{{ .KubeletDirPath }}/pods" - mountPropagation: "Bidirectional" - - name: plugin-mount-dir - mountPath: "{{ .KubeletDirPath }}/plugins" - mountPropagation: "Bidirectional" - - mountPath: /dev - name: host-dev - - mountPath: /sys - name: host-sys - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - name: ceph-csi-configs - mountPath: /etc/ceph-csi-config/ - - name: keys-tmp-dir - mountPath: /tmp/csi/keys - - name: host-run-mount - mountPath: /run/mount - {{ if .CSILogRotation }} - - mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }} - name: csi-log - {{ end }} - {{ if .EnablePluginSelinuxHostMount }} - - name: etc-selinux - mountPath: /etc/selinux - readOnly: true - {{ end }} - {{ if .MountCustomCephConf }} - - name: ceph-config - mountPath: /etc/ceph/ceph.conf - subPath: ceph.conf - {{ end }} - - name: oidc-token - mountPath: /run/secrets/tokens - readOnly: true - {{ if .EnableCSIEncryption }} - - name: rook-ceph-csi-kms-config - mountPath: /etc/ceph-csi-encryption-kms-config/ - {{ end }} - {{ if .EnableCSIAddonsSideCar }} - - name: csi-addons - securityContext: - privileged: true - capabilities: - add: [] - drop: ["ALL"] - image: {{ .CSIAddonsImage }} - args : - - "--node-id=$(NODE_ID)" - - "--v={{ .LogLevel }}" - - "--csi-addons-address=$(CSIADDONS_ENDPOINT)" - - "--controller-port={{ .CSIAddonsPort }}" - - "--pod=$(POD_NAME)" - - "--namespace=$(POD_NAMESPACE)" - - "--pod-uid=$(POD_UID)" - - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/" - {{ if .CSILogRotation }} - - "--logtostderr=false" - - "--alsologtostderr=true" - - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-addons.log" - {{ end }} - ports: - - containerPort: {{ .CSIAddonsPort }} - env: - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CSIADDONS_ENDPOINT - value: unix:///csi/csi-addons.sock - imagePullPolicy: {{ .ImagePullPolicy }} - volumeMounts: - - name: plugin-dir - mountPath: /csi - {{ if .CSILogRotation }} - - mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }} - name: csi-log - {{ end }} - {{ end }} - {{ if .EnableLiveness }} - - name: liveness-prometheus - securityContext: - privileged: true - capabilities: - add: [] - drop: ["ALL"] - image: {{ .CSIPluginImage }} - args: - - "--type=liveness" - - "--endpoint=$(CSI_ENDPOINT)" - - "--metricsport={{ .RBDLivenessMetricsPort }}" - - "--metricspath=/metrics" - - "--polltime=60s" - - "--timeout=3s" - env: - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - volumeMounts: - - name: plugin-dir - mountPath: /csi - imagePullPolicy: {{ .ImagePullPolicy }} - {{ end }} - volumes: - - name: plugin-dir - hostPath: - path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com" - type: DirectoryOrCreate - {{ if .CSILogRotation }} - - name: csi-log - hostPath: - path: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }} - type: DirectoryOrCreate - - name: csi-logs-logrotate - emptyDir: - type: DirectoryOrCreate - {{ end }} - - name: plugin-mount-dir - hostPath: - path: "{{ .KubeletDirPath }}/plugins" - type: Directory - - name: registration-dir - hostPath: - path: "{{ .KubeletDirPath }}/plugins_registry/" - type: Directory - - name: pods-mount-dir - hostPath: - path: "{{ .KubeletDirPath }}/pods" - type: Directory - - name: host-dev - hostPath: - path: /dev - - name: host-sys - hostPath: - path: /sys - - name: lib-modules - hostPath: - path: /lib/modules - - name: ceph-csi-configs - projected: - sources: - - name: ceph-csi-config - configMap: - name: rook-ceph-csi-config - items: - - key: csi-cluster-config-json - path: config.json - - name: ceph-csi-mapping-config - configMap: - name: rook-ceph-csi-mapping-config - items: - - key: csi-mapping-config-json - path: cluster-mapping.json - - name: keys-tmp-dir - emptyDir: { - medium: "Memory" - } - - name: host-run-mount - hostPath: - path: /run/mount - {{ if .EnablePluginSelinuxHostMount }} - - name: etc-selinux - hostPath: - path: /etc/selinux - {{ end }} - {{ if .MountCustomCephConf }} - - name: ceph-config - configMap: - name: csi-ceph-conf-override - items: - - key: ceph.conf - path: ceph.conf - {{ end }} - - name: oidc-token - projected: - sources: - - serviceAccountToken: - path: oidc-token - expirationSeconds: 3600 - audience: ceph-csi-kms - {{ if .EnableCSIEncryption }} - - name: rook-ceph-csi-kms-config - configMap: - name: rook-ceph-csi-kms-config - items: - - key: config.json - path: config.json - {{ end }} diff --git a/pkg/operator/ceph/csi/util.go b/pkg/operator/ceph/csi/util.go deleted file mode 100644 index 60f549bb1..000000000 --- a/pkg/operator/ceph/csi/util.go +++ /dev/null @@ -1,278 +0,0 @@ -/* -Copyright 2019 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "bytes" - "fmt" - "strconv" - "strings" - "text/template" - - "github.com/pkg/errors" - k8sutil "github.com/rook/rook/pkg/operator/k8sutil" - apps "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/yaml" -) - -func loadTemplate(name, templateData string, p templateParam) ([]byte, error) { - var writer bytes.Buffer - t := template.New(name) - t, err := t.Parse(templateData) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse template %v", name) - } - err = t.Execute(&writer, p) - return writer.Bytes(), err -} - -func templateToService(name, templateData string, p templateParam) (*corev1.Service, error) { - var svc corev1.Service - t, err := loadTemplate(name, templateData, p) - if err != nil { - return nil, errors.Wrap(err, "failed to load service template") - } - - err = yaml.Unmarshal(t, &svc) - if err != nil { - return nil, errors.Wrap(err, "failed to unmarshal service template") - } - return &svc, nil -} - -func templateToDaemonSet(name, templateData string, p templateParam) (*apps.DaemonSet, error) { - var ds apps.DaemonSet - t, err := loadTemplate(name, templateData, p) - if err != nil { - return nil, errors.Wrap(err, "failed to load daemonset template") - } - - err = yaml.Unmarshal(t, &ds) - if err != nil { - return nil, errors.Wrap(err, "failed to unmarshal daemonset template") - } - return &ds, nil -} - -func templateToDeployment(name, templateData string, p templateParam) (*apps.Deployment, error) { - var dep apps.Deployment - t, err := loadTemplate(name, templateData, p) - if err != nil { - return nil, errors.Wrap(err, "failed to load deployment template") - } - - err = yaml.Unmarshal(t, &dep) - if err != nil { - return nil, errors.Wrap(err, "failed to unmarshal deployment template") - } - return &dep, nil -} - -func applyLogrotateSidecar(specTemplate *corev1.PodTemplateSpec, name, templateData string, p templateParam) { - var logrotateSidecarContainer corev1.Container - t, err := loadTemplate(name, templateData, p) - if err != nil { - panic(errors.Wrap(err, "failed to load logrotate container template")) - } - - err = yaml.Unmarshal(t, &logrotateSidecarContainer) - if err != nil { - panic(errors.Wrap(err, "failed to unmarshal logrotate container template")) - } - specTemplate.Spec.Containers = append(specTemplate.Spec.Containers, logrotateSidecarContainer) -} - -func applyResourcesToContainers(key string, podspec *corev1.PodSpec) { - resource := getComputeResource(key) - - for _, r := range resource { - for i, c := range podspec.Containers { - if c.Name == r.Name { - podspec.Containers[i].Resources = r.Resource - } - } - } -} - -func getComputeResource(key string) []k8sutil.ContainerResource { - // Add Resource list if any - resource := []k8sutil.ContainerResource{} - var err error - - if resourceRaw := k8sutil.GetOperatorSetting(key, ""); resourceRaw != "" { - resource, err = k8sutil.YamlToContainerResourceArray(resourceRaw) - if err != nil { - logger.Warningf("failed to parse %q. %v", resourceRaw, err) - } - } - return resource -} - -func getToleration(tolerationsName string, defaultTolerations []corev1.Toleration) []corev1.Toleration { - // Add toleration if any, otherwise return defaultTolerations - tolerationsRaw := k8sutil.GetOperatorSetting(tolerationsName, "") - if tolerationsRaw == "" { - return defaultTolerations - } - tolerations, err := k8sutil.YamlToTolerations(tolerationsRaw) - if err != nil { - logger.Warningf("failed to parse %q for %q. %v", tolerationsRaw, tolerationsName, err) - return defaultTolerations - } - for i := range tolerations { - if tolerations[i].Key == "" { - tolerations[i].Operator = corev1.TolerationOpExists - } - - if tolerations[i].Operator == corev1.TolerationOpExists { - tolerations[i].Value = "" - } - } - return tolerations -} - -func getNodeAffinity(nodeAffinityName string, defaultNodeAffinity *corev1.NodeAffinity) *corev1.NodeAffinity { - // Add NodeAffinity if any, otherwise return defaultNodeAffinity - nodeAffinity := k8sutil.GetOperatorSetting(nodeAffinityName, "") - if nodeAffinity == "" { - return defaultNodeAffinity - } - v1NodeAffinity, err := k8sutil.GenerateNodeAffinity(nodeAffinity) - if err != nil { - logger.Warningf("failed to parse %q for %q. %v", nodeAffinity, nodeAffinityName, err) - return defaultNodeAffinity - } - return v1NodeAffinity -} - -func applyToPodSpec(pod *corev1.PodSpec, n *corev1.NodeAffinity, t []corev1.Toleration) { - pod.Tolerations = t - pod.Affinity = &corev1.Affinity{ - NodeAffinity: n, - } -} - -func getPortFromConfig(env string, defaultPort uint16) (uint16, error) { - port := k8sutil.GetOperatorSetting(env, strconv.Itoa(int(defaultPort))) - if strings.TrimSpace(k8sutil.GetOperatorSetting(env, strconv.Itoa(int(defaultPort)))) == "" { - return defaultPort, nil - } - p, err := strconv.ParseUint(port, 10, 64) - if err != nil { - return defaultPort, errors.Wrapf(err, "failed to parse port value for %q.", env) - } - if p > 65535 { - return defaultPort, errors.Errorf("%s port value is greater than 65535 for %s.", port, env) - } - return uint16(p), nil -} - -// Get PodAntiAffinity from a key and value pair -func GetPodAntiAffinity(key, value string) corev1.PodAntiAffinity { - return corev1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: key, - Operator: metav1.LabelSelectorOpIn, - Values: []string{value}, - }, - }, - }, - TopologyKey: k8sutil.LabelHostname(), - }, - }, - } -} - -func applyVolumeToPodSpec(configName string, podspec *corev1.PodSpec) { - volumesRaw := k8sutil.GetOperatorSetting(configName, "") - if volumesRaw == "" { - return - } - volumes, err := k8sutil.YamlToVolumes(volumesRaw) - if err != nil { - logger.Warningf("failed to parse %q for %q. %v", volumesRaw, configName, err) - return - } - for i := range volumes { - found := false - for j := range podspec.Volumes { - // check do we need to override any existing volumes - if volumes[i].Name == podspec.Volumes[j].Name { - podspec.Volumes[j] = volumes[i] - found = true - break - } - } - if !found { - // if not found add volume to volumes list - podspec.Volumes = append(podspec.Volumes, volumes[i]) - } - } -} - -func applyVolumeMountToContainer(configName, containerName string, podspec *corev1.PodSpec) { - volumeMountsRaw := k8sutil.GetOperatorSetting(configName, "") - if volumeMountsRaw == "" { - return - } - volumeMounts, err := k8sutil.YamlToVolumeMounts(volumeMountsRaw) - if err != nil { - logger.Warningf("failed to parse %q for %q. %v", volumeMountsRaw, configName, err) - return - } - if len(volumeMounts) > 0 { - for i, c := range podspec.Containers { - if c.Name == containerName { - for j := range volumeMounts { - found := false - for k := range podspec.Containers[i].VolumeMounts { - // override if the name is matching - if volumeMounts[j].Name == podspec.Containers[i].VolumeMounts[k].Name { - found = true - podspec.Containers[i].VolumeMounts[k] = volumeMounts[j] - break - } - } - if !found { - // if not found append it to the exiting volumes - podspec.Containers[i].VolumeMounts = append(podspec.Containers[i].VolumeMounts, volumeMounts[j]) - } - } - // return as we finished with found container - return - } - } - } -} - -// getImage returns the image for the given setting name. If the image does not contain version, -// the default version is appended from the default image. -func getImage(settingName, defaultImage string) string { - image := k8sutil.GetOperatorSetting(settingName, defaultImage) - if !strings.Contains(image, ":") { - version := strings.SplitN(defaultImage, ":", 2)[1] - image = fmt.Sprintf("%s:%s", image, version) - } - - return image -} diff --git a/pkg/operator/ceph/csi/util_test.go b/pkg/operator/ceph/csi/util_test.go deleted file mode 100644 index 845b034d4..000000000 --- a/pkg/operator/ceph/csi/util_test.go +++ /dev/null @@ -1,314 +0,0 @@ -/* -Copyright 2019 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "os" - "testing" - - "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" - corev1 "k8s.io/api/core/v1" -) - -func TestDaemonSetTemplate(t *testing.T) { - tp := templateParam{ - Param: CSIParam, - Namespace: "foo", - } - ds, err := templateToDaemonSet("test-ds", RBDPluginTemplatePath, tp) - assert.Nil(t, err) - assert.Equal(t, "driver-registrar", ds.Spec.Template.Spec.Containers[0].Name) -} - -func TestDeploymentTemplate(t *testing.T) { - tp := templateParam{ - Param: CSIParam, - Namespace: "foo", - } - _, err := templateToDeployment("test-dep", RBDProvisionerDepTemplatePath, tp) - assert.Nil(t, err) -} - -func TestGetPortFromConfig(t *testing.T) { - key := "TEST_CSI_PORT_ENV" - var defaultPort uint16 = 8000 - - // empty env variable - port, err := getPortFromConfig(key, defaultPort) - assert.Nil(t, err) - assert.Equal(t, port, defaultPort) - - // valid port is set in env - t.Setenv(key, "9000") - port, err = getPortFromConfig(key, defaultPort) - assert.Nil(t, err) - assert.Equal(t, port, uint16(9000)) - - // higher port value is set in env - t.Setenv(key, "65536") - port, err = getPortFromConfig(key, defaultPort) - assert.Error(t, err) - assert.Equal(t, port, defaultPort) - - // negative port is set in env - t.Setenv(key, "-1") - port, err = getPortFromConfig(key, defaultPort) - assert.Error(t, err) - assert.Equal(t, port, defaultPort) - - err = os.Unsetenv(key) - assert.Nil(t, err) -} - -func TestApplyingResourcesToRBDPlugin(t *testing.T) { - tp := templateParam{} - rbdPlugin, err := templateToDaemonSet("rbdplugin", RBDPluginTemplatePath, tp) - assert.Nil(t, err) - - // need to build using map[string]interface{} because the following resource - // doesn't serialise nicely - // https://pkg.go.dev/k8s.io/apimachinery/pkg/api/resource#Quantity - resource := []map[string]interface{}{ - { - "name": "driver-registrar", - "resource": map[string]interface{}{ - "limits": map[string]interface{}{ - "cpu": "200m", - "memory": "256Mi", - }, - "requests": map[string]interface{}{ - "cpu": "100m", - "memory": "128Mi", - }, - }, - }, - } - - resourceRaw, err := yaml.Marshal(resource) - assert.Nil(t, err) - os.Setenv(rbdPluginResource, string(resourceRaw)) - defer os.Unsetenv(rbdPluginResource) - applyResourcesToContainers(rbdPluginResource, &rbdPlugin.Spec.Template.Spec) - assert.Equal(t, rbdPlugin.Spec.Template.Spec.Containers[0].Resources.Requests.Memory().String(), "128Mi") - assert.Equal(t, rbdPlugin.Spec.Template.Spec.Containers[0].Resources.Limits.Memory().String(), "256Mi") - assert.Equal(t, rbdPlugin.Spec.Template.Spec.Containers[0].Resources.Requests.Cpu().String(), "100m") - assert.Equal(t, rbdPlugin.Spec.Template.Spec.Containers[0].Resources.Limits.Cpu().String(), "200m") -} - -func Test_applyVolumeToPodSpec(t *testing.T) { - // when no volumes specified - configKey := "TEST_CSI_PLUGIN_VOLUME" - dsName := "test-ds" - tp := templateParam{ - Param: CSIParam, - Namespace: "foo", - } - // rbdplugin has 11 volumes by default - defaultVolumes := 11 - ds, err := templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) - assert.Nil(t, err) - applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) - - assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes) - - // enable csi logrotate, two more volume mounts get added - tp.CSILogRotation = true - ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) - assert.Nil(t, err) - applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) - assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+2) - tp.CSILogRotation = false - - // add new volume - volumes := []corev1.Volume{ - { - Name: "test-host-dev", - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/dev"}}, - }, - } - volumeRaw, err := yaml.Marshal(volumes) - assert.Nil(t, err) - os.Setenv(configKey, string(volumeRaw)) - defer os.Unsetenv(configKey) - ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) - assert.Nil(t, err) - applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) - - assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+1) - // add one more volume - volumes = append(volumes, corev1.Volume{ - Name: "test-host-run", - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run"}}, - }) - volumeRaw, err = yaml.Marshal(volumes) - assert.Nil(t, err) - os.Setenv(configKey, string(volumeRaw)) - ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) - assert.Nil(t, err) - applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) - assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+2) - // override existing volume configuration - volumes[1].VolumeSource = corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{Path: "/run/test/run"}, - } - volumeRaw, err = yaml.Marshal(volumes) - assert.Nil(t, err) - os.Setenv(configKey, string(volumeRaw)) - ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) - assert.Nil(t, err) - applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) - assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+2) - // remove existing volume configuration - volumes = []corev1.Volume{ - { - Name: "test-host-dev", - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/dev"}}, - }, - } - volumeRaw, err = yaml.Marshal(volumes) - assert.Nil(t, err) - os.Setenv(configKey, string(volumeRaw)) - ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) - assert.Nil(t, err) - applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) - assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+1) -} - -func Test_applyVolumeMountToContainer(t *testing.T) { - // when no volumes specified - configKey := "TEST_CSI_PLUGIN_VOLUME_MOUNT" - dsName := "test-ds" - rbdContainerName := "csi-rbdplugin" - tp := templateParam{ - Param: CSIParam, - Namespace: "foo", - } - // csi-rbdplugin has 10 volumes by default - defaultVolumes := 10 - ds, err := templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) - assert.Nil(t, err) - applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) - - assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes) - - // enable csi logrotate, one more volumes get added - tp.CSILogRotation = true - ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) - assert.Nil(t, err) - applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) - assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+1) - tp.CSILogRotation = false - - // add new volume mount - volumeMounts := []corev1.VolumeMount{ - { - Name: "test-host-dev", - MountPath: "/dev/test", - }, - } - volumeMountsRaw, err := yaml.Marshal(volumeMounts) - assert.Nil(t, err) - os.Setenv(configKey, string(volumeMountsRaw)) - defer os.Unsetenv(configKey) - ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) - assert.Nil(t, err) - applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) - - assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+1) - // add one more volumemount - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: "test-host-run", - MountPath: "/run/test", - }) - volumeMountsRaw, err = yaml.Marshal(volumeMounts) - assert.Nil(t, err) - os.Setenv(configKey, string(volumeMountsRaw)) - ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) - assert.Nil(t, err) - applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) - assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+2) - // override existing volume configuration - volumeMounts[1].MountPath = "/run/test/run" - volumeMountsRaw, err = yaml.Marshal(volumeMounts) - assert.Nil(t, err) - os.Setenv(configKey, string(volumeMountsRaw)) - ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) - assert.Nil(t, err) - applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) - assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+2) - // remove existing volume configuration - volumeMounts = []corev1.VolumeMount{ - { - Name: "test-host-dev", - MountPath: "/dev/test", - }, - } - volumeMountsRaw, err = yaml.Marshal(volumeMounts) - assert.Nil(t, err) - os.Setenv(configKey, string(volumeMountsRaw)) - ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) - assert.Nil(t, err) - applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) - assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+1) -} - -func Test_getImage(t *testing.T) { - type args struct { - userImage string - defaultImage string - } - tests := []struct { - name string - args args - want string - }{ - { - name: "test with default image", - args: args{ - defaultImage: "quay.io/cephcsi/cephcsi:v3.16.2", - }, - want: DefaultCSIPluginImage, - }, - { - name: "test with user image", - args: args{ - userImage: "registry.io/private/cephcsi:v8", - defaultImage: "quay.io/cephcsi/cephcsi:v3.11.0", - }, - want: "registry.io/private/cephcsi:v8", - }, - { - name: "test with user image without version", - args: args{ - userImage: "registry.io/private/cephcsi", - defaultImage: "quay.io/cephcsi/cephcsi:v3.11.0", - }, - want: "registry.io/private/cephcsi:v3.11.0", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.args.userImage != "" { - os.Setenv("ROOK_CSI_CEPH_IMAGE", tt.args.userImage) - defer os.Unsetenv("ROOK_CSI_CEPH_IMAGE") - } - got := getImage("ROOK_CSI_CEPH_IMAGE", tt.args.defaultImage) - assert.Equal(t, tt.want, got) - }) - } -} diff --git a/pkg/operator/ceph/file/subvolumegroup/controller.go b/pkg/operator/ceph/file/subvolumegroup/controller.go index cf0955a1d..8cea3c917 100644 --- a/pkg/operator/ceph/file/subvolumegroup/controller.go +++ b/pkg/operator/ceph/file/subvolumegroup/controller.go @@ -51,8 +51,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" - - cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes" ) const ( @@ -245,13 +243,6 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ log.NamedInfo(request.NamespacedName, logger, "Removing finalizer from SVG CR %s without checking if the subvolume group contains any data as more than one SVG(count %d) contains the same filesystem and same SVG.", cephFilesystemSubVolumeGroup.Name, len(cephFsSvgList.Items)) } - if len(cephFsSvgList.Items) <= 1 { - err = csi.SaveClusterConfig(r.context.Clientset, buildClusterID(cephFilesystemSubVolumeGroup), cephCluster.Namespace, r.clusterInfo, nil) - if err != nil { - return reconcile.Result{}, errors.Wrap(err, "failed to save cluster config") - } - } - // Remove finalizer err = opcontroller.RemoveFinalizer(r.opManagerContext, r.client, cephFilesystemSubVolumeGroup) if err != nil { @@ -275,16 +266,10 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ } if cephCluster.Spec.External.Enable { log.NamedDebug(request.NamespacedName, logger, "skip creating external subvolume in external mode, create it manually, the controller will assume it's there") - err = r.updateClusterConfig(cephFilesystemSubVolumeGroup, cephCluster) - if err != nil { - return reconcile.Result{}, errors.Wrap(err, "failed to save cluster config") - } r.updateStatus(observedGeneration, namespacedName, cephv1.ConditionReady) - if csi.EnableCSIOperator() { - err = csi.CreateUpdateClientProfileSubVolumeGroup(r.clusterInfo.Context, r.client, r.clusterInfo, cephFilesystemSubVolumeGroupName, buildClusterID(cephFilesystemSubVolumeGroup), cephFilesystemSubVolumeGroup.Spec.CSIMetadataRadosNamespace) - if err != nil { - return reconcile.Result{}, errors.Wrap(err, "failed to create ceph csi-op config CR for subvolume") - } + err = csi.CreateUpdateClientProfileSubVolumeGroup(r.clusterInfo.Context, r.client, r.clusterInfo, cephFilesystemSubVolumeGroupName, buildClusterID(cephFilesystemSubVolumeGroup), cephFilesystemSubVolumeGroup.Spec.CSIMetadataRadosNamespace) + if err != nil { + return reconcile.Result{}, errors.Wrap(err, "failed to create ceph csi-op config CR for subvolume") } return reconcile.Result{}, nil } @@ -321,11 +306,6 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ return reconcile.Result{}, errors.Wrapf(err, "failed to create or update ceph filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name) } - err = r.updateClusterConfig(cephFilesystemSubVolumeGroup, cephCluster) - if err != nil { - return reconcile.Result{}, errors.Wrap(err, "failed to save cluster config") - } - err = cephclient.PinCephFSSubVolumeGroup(r.context, r.clusterInfo, cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup, getSubvolumeGroupName(cephFilesystemSubVolumeGroup)) if err != nil { return reconcile.Result{}, errors.Wrapf(err, "failed to pin filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name) @@ -333,11 +313,9 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ r.updateStatus(observedGeneration, request.NamespacedName, cephv1.ConditionReady) - if csi.EnableCSIOperator() { - err = csi.CreateUpdateClientProfileSubVolumeGroup(r.clusterInfo.Context, r.client, r.clusterInfo, cephFilesystemSubVolumeGroupName, buildClusterID(cephFilesystemSubVolumeGroup), cephFilesystemSubVolumeGroup.Spec.CSIMetadataRadosNamespace) - if err != nil { - return reconcile.Result{}, errors.Wrap(err, "failed to create ceph csi-op config CR for subvolumeGroup") - } + err = csi.CreateUpdateClientProfileSubVolumeGroup(r.clusterInfo.Context, r.client, r.clusterInfo, cephFilesystemSubVolumeGroupName, buildClusterID(cephFilesystemSubVolumeGroup), cephFilesystemSubVolumeGroup.Spec.CSIMetadataRadosNamespace) + if err != nil { + return reconcile.Result{}, errors.Wrap(err, "failed to create ceph csi-op config CR for subvolumeGroup") } // Return and do not requeue @@ -352,35 +330,6 @@ func getSubvolumeGroupName(cephFilesystemSubVolumeGroup *cephv1.CephFilesystemSu return cephFilesystemSubVolumeGroup.Name } -func (r *ReconcileCephFilesystemSubVolumeGroup) updateClusterConfig(cephFilesystemSubVolumeGroup *cephv1.CephFilesystemSubVolumeGroup, cephCluster cephv1.CephCluster) error { - // Update CSI config map - // If the mon endpoints change, the mon health check go routine will take care of updating the - // config map, so no special care is needed in this controller - csiClusterConfigEntry := csi.CSIClusterConfigEntry{ - Namespace: r.clusterInfo.Namespace, - ClusterInfo: cephcsi.ClusterInfo{ - Monitors: csi.MonEndpoints(r.clusterInfo.AllMonitors(), cephCluster.Spec.RequireMsgr2()), - CephFS: cephcsi.CephFS{ - SubvolumeGroup: getSubvolumeGroupName(cephFilesystemSubVolumeGroup), - KernelMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.KernelMountOptions, - FuseMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.FuseMountOptions, - }, - ReadAffinity: cephcsi.ReadAffinity{ - Enabled: csi.ReadAffinityEnabled(r.clusterInfo.CSIDriverSpec.ReadAffinity.Enabled, r.clusterInfo.CephVersion), - CrushLocationLabels: r.clusterInfo.CSIDriverSpec.ReadAffinity.CrushLocationLabels, - }, - }, - } - - csiClusterConfigEntry.CephFS.NetNamespaceFilePath = "" - - err := csi.SaveClusterConfig(r.context.Clientset, buildClusterID(cephFilesystemSubVolumeGroup), cephCluster.Namespace, r.clusterInfo, &csiClusterConfigEntry) - if err != nil { - return errors.Wrap(err, "failed to save cluster config") - } - return nil -} - // Create the ceph filesystem subvolume group func (r *ReconcileCephFilesystemSubVolumeGroup) createOrUpdateSubVolumeGroup(cephFilesystemSubVolumeGroup *cephv1.CephFilesystemSubVolumeGroup) error { nsName := opcontroller.NsName(cephFilesystemSubVolumeGroup.Namespace, cephFilesystemSubVolumeGroup.Name) diff --git a/pkg/operator/ceph/file/subvolumegroup/controller_test.go b/pkg/operator/ceph/file/subvolumegroup/controller_test.go index 78524c48f..a1a6af4f8 100644 --- a/pkg/operator/ceph/file/subvolumegroup/controller_test.go +++ b/pkg/operator/ceph/file/subvolumegroup/controller_test.go @@ -223,7 +223,7 @@ func TestFilesystemSubvolumeGroupController(t *testing.T) { } c.Executor = executor - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &csiopv1.ClientProfile{}) // Create a ReconcileCephFilesystemSubVolumeGroup object with the scheme and fake client. r = &ReconcileCephFilesystemSubVolumeGroup{ client: cl, @@ -232,8 +232,6 @@ func TestFilesystemSubvolumeGroupController(t *testing.T) { opManagerContext: ctx, } - // Enable CSI - csi.EnableRBD = true t.Setenv("POD_NAMESPACE", namespace) // Create CSI config map ownerRef := &metav1.OwnerReference{} @@ -251,11 +249,8 @@ func TestFilesystemSubvolumeGroupController(t *testing.T) { assert.NotEmpty(t, cephFilesystemSubVolumeGroup.Status.Info["clusterID"]) // test that csi configmap is created - cm, err := c.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, csi.ConfigName, metav1.GetOptions{}) + _, err = c.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, csi.ConfigName, metav1.GetOptions{}) assert.NoError(t, err) - assert.NotEmpty(t, cm.Data[csi.ConfigKey]) - assert.Contains(t, cm.Data[csi.ConfigKey], "clusterID") - assert.Contains(t, cm.Data[csi.ConfigKey], "group-a") err = c.Clientset.CoreV1().ConfigMaps(namespace).Delete(ctx, csi.ConfigName, metav1.DeleteOptions{}) assert.NoError(t, err) }) @@ -282,8 +277,6 @@ func TestFilesystemSubvolumeGroupController(t *testing.T) { opManagerContext: ctx, } - // Enable CSI - csi.EnableRBD = true t.Setenv("POD_NAMESPACE", namespace) // Create CSI config map ownerRef := &metav1.OwnerReference{} @@ -300,12 +293,8 @@ func TestFilesystemSubvolumeGroupController(t *testing.T) { assert.Equal(t, cephv1.ConditionReady, cephFilesystemSubVolumeGroup.Status.Phase) assert.NotEmpty(t, cephFilesystemSubVolumeGroup.Status.Info["clusterID"]) - // test that csi configmap is created - cm, err := c.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, csi.ConfigName, metav1.GetOptions{}) + _, err = c.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, csi.ConfigName, metav1.GetOptions{}) assert.NoError(t, err) - assert.NotEmpty(t, cm.Data[csi.ConfigKey]) - assert.Contains(t, cm.Data[csi.ConfigKey], "clusterID") - assert.Contains(t, cm.Data[csi.ConfigKey], "group-a") }) t.Run("success - test with multus ceph cluster", func(t *testing.T) { @@ -331,8 +320,6 @@ func TestFilesystemSubvolumeGroupController(t *testing.T) { opManagerContext: ctx, } - // Enable CSI - csi.EnableRBD = true t.Setenv("POD_NAMESPACE", namespace) // Create CSI config map ownerRef := &metav1.OwnerReference{} @@ -345,12 +332,8 @@ func TestFilesystemSubvolumeGroupController(t *testing.T) { assert.False(t, res.Requeue) // test that csi configmap is created - cm, err := c.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, csi.ConfigName, metav1.GetOptions{}) + _, err = c.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, csi.ConfigName, metav1.GetOptions{}) assert.NoError(t, err) - assert.NotEmpty(t, cm.Data[csi.ConfigKey]) - assert.Contains(t, cm.Data[csi.ConfigKey], "clusterID") - assert.Contains(t, cm.Data[csi.ConfigKey], "group-a") - assert.Contains(t, cm.Data[csi.ConfigKey], "netNamespaceFilePath") }) } diff --git a/pkg/operator/ceph/pool/radosnamespace/controller.go b/pkg/operator/ceph/pool/radosnamespace/controller.go index 913f5e95c..fc5f05381 100644 --- a/pkg/operator/ceph/pool/radosnamespace/controller.go +++ b/pkg/operator/ceph/pool/radosnamespace/controller.go @@ -50,8 +50,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - - cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes" ) const ( @@ -270,13 +268,6 @@ func (r *ReconcileCephBlockPoolRadosNamespace) reconcile(request reconcile.Reque log.NamedInfo(namespacedName, logger, "Removing finalizer from RNS without checking if the radosnamespaceName contains any data since more than one RNS(count %d) contains the same blockPool and rados name", len(cephRNSList.Items)) } - if len(cephRNSList.Items) <= 1 { - err = csi.SaveClusterConfig(r.context.Clientset, buildClusterID(radosNamespace), cephCluster.Namespace, r.clusterInfo, nil) - if err != nil { - return reconcile.Result{}, radosNamespace, errors.Wrap(err, "failed to save cluster config") - } - } - // Remove finalizer err = opcontroller.RemoveFinalizer(r.opManagerContext, r.client, radosNamespace) if err != nil { @@ -291,16 +282,10 @@ func (r *ReconcileCephBlockPoolRadosNamespace) reconcile(request reconcile.Reque if cephCluster.Spec.External.Enable { log.NamedDebug(namespacedName, logger, "skip creating external radosnamespace in external mode, create it manually, the controller will assume it's there") - err = r.updateClusterConfig(radosNamespace, cephCluster) - if err != nil { - return reconcile.Result{}, radosNamespace, errors.Wrap(err, "failed to save cluster config") - } r.updateStatus(r.client, namespacedName, cephv1.ConditionReady) - if csi.EnableCSIOperator() { - err = csi.CreateUpdateClientProfileRadosNamespace(r.clusterInfo.Context, r.client, r.clusterInfo, radosNamespaceName, buildClusterID(radosNamespace)) - if err != nil { - return reconcile.Result{}, radosNamespace, errors.Wrap(err, "failed to create ceph csi-op config CR for RadosNamespace") - } + err = csi.CreateUpdateClientProfileRadosNamespace(r.clusterInfo.Context, r.client, r.clusterInfo, radosNamespaceName, buildClusterID(radosNamespace)) + if err != nil { + return reconcile.Result{}, radosNamespace, errors.Wrap(err, "failed to create ceph csi-op config CR for RadosNamespace") } return reconcile.Result{}, radosNamespace, nil } @@ -348,11 +333,6 @@ func (r *ReconcileCephBlockPoolRadosNamespace) reconcile(request reconcile.Reque return reconcile.Result{}, radosNamespace, errors.Wrapf(err, "failed to create or update ceph pool rados namespace %q", radosNamespace.Name) } - err = r.updateClusterConfig(radosNamespace, cephCluster) - if err != nil { - return reconcile.Result{}, radosNamespace, errors.Wrap(err, "failed to save cluster config") - } - err = r.reconcileMirroring(radosNamespace, cephBlockPool) if err != nil { return reconcile.Result{}, radosNamespace, err @@ -360,11 +340,9 @@ func (r *ReconcileCephBlockPoolRadosNamespace) reconcile(request reconcile.Reque r.updateStatus(r.client, namespacedName, cephv1.ConditionReady) - if csi.EnableCSIOperator() { - err = csi.CreateUpdateClientProfileRadosNamespace(r.clusterInfo.Context, r.client, r.clusterInfo, radosNamespaceName, buildClusterID(radosNamespace)) - if err != nil { - return reconcile.Result{}, radosNamespace, errors.Wrap(err, "failed to create ceph csi-op config CR for RadosNamespace") - } + err = csi.CreateUpdateClientProfileRadosNamespace(r.clusterInfo.Context, r.client, r.clusterInfo, radosNamespaceName, buildClusterID(radosNamespace)) + if err != nil { + return reconcile.Result{}, radosNamespace, errors.Wrap(err, "failed to create ceph csi-op config CR for RadosNamespace") } // Return and do not requeue @@ -372,39 +350,6 @@ func (r *ReconcileCephBlockPoolRadosNamespace) reconcile(request reconcile.Reque return reconcile.Result{}, radosNamespace, nil } -func (r *ReconcileCephBlockPoolRadosNamespace) updateClusterConfig(cephBlockPoolRadosNamespace *cephv1.CephBlockPoolRadosNamespace, cephCluster cephv1.CephCluster) error { - // Update CSI config map - // If the mon endpoints change, the mon health check go routine will take care of updating the - // config map, so no special care is needed in this controller - csiClusterConfigEntry := csi.CSIClusterConfigEntry{ - Namespace: r.clusterInfo.Namespace, - ClusterInfo: cephcsi.ClusterInfo{ - Monitors: csi.MonEndpoints(r.clusterInfo.AllMonitors(), cephCluster.Spec.RequireMsgr2()), - RBD: cephcsi.RBD{ - RadosNamespace: cephv1.GetRadosNamespaceName(cephBlockPoolRadosNamespace), - }, - CephFS: cephcsi.CephFS{ - KernelMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.KernelMountOptions, - FuseMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.FuseMountOptions, - }, - ReadAffinity: cephcsi.ReadAffinity{ - Enabled: csi.ReadAffinityEnabled(r.clusterInfo.CSIDriverSpec.ReadAffinity.Enabled, r.clusterInfo.CephVersion), - CrushLocationLabels: r.clusterInfo.CSIDriverSpec.ReadAffinity.CrushLocationLabels, - }, - }, - } - - csiClusterConfigEntry.RBD.NetNamespaceFilePath = "" - - // Save cluster config in the csi config map - err := csi.SaveClusterConfig(r.context.Clientset, buildClusterID(cephBlockPoolRadosNamespace), cephCluster.Namespace, r.clusterInfo, &csiClusterConfigEntry) - if err != nil { - return errors.Wrap(err, "failed to save cluster config") - } - - return nil -} - // Create the ceph blockpool rados namespace func (r *ReconcileCephBlockPoolRadosNamespace) createOrUpdateRadosNamespace(cephBlockPoolRadosNamespace *cephv1.CephBlockPoolRadosNamespace) error { nsName := opcontroller.NsName(cephBlockPoolRadosNamespace.Namespace, cephBlockPoolRadosNamespace.Name) diff --git a/pkg/operator/ceph/pool/radosnamespace/controller_test.go b/pkg/operator/ceph/pool/radosnamespace/controller_test.go index ba64df19a..ee36cc0f0 100644 --- a/pkg/operator/ceph/pool/radosnamespace/controller_test.go +++ b/pkg/operator/ceph/pool/radosnamespace/controller_test.go @@ -241,7 +241,7 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { } c.Executor = executor - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &csiopv1.ClientProfile{}) // Create a ReconcileCephBlockPoolRadosNamespace object with the scheme and fake client. r = &ReconcileCephBlockPoolRadosNamespace{ client: cl, @@ -253,8 +253,6 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { recorder: events.NewFakeRecorder(50), } - // Enable CSI - csi.EnableRBD = true t.Setenv("POD_NAMESPACE", namespace) // Create CSI config map ownerRef := &metav1.OwnerReference{} @@ -270,13 +268,8 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { assert.NoError(t, err) assert.Equal(t, cephv1.ConditionReady, cephBlockPoolRadosNamespace.Status.Phase) - // test that csi configmap is created - cm, err := c.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, csi.ConfigName, metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotEmpty(t, cm.Data[csi.ConfigKey]) - assert.Contains(t, cm.Data[csi.ConfigKey], "clusterID") - assert.Contains(t, cm.Data[csi.ConfigKey], name) - err = c.Clientset.CoreV1().ConfigMaps(namespace).Delete(ctx, csi.ConfigName, metav1.DeleteOptions{}) + // csi configmap exists but is no longer populated by SaveClusterConfig + _, err = c.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, csi.ConfigName, metav1.GetOptions{}) assert.NoError(t, err) }) @@ -284,7 +277,7 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { cephCluster.Spec.External.Enable = true csiOpClientProfile := &csiopv1.ClientProfile{} - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &csiopv1.ClientProfile{}, &csiopv1.ClientProfile{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &csiopv1.ClientProfile{}) objects := []runtime.Object{ cephBlockPoolRadosNamespace, cephCluster, @@ -306,8 +299,6 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { recorder: events.NewFakeRecorder(50), } - // Enable CSI - csi.EnableRBD = true t.Setenv("POD_NAMESPACE", namespace) // Create CSI config map ownerRef := &metav1.OwnerReference{} @@ -324,12 +315,6 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { assert.Equal(t, cephv1.ConditionReady, cephBlockPoolRadosNamespace.Status.Phase) assert.NotEmpty(t, cephBlockPoolRadosNamespace.Status.Info["clusterID"]) - // test that csi configmap is created - cm, err := c.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, csi.ConfigName, metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotEmpty(t, cm.Data[csi.ConfigKey]) - assert.Contains(t, cm.Data[csi.ConfigKey], "clusterID") - assert.Contains(t, cm.Data[csi.ConfigKey], name) cephCluster.Spec.External.Enable = false }) diff --git a/tests/framework/installer/ceph_helm_installer.go b/tests/framework/installer/ceph_helm_installer.go index 0f49f2a88..3791355ab 100644 --- a/tests/framework/installer/ceph_helm_installer.go +++ b/tests/framework/installer/ceph_helm_installer.go @@ -31,6 +31,12 @@ import ( const ( OperatorChartName = "rook-ceph" CephClusterChartName = "rook-ceph-cluster" + + cephCsiOperatorHelmRepoURL = "https://ceph.github.io/ceph-csi-operator" + cephCsiOperatorHelmRepoName = "ceph-csi-operator" + cephCsiDriversChartName = "ceph-csi-drivers" + cephCsiDriversChartVersion = "0.6.0" + cephCsiDriversReleaseName = "ceph-csi-drivers" ) // The Ceph Storage CustomResource and StorageClass names used in testing @@ -60,17 +66,6 @@ func (h *CephInstaller) configureRookOperatorViaHelm(upgrade bool) error { "revisionHistoryLimit": "3", "enforceHostNetwork": "false", } - values["csi"] = map[string]interface{}{ - "csiRBDProvisionerResource": nil, - "csiRBDPluginResource": nil, - "csiCephFSProvisionerResource": nil, - "csiCephFSPluginResource": nil, - } - - if !h.settings.EnableCsiOperator { - values["rookUseCsiOperator"] = false - } - // create the operator namespace if err := h.k8shelper.CreateNamespace(h.settings.OperatorNamespace); err != nil { return errors.Errorf("failed to create namespace %s. %v", h.settings.Namespace, err) @@ -165,9 +160,103 @@ func (h *CephInstaller) configureRookCephClusterViaHelm(upgrade bool) error { } } + if h.settings.RookVersion == LocalBuildTag && h.settings.UseHelm { + if err := h.InstallCephCsiDriversViaHelm(); err != nil { + return errors.Wrap(err, "failed to install ceph-csi-drivers chart") + } + } + return nil } +func csiDriverChartValues(driverName string) map[string]interface{} { + return map[string]interface{}{ + "name": driverName, + "enabled": true, + "snapshotPolicy": "volumeSnapshot", + "imageSet": map[string]interface{}{ + "name": "rook-csi-operator-image-set-configmap", + }, + "nodePlugin": map[string]interface{}{ + "kubeletDirPath": "/var/lib/kubelet", + "priorityClassName": "system-node-critical", + "enableSeLinuxHostMount": false, + }, + "controllerPlugin": map[string]interface{}{ + "priorityClassName": "system-cluster-critical", + "replicas": 2, + "deploymentStrategy": map[string]interface{}{ + "type": "Recreate", + }, + }, + } +} + +func (h *CephInstaller) InstallCephCsiDriversViaHelm() error { + if err := h.k8shelper.CreateSnapshotCRD("create"); err != nil { + return errors.Wrap(err, "failed to install snapshot CRDs") + } + if err := h.k8shelper.CreateSnapshotController("create"); err != nil { + return errors.Wrap(err, "failed to install snapshot controller") + } + if err := h.k8shelper.WaitForSnapshotController(15); err != nil { + return errors.Wrap(err, "snapshot controller is not ready") + } + op := h.settings.OperatorNamespace + + drivers := map[string]interface{}{ + "rbd": csiDriverChartValues(fmt.Sprintf("%s.rbd.csi.ceph.com", op)), + "cephfs": csiDriverChartValues(fmt.Sprintf("%s.cephfs.csi.ceph.com", op)), + "nvmeof": map[string]interface{}{"enabled": false}, + } + if h.settings.TestNFSCSI { + drivers["nfs"] = csiDriverChartValues(fmt.Sprintf("%s.nfs.csi.ceph.com", op)) + } else { + drivers["nfs"] = map[string]interface{}{"enabled": false} + } + values := map[string]interface{}{ + "operatorConfig": map[string]interface{}{ + "name": "ceph-csi-operator-config", + "namespace": op, + "create": true, + "driverSpecDefaults": map[string]interface{}{ + "log": map[string]interface{}{"verbosity": 0}, + "imageSet": map[string]interface{}{ + "name": "rook-csi-operator-image-set-configmap", + }, + "snapshotPolicy": "volumeSnapshot", + "enableMetadata": false, + "generateOMapInfo": false, + "fsGroupPolicy": "File", + "deployCsiAddons": false, + "cephFsClientType": "kernel", + "nodePlugin": map[string]interface{}{ + "kubeletDirPath": "/var/lib/kubelet", + "priorityClassName": "system-node-critical", + "enableSeLinuxHostMount": false, + }, + "controllerPlugin": map[string]interface{}{ + "priorityClassName": "system-cluster-critical", + "replicas": 2, + "deploymentStrategy": map[string]interface{}{ + "type": "Recreate", + }, + }, + }, + }, + "drivers": drivers, + } + return h.helmHelper.InstallOrUpgradeHelmRepoChart( + op, + cephCsiDriversReleaseName, + cephCsiOperatorHelmRepoURL, + cephCsiOperatorHelmRepoName, + cephCsiDriversChartName, + cephCsiDriversChartVersion, + values, + ) +} + // removeCephClusterHelmResources tidies up the helm created CRs and Storage Classes, as they interfere with other tests. func (h *CephInstaller) removeCephClusterHelmResources() { if err := h.k8shelper.Clientset.StorageV1().StorageClasses().Delete(context.TODO(), BlockPoolSCName, v1.DeleteOptions{}); err != nil { diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 95416fd53..1e178175b 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -129,28 +129,30 @@ func (h *CephInstaller) CreateCephOperator() (err error) { return errors.Errorf("Failed to create rook-operator pod: %v ", err) } - if h.settings.TestNFSCSI { - csiNFSRBAC := h.Manifests.GetCSINFSRBAC() - if _, err = h.k8shelper.KubectlWithStdin(csiNFSRBAC, createFromStdinArgs...); err != nil { - return err - } - } - if err := h.CreateVolumeReplicationCRDs(); err != nil { return errors.Wrap(err, "failed to create volume replication CRDs") } + // Create CSI operator CRs and wait for the CSI operator pod + if err := h.InstallCSIOperator(); err != nil { + return err + } + _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetOperator(), createFromStdinArgs...) if err != nil { return errors.Errorf("Failed to create rook-operator pod: %v", err) } - logger.Infof("Rook operator started") - - if err := h.InstallCSIOperator(); err != nil { - return err + if h.settings.TestNFSCSI { + logger.Info("Creating NFS CSI Driver CR") + nfsDriver := h.Manifests.Settings().readManifest("csi/nfs/driver.yaml") + if _, err = h.k8shelper.KubectlWithStdin(nfsDriver, createFromStdinArgs...); err != nil { + return errors.Wrap(err, "failed to create NFS CSI driver") + } } + logger.Infof("Rook operator started") + return nil } @@ -695,6 +697,8 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(manifests ...CephManifests) } err = h.helmHelper.DeleteLocalRookHelmChart(namespace, CephClusterChartName) checkError(h.T(), err, fmt.Sprintf("cannot uninstall helm chart %s", CephClusterChartName)) + err = h.helmHelper.UninstallHelmReleaseIfExists(h.settings.OperatorNamespace, cephCsiDriversReleaseName) + checkError(h.T(), err, fmt.Sprintf("cannot uninstall helm chart %s", cephCsiDriversReleaseName)) } else { err = h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cephcluster", clusterName) checkError(h.T(), err, fmt.Sprintf("cannot remove cluster %s", namespace)) @@ -760,15 +764,6 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(manifests ...CephManifests) } else { logger.Infof("done deleting all the resources in the common manifest") } - if h.settings.TestNFSCSI { - _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetCSINFSRBAC(), deleteFromStdinArgs...) - if err != nil { - logger.Errorf("failed to remove csi nfs rbac manifest. %v", err) - } else { - logger.Info("done deleting all the resources in the csi nfs rbac manifest") - } - } - } } diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 0b3b4333a..f6a38b7af 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -29,7 +29,6 @@ import ( type CephManifests interface { Settings() *TestCephSettings GetCRDs(k8shelper *utils.K8sHelper) string - GetCSINFSRBAC() string GetCSIOperator() string GetOperator() string GetCommon() string @@ -86,10 +85,6 @@ func (m *CephManifestsMaster) GetCRDs(k8shelper *utils.K8sHelper) string { return m.settings.readManifest("crds.yaml") } -func (m *CephManifestsMaster) GetCSINFSRBAC() string { - return m.settings.readManifest("/csi/nfs/rbac.yaml") -} - func (m *CephManifestsMaster) GetOperator() string { var manifest string if utils.IsPlatformOpenShift() { diff --git a/tests/framework/installer/ceph_manifests_previous.go b/tests/framework/installer/ceph_manifests_previous.go index 0aabf71f8..cecb0146c 100644 --- a/tests/framework/installer/ceph_manifests_previous.go +++ b/tests/framework/installer/ceph_manifests_previous.go @@ -41,10 +41,6 @@ func (m *CephManifestsPreviousVersion) GetCRDs(k8shelper *utils.K8sHelper) strin return m.settings.readManifestFromGitHub("crds.yaml") } -func (m *CephManifestsPreviousVersion) GetCSINFSRBAC() string { - return m.settings.readManifestFromGitHub("/csi/nfs/rbac.yaml") -} - func (m *CephManifestsPreviousVersion) GetCSIOperator() string { manifest := m.settings.readManifestFromGitHub("csi-operator.yaml") return m.settings.replaceCSIOperatorSettings(m.settings.OperatorNamespace, manifest) diff --git a/tests/framework/installer/ceph_settings.go b/tests/framework/installer/ceph_settings.go index a5f241898..5383f557b 100644 --- a/tests/framework/installer/ceph_settings.go +++ b/tests/framework/installer/ceph_settings.go @@ -92,13 +92,9 @@ func (s *TestCephSettings) replaceOperatorSettings(manifest string) string { manifest = strings.ReplaceAll(manifest, `# CSI_LOG_LEVEL: "0"`, `CSI_LOG_LEVEL: "5"`) manifest = strings.ReplaceAll(manifest, `ROOK_ENABLE_DISCOVERY_DAEMON: "false"`, fmt.Sprintf(`ROOK_ENABLE_DISCOVERY_DAEMON: "%t"`, s.EnableDiscovery)) manifest = strings.ReplaceAll(manifest, `CSI_ENABLE_VOLUME_REPLICATION: "false"`, fmt.Sprintf(`CSI_ENABLE_VOLUME_REPLICATION: "%t"`, s.EnableVolumeReplication)) - manifest = strings.ReplaceAll(manifest, `ROOK_CSI_ENABLE_NFS: "false"`, fmt.Sprintf(`ROOK_CSI_ENABLE_NFS: "%t"`, s.TestNFSCSI)) manifest = strings.ReplaceAll(manifest, `# ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS: "maxObjects,maxSize" # default allowed configs`, `ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS: "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize,bucketPolicy,bucketLifecycle,bucketOwner"`) - if !s.EnableCsiOperator { - manifest = strings.ReplaceAll(manifest, `ROOK_USE_CSI_OPERATOR: "true"`, `ROOK_USE_CSI_OPERATOR: "false"`) - } if s.ClusterConcurrency > 1 { manifest = strings.ReplaceAll(manifest, `ROOK_RECONCILE_CONCURRENT_CLUSTERS: "1"`, fmt.Sprintf(`ROOK_RECONCILE_CONCURRENT_CLUSTERS: "%d"`, s.ClusterConcurrency)) } @@ -130,6 +126,7 @@ func replaceNamespaces(name, manifest, operatorNamespace, clusterNamespace strin // CSI Drivers manifest = strings.ReplaceAll(manifest, "rook-ceph.cephfs.csi.ceph.com # csi-provisioner-name", operatorNamespace+".cephfs.csi.ceph.com") manifest = strings.ReplaceAll(manifest, "rook-ceph.rbd.csi.ceph.com # csi-provisioner-name", operatorNamespace+".rbd.csi.ceph.com") + manifest = strings.ReplaceAll(manifest, "rook-ceph.nfs.csi.ceph.com # csi-provisioner-name", operatorNamespace+".nfs.csi.ceph.com") // Bucket storage class manifest = strings.ReplaceAll(manifest, "rook-ceph.ceph.rook.io/bucket # driver:namespace:cluster", clusterNamespace+".ceph.rook.io/bucket") diff --git a/tests/framework/utils/helm_helper.go b/tests/framework/utils/helm_helper.go index 75669c0cc..f210b9a4c 100644 --- a/tests/framework/utils/helm_helper.go +++ b/tests/framework/utils/helm_helper.go @@ -21,6 +21,7 @@ import ( "os" "path" "path/filepath" + "strings" "time" "gopkg.in/yaml.v2" @@ -137,6 +138,37 @@ func (h *HelmHelper) InstallVersionedChart(namespace, chart, version string, val return nil } +func (h *HelmHelper) InstallOrUpgradeHelmRepoChart(namespace, release, repoURL, repoName, chartName, version string, values map[string]interface{}) error { + if _, err := h.Execute("repo", "add", repoName, repoURL, "--force-update"); err != nil { + return fmt.Errorf("failed to add helm repo %q: %w", repoName, err) + } + if _, err := h.Execute("repo", "update"); err != nil { + logger.Warningf("helm repo update failed: %v", err) + } + cmdArgs := []string{ + "upgrade", "--install", "--create-namespace", + release, repoName + "/" + chartName, + "--version", version, + "--namespace", namespace, + } + if err := h.installChart(cmdArgs, values); err != nil { + return fmt.Errorf("helm upgrade --install %q in namespace %q: %w", release, namespace, err) + } + return nil +} + +func (h *HelmHelper) UninstallHelmReleaseIfExists(namespace, release string) error { + out, err := h.Execute("uninstall", release, "--namespace", namespace) + if err == nil { + return nil + } + msg := strings.ToLower(out + err.Error()) + if strings.Contains(msg, "not found") { + return nil + } + return fmt.Errorf("helm uninstall %q in namespace %q: %w; output=%s", release, namespace, err, out) +} + func (h *HelmHelper) installChart(cmdArgs []string, values map[string]interface{}) error { if values != nil { testValuesPath := "values-test.yaml" diff --git a/tests/framework/utils/snapshot.go b/tests/framework/utils/snapshot.go index af2c4aba0..966a89810 100644 --- a/tests/framework/utils/snapshot.go +++ b/tests/framework/utils/snapshot.go @@ -62,7 +62,7 @@ func (k8sh *K8sHelper) CheckSnapshotISReadyToUse(name, namespace string, retries return false, fmt.Errorf("giving up waiting for %q snapshot in namespace %q", name, namespace) } -// snapshotController creates or deletes the snapshotcontroller and required RBAC +// snapshotController creates/applies or deletes the snapshotcontroller and required RBAC func (k8sh *K8sHelper) snapshotController(action string) error { controllerURL := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, controllerPath) controllerManifest, err := getManifestFromURL(controllerURL) @@ -106,8 +106,8 @@ func (k8sh *K8sHelper) WaitForSnapshotController(retries int) error { } // CreateSnapshotController creates the snapshotcontroller and required RBAC -func (k8sh *K8sHelper) CreateSnapshotController() error { - return k8sh.snapshotController("create") +func (k8sh *K8sHelper) CreateSnapshotController(action string) error { + return k8sh.snapshotController(action) } // DeleteSnapshotController delete the snapshotcontroller and required RBAC @@ -115,9 +115,9 @@ func (k8sh *K8sHelper) DeleteSnapshotController() error { return k8sh.snapshotController("delete") } -// snapshotCRD can be used for creating or deleting the snapshot CRD's +// snapshotCRD can be used for creating, applying or deleting the snapshot CRD's func (k8sh *K8sHelper) snapshotCRD(action string) error { - // setting validate=false to skip CRD validation during create operation to + // setting validate=false to skip CRD validation during create/apply to // support lower Kubernetes versions. args := func(crdpath string) []string { a := []string{ @@ -125,7 +125,7 @@ func (k8sh *K8sHelper) snapshotCRD(action string) error { "-f", crdpath, } - if action == "create" { + if action == "create" || action == "apply" { a = append(a, "--validate=false") } return a @@ -169,8 +169,8 @@ func (k8sh *K8sHelper) snapshotCRD(action string) error { } // CreateSnapshotCRD creates the snapshot CRD -func (k8sh *K8sHelper) CreateSnapshotCRD() error { - return k8sh.snapshotCRD("create") +func (k8sh *K8sHelper) CreateSnapshotCRD(action string) error { + return k8sh.snapshotCRD(action) } // DeleteSnapshotCRD deletes the snapshot CRD diff --git a/tests/integration/ceph_base_block_test.go b/tests/integration/ceph_base_block_test.go index 142dda891..2c3acb211 100644 --- a/tests/integration/ceph_base_block_test.go +++ b/tests/integration/ceph_base_block_test.go @@ -111,10 +111,10 @@ func blockCSICloneTest(helper *clients.TestClient, k8sh *utils.K8sHelper, s *sui func blockCSISnapshotTest(helper *clients.TestClient, k8sh *utils.K8sHelper, s *suite.Suite, storageClassName, namespace string) { logger.Infof("install snapshot CRD") - err := k8sh.CreateSnapshotCRD() + err := k8sh.CreateSnapshotCRD("apply") require.NoError(s.T(), err) logger.Infof("install snapshot controller") - err = k8sh.CreateSnapshotController() + err = k8sh.CreateSnapshotController("apply") require.NoError(s.T(), err) // cleanup the CRD and controller in defer to make sure the CRD and // controller are removed as filesystem test also install CRD and controller. diff --git a/tests/integration/ceph_base_file_test.go b/tests/integration/ceph_base_file_test.go index 39e5f9098..0fcbc5496 100644 --- a/tests/integration/ceph_base_file_test.go +++ b/tests/integration/ceph_base_file_test.go @@ -118,11 +118,10 @@ func fileSystemCSICloneTest(helper *clients.TestClient, k8sh *utils.K8sHelper, s func fileSystemCSISnapshotTest(helper *clients.TestClient, k8sh *utils.K8sHelper, s *suite.Suite, storageClassName, namespace string, testNFS bool) { logger.Infof("install snapshot CRD") - err := k8sh.CreateSnapshotCRD() + err := k8sh.CreateSnapshotCRD("apply") require.NoError(s.T(), err) - logger.Infof("install snapshot controller") - err = k8sh.CreateSnapshotController() + err = k8sh.CreateSnapshotController("apply") require.NoError(s.T(), err) // cleanup the CRD and controller in defer to make sure the CRD and // controller are removed as block test also install CRD and controller. diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index 8b1dcad21..0ea167847 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -444,7 +444,6 @@ func (s *UpgradeSuite) upgradeToMaster() { if s.settings.UseHelm { logger.Info("Requiring msgr2 during helm upgrade to test the port conversion from 6789 to 3300") s.settings.RequireMsgr2 = true - // Upgrade the operator chart err := s.installer.UpgradeRookOperatorViaHelm() require.NoError(s.T(), err, "failed to upgrade the operator chart") diff --git a/tests/scripts/create-dev-cluster.sh b/tests/scripts/create-dev-cluster.sh index 33f7bfa03..659e46b62 100755 --- a/tests/scripts/create-dev-cluster.sh +++ b/tests/scripts/create-dev-cluster.sh @@ -122,7 +122,8 @@ create_rook_cluster() { if ! kubectl get namespace "$ROOK_OPERATOR_NS" &> /dev/null; then kubectl create namespace "$ROOK_OPERATOR_NS" fi - $KUBECTL apply -f crds.yaml -f common.yaml -f operator.yaml -f csi-operator.yaml + $KUBECTL apply -f crds.yaml -f common.yaml -f csi-operator.yaml + $KUBECTL apply -f operator.yaml $KUBECTL apply -f "$ROOK_CLUSTER_SPEC_FILE" -f toolbox.yaml $KUBECTL apply -f dashboard-external-http.yaml } diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 5299bf42b..b97ab2772 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -271,7 +271,7 @@ function build_rook_all() { function validate_yaml() { cd "${REPO_DIR}/deploy/examples" - kubectl create -f crds.yaml -f common.yaml -f csi/nfs/rbac.yaml + kubectl create -f crds.yaml -f common.yaml -f csi-operator.yaml # create the volume replication CRDs replication_version=v0.3.0 @@ -297,15 +297,14 @@ function validate_yaml() { function create_cluster_prerequisites() { # this might be called from another function that has already done a cd - (cd "${REPO_DIR}/deploy/examples" && kubectl create -f crds.yaml -f common.yaml -f csi/nfs/rbac.yaml) + (cd "${REPO_DIR}/deploy/examples" && kubectl create -f crds.yaml -f common.yaml -f csi-operator.yaml) } function remove_cluster_prerequisites() { - (cd "${REPO_DIR}/deploy/examples" && kubectl delete -f crds.yaml -f common.yaml -f csi/nfs/rbac.yaml) + (cd "${REPO_DIR}/deploy/examples" && kubectl delete -f crds.yaml -f common.yaml) } function deploy_manifest_with_local_build() { - sed -i 's/.*ROOK_CSI_ENABLE_NFS:.*/ ROOK_CSI_ENABLE_NFS: \"true\"/g' $1 if [[ "$USE_LOCAL_BUILD" != "false" ]]; then sed -i "s|image: docker.io/rook/ceph:.*|image: docker.io/rook/ceph:local-build|g" $1 fi @@ -348,7 +347,6 @@ function deploy_cluster() { cd "${REPO_DIR}/deploy/examples" deploy_manifest_with_local_build operator.yaml - kubectl create -f csi-operator.yaml if [ $# == 0 ]; then sed -i "s|#deviceFilter:|deviceFilter: $(block_dev_basename)|g" cluster-test.yaml @@ -410,6 +408,7 @@ function deploy_csi_hostnetwork_disabled_cluster() { cd "${REPO_DIR}/deploy/examples" sed -i 's/.*CSI_ENABLE_HOST_NETWORK:.*/ CSI_ENABLE_HOST_NETWORK: \"false\"/g' operator.yaml deploy_manifest_with_local_build operator.yaml + deploy_manifest_with_local_build csi/nfs/driver.yaml if [ $# == 0 ]; then sed -i "s|#deviceFilter:|deviceFilter: $(block_dev_basename)|g" cluster-test.yaml elif [ "$1" = "two_osds_in_device" ]; then @@ -531,7 +530,6 @@ function deploy_first_rook_cluster() { cd "${REPO_DIR}/deploy/examples" deploy_manifest_with_local_build operator.yaml - deploy_manifest_with_local_build csi-operator.yaml yq w -i -d0 cluster-test.yaml spec.dashboard.enabled false yq w -i -d0 cluster-test.yaml spec.storage.useAllDevices false yq w -i -d0 cluster-test.yaml spec.storage.deviceFilter "${DEVICE_NAME}"[1-3]