From b5b8cbf6aeabaf5cc1f39416d8d97999d0def222 Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Thu, 18 Aug 2022 11:27:51 +0530 Subject: [PATCH 1/2] csi: provide a way to enable `--setmetadata` at csi driver One can now tweak the operator configmap for CSI_ENABLE_METADATA boolean to enable/disable setting of PV/PVC/PVCNamespace & snapshot-name/snapshot-namespace/snapshotcontent-name on the RBD images and CephFS subvolume. Signed-off-by: Prasanna Kumar Kalever --- Documentation/Helm-Charts/operator-chart.md | 1 + deploy/charts/rook-ceph/templates/configmap.yaml | 1 + deploy/charts/rook-ceph/values.yaml | 5 +++++ deploy/examples/operator-openshift.yaml | 3 +++ deploy/examples/operator.yaml | 5 +++++ pkg/operator/ceph/csi/csi.go | 4 ++++ pkg/operator/ceph/csi/spec.go | 1 + .../template/cephfs/csi-cephfsplugin-provisioner-dep.yaml | 3 +++ .../ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml | 3 +++ 9 files changed, 26 insertions(+) diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md index 7a9769e34..f92c676d0 100644 --- a/Documentation/Helm-Charts/operator-chart.md +++ b/Documentation/Helm-Charts/operator-chart.md @@ -73,6 +73,7 @@ The following tables lists the configurable parameters of the rook-operator char | `csi.rbdFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted | `ReadWriteOnceWithFSType` | | `csi.cephFSFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted | `ReadWriteOnceWithFSType` | | `csi.nfsFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted | `ReadWriteOnceWithFSType` | +| `csi.enableMetadata` | Set volume metadata on the CephFS subvolumes and RBD images. | `false` | | `csi.logLevel` | Set logging level for cephCSI containers maintained by the cephCSI. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. | `0` | | `csi.sidecarLogLevel` | Set logging level for cephCSI sidecar containers. Supported values from 0 to 5. 0 for general useful logs (the default), 5 for trace level verbosity. | `0` | | `csi.grpcTimeoutInSeconds` | Set GRPC timeout for csi containers. | `150` | diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml index 2864d73f8..2b4b2244b 100644 --- a/deploy/charts/rook-ceph/templates/configmap.yaml +++ b/deploy/charts/rook-ceph/templates/configmap.yaml @@ -17,6 +17,7 @@ data: CSI_ENABLE_ENCRYPTION: {{ .Values.csi.enableCSIEncryption | quote }} CSI_ENABLE_OMAP_GENERATOR: {{ .Values.csi.enableOMAPGenerator | quote }} CSI_ENABLE_HOST_NETWORK: {{ .Values.csi.enableCSIHostNetwork | quote }} + CSI_ENABLE_METADATA: {{ .Values.csi.enableMetadata | quote }} {{- if .Values.csi.pluginPriorityClassName }} CSI_PLUGIN_PRIORITY_CLASSNAME: {{ .Values.csi.pluginPriorityClassName | quote }} {{- end }} diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index 77e28f0d7..72b86ee57 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -98,6 +98,11 @@ csi: # sidecar with CSI provisioner pod, to enable set it to true. enableOMAPGenerator: false + # (Optional) set to true to enable adding volume metadata on the CephFS subvolumes and RBD images. + # Not all users might be interested in getting volume/snapshot details as metadata on CephFS subvolume and RBD images. + # Hence enable metadata is false by default. + enableMetadata: false + # Set replicas for csi provisioner deployment. provisionerReplicas: 2 diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index 2f8678c2e..c6b6975a0 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -122,6 +122,9 @@ data: # Set to true to enable Ceph CSI pvc encryption support. CSI_ENABLE_ENCRYPTION: "false" + # set to true to enable adding volume metadata on the CephFS subvolume and RBD images. + # CSI_ENABLE_METADATA: "true" + # Set to true to enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary # in some network configurations where the SDN does not provide access to an external cluster or # there is significant drop in read/write performance. diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index e1a6a67ab..e12ea9c84 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -42,6 +42,11 @@ data: # there is significant drop in read/write performance. # CSI_ENABLE_HOST_NETWORK: "true" + # Set to true to enable adding volume metadata on the CephFS subvolume and RBD images. + # Not all users might be interested in getting volume/snapshot details as metadata on CephFS subvolume and RBD images. + # Hence enable metadata is false by default. + # CSI_ENABLE_METADATA: "true" + # Set logging level for cephCSI containers maintained by the cephCSI. # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. # CSI_LOG_LEVEL: "0" diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index 14b110c10..695aabccd 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -180,6 +180,10 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error { CSIParam.EnableCSIEncryption = true } + if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_METADATA", "false"), "true") { + CSIParam.CSIEnableMetadata = true + } + if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { CSIParam.CephFSPluginUpdateStrategy = onDelete } else { diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index ae0074816..925d0850c 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -58,6 +58,7 @@ type Param struct { VolumeReplicationImage string CSIAddonsImage string GRPCTimeout time.Duration + CSIEnableMetadata bool EnablePluginSelinuxHostMount bool EnableCSIHostNetwork bool EnableOMAPGenerator bool diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index a4c023e6e..fc0981c1c 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -114,6 +114,9 @@ spec: - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}" - "--metricspath=/metrics" - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" + {{ if .CSIEnableMetadata }} + - "--setmetadata={{ .CSIEnableMetadata }}" + {{ end }} env: - name: POD_IP valueFrom: diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 5deaca9c9..02a5a09c0 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -199,6 +199,9 @@ spec: {{ if .EnableCSIAddonsSideCar }} - "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)" {{ end }} + {{ if .CSIEnableMetadata }} + - "--setmetadata={{ .CSIEnableMetadata }}" + {{ end }} env: - name: POD_IP valueFrom: From e376c17bfb96ec5ca5ebe4e54d772eb214b64571 Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Thu, 18 Aug 2022 11:38:59 +0530 Subject: [PATCH 2/2] csi: provide a way to set `--clustername` at csi driver One can now use the operator configmap for CSI_CLUSTER_NAME variable for setting ClusterName on the RBD images and CephFS subvolume. This will be useful in cases like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster. Signed-off-by: Prasanna Kumar Kalever --- Documentation/Helm-Charts/operator-chart.md | 1 + deploy/charts/rook-ceph/templates/configmap.yaml | 3 +++ deploy/charts/rook-ceph/values.yaml | 3 +++ deploy/examples/operator-openshift.yaml | 4 ++++ deploy/examples/operator.yaml | 4 ++++ pkg/operator/ceph/csi/csi.go | 1 + pkg/operator/ceph/csi/spec.go | 1 + .../csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml | 3 +++ .../ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml | 3 +++ 9 files changed, 23 insertions(+) diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md index f92c676d0..579d7d9e7 100644 --- a/Documentation/Helm-Charts/operator-chart.md +++ b/Documentation/Helm-Charts/operator-chart.md @@ -74,6 +74,7 @@ The following tables lists the configurable parameters of the rook-operator char | `csi.cephFSFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted | `ReadWriteOnceWithFSType` | | `csi.nfsFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted | `ReadWriteOnceWithFSType` | | `csi.enableMetadata` | Set volume metadata on the CephFS subvolumes and RBD images. | `false` | +| `csi.clusterName` | Cluster name identifier to set as metadata on the CephFS subvolumes and RBD images. | `` | | `csi.logLevel` | Set logging level for cephCSI containers maintained by the cephCSI. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. | `0` | | `csi.sidecarLogLevel` | Set logging level for cephCSI sidecar containers. Supported values from 0 to 5. 0 for general useful logs (the default), 5 for trace level verbosity. | `0` | | `csi.grpcTimeoutInSeconds` | Set GRPC timeout for csi containers. | `150` | diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml index 2b4b2244b..2289b7062 100644 --- a/deploy/charts/rook-ceph/templates/configmap.yaml +++ b/deploy/charts/rook-ceph/templates/configmap.yaml @@ -178,6 +178,9 @@ data: {{- if .Values.csi.sidecarLogLevel }} CSI_SIDECAR_LOG_LEVEL: {{ .Values.csi.sidecarLogLevel | quote }} {{- end }} +{{- if .Values.csi.clusterName }} + CSI_CLUSTER_NAME: {{ .Values.csi.clusterName | quote }} +{{- end }} {{- if .Values.csi.grpcTimeoutInSeconds }} CSI_GRPC_TIMEOUT_SECONDS: {{ .Values.csi.grpcTimeoutInSeconds | quote }} {{- end }} diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index 72b86ee57..860427cbd 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -106,6 +106,9 @@ csi: # Set replicas for csi provisioner deployment. provisionerReplicas: 2 + # (Optional) cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful + # in cases like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster. + # clusterName: "my-prod-cluster" # Set logging level for cephCSI containers maintained by the cephCSI. # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. #logLevel: 0 diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index c6b6975a0..ce36e76a6 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -125,6 +125,10 @@ data: # set to true to enable adding volume metadata on the CephFS subvolume and RBD images. # CSI_ENABLE_METADATA: "true" + # cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful in cases + # like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster. + # CSI_CLUSTER_NAME: "my-prod-cluster" + # Set to true to enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary # in some network configurations where the SDN does not provide access to an external cluster or # there is significant drop in read/write performance. diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index e12ea9c84..705732155 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -47,6 +47,10 @@ data: # Hence enable metadata is false by default. # CSI_ENABLE_METADATA: "true" + # cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful in cases + # like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster. + # CSI_CLUSTER_NAME: "my-prod-cluster" + # Set logging level for cephCSI containers maintained by the cephCSI. # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. # CSI_LOG_LEVEL: "0" diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index 695aabccd..87cffea51 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -265,6 +265,7 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error { CSIParam.CSINFSPodLabels = k8sutil.ParseStringToLabels(csiNFSPodLabels) csiRBDPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_RBD_POD_LABELS", "") CSIParam.CSIRBDPodLabels = k8sutil.ParseStringToLabels(csiRBDPodLabels) + CSIParam.CSIClusterName = k8sutil.GetValue(r.opConfig.Parameters, "CSI_CLUSTER_NAME", "") return nil } diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 925d0850c..6077d6aff 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -57,6 +57,7 @@ type Param struct { ProvisionerPriorityClassName string VolumeReplicationImage string CSIAddonsImage string + CSIClusterName string GRPCTimeout time.Duration CSIEnableMetadata bool EnablePluginSelinuxHostMount bool diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index fc0981c1c..d865a1e73 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -117,6 +117,9 @@ spec: {{ if .CSIEnableMetadata }} - "--setmetadata={{ .CSIEnableMetadata }}" {{ end }} + {{ if .CSIClusterName }} + - "--clustername={{ .CSIClusterName }}" + {{ end }} env: - name: POD_IP valueFrom: diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 02a5a09c0..243f5a7cb 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -202,6 +202,9 @@ spec: {{ if .CSIEnableMetadata }} - "--setmetadata={{ .CSIEnableMetadata }}" {{ end }} + {{ if .CSIClusterName }} + - "--clustername={{ .CSIClusterName }}" + {{ end }} env: - name: POD_IP valueFrom: