From 9b4361b3790d9528dca007e7fb79ee179e02a7e9 Mon Sep 17 00:00:00 2001 From: yati1998 Date: Fri, 2 Sep 2022 12:09:47 +0530 Subject: [PATCH 1/3] rbdmirror: remove volume replication sidecar The volume replication operator is being moved to kubernetes-csi-addons. This commit hence, removes the volume replication sidecar and updates the related documentation. It will also update the csi-addons sidecar version to the latest one. Closes: #10655 Signed-off-by: yati1998 --- Documentation/Helm-Charts/operator-chart.md | 4 +- .../Block-Storage-RBD/rbd-mirroring.md | 18 ++-- .../Ceph-CSI/ceph-csi-drivers.md | 34 +++---- .../Ceph-CSI/custom-images.md | 3 +- .../rook-ceph/templates/clusterrole.yaml | 12 --- .../charts/rook-ceph/templates/configmap.yaml | 6 -- deploy/charts/rook-ceph/values.yaml | 8 +- deploy/examples/common.yaml | 12 --- deploy/examples/images.txt | 3 +- deploy/examples/operator-openshift.yaml | 7 +- deploy/examples/operator.yaml | 7 +- pkg/operator/ceph/csi/csi.go | 6 -- pkg/operator/ceph/csi/spec.go | 98 +++++++++---------- .../rbd/csi-rbdplugin-provisioner-dep.yaml | 19 ---- tests/framework/installer/ceph_installer.go | 4 +- 15 files changed, 77 insertions(+), 164 deletions(-) diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md index 7db6fadea..f7320f553 100644 --- a/Documentation/Helm-Charts/operator-chart.md +++ b/Documentation/Helm-Charts/operator-chart.md @@ -124,10 +124,8 @@ The following tables lists the configurable parameters of the rook-operator char | `csi.attacher.image` | Kubernetes CSI Attacher image. | `registry.k8s.io/sig-storage/csi-attacher:v3.4.0` | | `csi.cephfsPodLabels` | Labels to add to the CSI CephFS Pods. | `` | | `csi.rbdPodLabels` | Labels to add to the CSI RBD Pods. | `` | -| `csi.volumeReplication.enabled` | Enable Volume Replication. | `false` | -| `csi.volumeReplication.image` | Volume Replication Controller image. | `quay.io/csiaddons/volumereplication-operator:v0.3.0` | | `csi.csiAddons.enabled` | Enable CSIAddons | `false` | -| `csi.csiAddons.image` | CSIAddons Sidecar image. | `quay.io/csiaddons/k8s-sidecar:v0.4.0` | +| `csi.csiAddons.image` | CSIAddons Sidecar image. | `quay.io/csiaddons/k8s-sidecar:v0.5.0` | | `csi.nfs.enabled` | Enable nfs driver. | `false` | | `admissionController.tolerations` | Array of tolerations in YAML format which will be added to admission controller deployment. | `` | | `admissionController.nodeAffinity` | The node labels for affinity of the admission controller deployment [^1] | `` | diff --git a/Documentation/Storage-Configuration/Block-Storage-RBD/rbd-mirroring.md b/Documentation/Storage-Configuration/Block-Storage-RBD/rbd-mirroring.md index bc6b3d2de..d22f1b925 100644 --- a/Documentation/Storage-Configuration/Block-Storage-RBD/rbd-mirroring.md +++ b/Documentation/Storage-Configuration/Block-Storage-RBD/rbd-mirroring.md @@ -165,13 +165,13 @@ APIs for storage disaster recovery. The extended APIs are provided via Custom Resource Definition(CRD). Create the VolumeReplication CRDs on all the peer clusters. ```console -kubectl create -f https://raw.githubusercontent.com/csi-addons/volume-replication-operator/v0.3.0/config/crd/bases/replication.storage.openshift.io_volumereplications.yaml -kubectl create -f https://raw.githubusercontent.com/csi-addons/volume-replication-operator/v0.3.0/config/crd/bases/replication.storage.openshift.io_volumereplicationclasses.yaml +kubectl create -f https://raw.githubusercontent.com/csi-addons/kubernetes-csi-addons/v0.5.0/config/crd/bases/replication.storage.openshift.io_volumereplicationclasses.yaml +kubectl create -f https://raw.githubusercontent.com/csi-addons/kubernetes-csi-addons/v0.5.0/config/crd/bases/replication.storage.openshift.io_volumereplications.yaml ``` ## Enable CSI Replication Sidecars -To achieve RBD Mirroring, `csi-omap-generator` and `volume-replication` +To achieve RBD Mirroring, `csi-omap-generator` and `csi-addons` containers need to be deployed in the RBD provisioner pods, which are not enabled by default. * **Omap Generator**: Omap generator is a sidecar container that when @@ -182,13 +182,13 @@ To achieve RBD Mirroring, `csi-omap-generator` and `volume-replication` * **Volume Replication Operator**: Volume Replication Operator is a kubernetes operator that provides common and reusable APIs for - storage disaster recovery. + storage disaster recovery. The volume replication operation is + supported by the [CSIAddons](https://github.com/csi-addons/kubernetes-csi-addons#readme) It is based on [csi-addons/spec](https://github.com/csi-addons/spec) specification and can be used by any storage provider. - For more details, refer to [volume replication operator](https://github.com/csi-addons/volume-replication-operator). Execute the following steps on each peer cluster to enable the - OMap generator and Volume Replication sidecars: + OMap generator and CSIADDONS sidecars: * Edit the `rook-ceph-operator-config` configmap and add the following configurations @@ -202,7 +202,7 @@ Add the following properties if not present: ```yaml data: CSI_ENABLE_OMAP_GENERATOR: "true" - CSI_ENABLE_VOLUME_REPLICATION: "true" + CSI_ENABLE_CSIADDONS: "true" ``` * After updating the configmap with those settings, two new sidecars @@ -220,10 +220,6 @@ the storage admin information required for the volume replication operator. * **VolumeReplication**: *VolumeReplication* is a namespaced resource that contains references to storage object to be replicated and VolumeReplicationClass corresponding to the driver providing replication. -!!! info - For more information, please refer to the - [volume-replication-operator](https://github.com/csi-addons/volume-replication-operator). - ## Enable mirroring on a PVC Below guide assumes that we have a PVC (rbd-pvc) in BOUND state; created using diff --git a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md index ea2f023e2..3c3dc40d7 100644 --- a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md +++ b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md @@ -71,26 +71,17 @@ PVC will be updated to new size. ## RBD Mirroring -To support RBD Mirroring, the [Volume Replication Operator](https://github.com/csi-addons/volume-replication-operator/blob/main/README.md) will be started in the RBD provisioner pod. -The Volume Replication Operator is a kubernetes operator that provides common and reusable APIs for storage disaster recovery. It is based on [csi-addons/spec](https://github.com/csi-addons/spec) specification and can be used by any storage provider. +To support RBD Mirroring, the [CSI-Addons sidecar](https://github.com/csi-addons/kubernetes-csi-addons#readme) will be started in the RBD provisioner pod. +The CSI-Addons supports the VolumeReplication operation. The volume replication controller provides common and reusable APIs for storage disaster recovery. It is based on [csi-addons/spec](https://github.com/csi-addons/spec) specification and can be used by any storage provider. It follows the controller pattern and provides extended APIs for storage disaster recovery. The extended APIs are provided via Custom Resource Definitions (CRDs). ### Prerequisites Kubernetes version 1.21 or greater is required. -### Enable volume replication +### Enable CSIAddons Sidecar -1. Install the volume replication CRDs: - - ```console - kubectl create -f https://raw.githubusercontent.com/csi-addons/volume-replication-operator/v0.3.0/config/crd/bases/replication.storage.openshift.io_volumereplications.yaml - kubectl create -f https://raw.githubusercontent.com/csi-addons/volume-replication-operator/v0.3.0/config/crd/bases/replication.storage.openshift.io_volumereplicationclasses.yaml - ``` - -2. Enable the volume replication controller: - * For Helm deployments see the [`csi.volumeReplication.enabled` setting](../../Helm-Charts/operator-chart.md#configuration). - * For non-Helm deployments set `CSI_ENABLE_VOLUME_REPLICATION: "true"` in operator.yaml +To enable the CSIAddons sidecar and deploy the controller, Please follow the steps [below](#csi-addons-controller) ## Ephemeral volume support @@ -132,9 +123,9 @@ The CSI-Addons Controller handles the requests from users to initiate an operati Users can deploy the controller by running the following commands: ```console -kubectl create -f https://raw.githubusercontent.com/csi-addons/kubernetes-csi-addons/v0.4.0/deploy/controller/crds.yaml -kubectl create -f https://raw.githubusercontent.com/csi-addons/kubernetes-csi-addons/v0.4.0/deploy/controller/rbac.yaml -kubectl create -f https://raw.githubusercontent.com/csi-addons/kubernetes-csi-addons/v0.4.0/deploy/controller/setup-controller.yaml +kubectl create -f https://raw.githubusercontent.com/csi-addons/kubernetes-csi-addons/v0.5.0/deploy/controller/crds.yaml +kubectl create -f https://raw.githubusercontent.com/csi-addons/kubernetes-csi-addons/v0.5.0/deploy/controller/rbac.yaml +kubectl create -f https://raw.githubusercontent.com/csi-addons/kubernetes-csi-addons/v0.5.0/deploy/controller/setup-controller.yaml ``` This creates the required crds and configure permissions. @@ -166,11 +157,14 @@ kubectl patch cm rook-ceph-operator-config -nrook-ceph -p $'data:\n "CSI_ENABLE_ CSI-Addons supports the following operations: * Reclaim Space - * [Creating a ReclaimSpaceJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.3.0/docs/reclaimspace.md#reclaimspacejob) - * [Creating a ReclaimSpaceCronJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.3.0/docs/reclaimspace.md#reclaimspacecronjob) - * [Annotating PersistentVolumeClaims](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.3.0/docs/reclaimspace.md#annotating-perstentvolumeclaims) + * [Creating a ReclaimSpaceJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.5.0/docs/reclaimspace.md#reclaimspacejob) + * [Creating a ReclaimSpaceCronJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.5.0/docs/reclaimspace.md#reclaimspacecronjob) + * [Annotating PersistentVolumeClaims](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.5.0/docs/reclaimspace.md#annotating-perstentvolumeclaims) * Network Fencing - * [Creating a NetworkFence](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.3.0/docs/networkfence.md) + * [Creating a NetworkFence](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.5.0/docs/networkfence.md) +* Volume Replication + * [Creating VolumeReplicationClass](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.5.0/docs/volumereplicationclass.md) + * [Creating VolumeReplication CR](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.5.0/docs/volumereplication.md) ## Enable RBD Encryption Support diff --git a/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md b/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md index f997c3ec6..66e3c15a7 100644 --- a/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md +++ b/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md @@ -24,8 +24,7 @@ ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v3.2.1" ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.5.0" ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.5.0" ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v6.0.1" -CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.3.0" -ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.4.0" +ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.5.0" ``` ### **Use default images** diff --git a/deploy/charts/rook-ceph/templates/clusterrole.yaml b/deploy/charts/rook-ceph/templates/clusterrole.yaml index 396b620e2..4673e582e 100644 --- a/deploy/charts/rook-ceph/templates/clusterrole.yaml +++ b/deploy/charts/rook-ceph/templates/clusterrole.yaml @@ -554,18 +554,6 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumereplications", "volumereplicationclasses"] - verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumereplications/finalizers"] - verbs: ["update"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumereplications/status"] - verbs: ["get", "patch", "update"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumereplicationclasses/status"] - verbs: ["get"] - apiGroups: [""] resources: ["serviceaccounts"] verbs: ["get"] diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml index 56f093aa7..8056ace6c 100644 --- a/deploy/charts/rook-ceph/templates/configmap.yaml +++ b/deploy/charts/rook-ceph/templates/configmap.yaml @@ -76,12 +76,6 @@ data: ROOK_CSI_RESIZER_IMAGE: {{ .Values.csi.resizer.image | quote }} {{- end }} {{- end }} -{{- if .Values.csi.volumeReplication }} - CSI_ENABLE_VOLUME_REPLICATION: {{ .Values.csi.volumeReplication.enabled | quote }} -{{- if .Values.csi.volumeReplication.image }} - CSI_VOLUME_REPLICATION_IMAGE: {{ .Values.csi.volumeReplication.image | quote }} -{{- end }} -{{- end }} {{- if .Values.csi.csiAddons }} CSI_ENABLE_CSIADDONS: {{ .Values.csi.csiAddons.enabled | quote }} {{- if .Values.csi.csiAddons.image }} diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index 5dcfda821..b3c780346 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -377,16 +377,10 @@ csi: #nfsPodLabels: "key1=value1,key2=value2" # Labels to add to the CSI RBD Deployments and DaemonSets Pods. #rbdPodLabels: "key1=value1,key2=value2" - # Enable the volume replication controller. - # Before enabling, ensure the Volume Replication CRDs are created. - # See https://rook.io/docs/rook/latest/ceph-csi-drivers.html#rbd-mirroring - volumeReplication: - enabled: false - #image: "quay.io/csiaddons/volumereplication-operator:v0.3.0" # Enable the CSIAddons sidecar. csiAddons: enabled: false - #image: "quay.io/csiaddons/k8s-sidecar:v0.4.0" + #image: "quay.io/csiaddons/k8s-sidecar:v0.5.0" # Enable the nfs csi driver. nfs: enabled: false diff --git a/deploy/examples/common.yaml b/deploy/examples/common.yaml index dfdebfbed..de4baeb35 100644 --- a/deploy/examples/common.yaml +++ b/deploy/examples/common.yaml @@ -142,18 +142,6 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumereplications", "volumereplicationclasses"] - verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumereplications/finalizers"] - verbs: ["update"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumereplications/status"] - verbs: ["get", "patch", "update"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumereplicationclasses/status"] - verbs: ["get"] - apiGroups: [""] resources: ["serviceaccounts"] verbs: ["get"] diff --git a/deploy/examples/images.txt b/deploy/examples/images.txt index 82b281797..9ed4e531f 100644 --- a/deploy/examples/images.txt +++ b/deploy/examples/images.txt @@ -1,7 +1,6 @@ quay.io/ceph/ceph:v17.2.3 quay.io/cephcsi/cephcsi:v3.7.0 - quay.io/csiaddons/k8s-sidecar:v0.4.0 - quay.io/csiaddons/volumereplication-operator:v0.3.0 + quay.io/csiaddons/k8s-sidecar:v0.5.0 registry.k8s.io/sig-storage/csi-attacher:v3.5.0 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1 registry.k8s.io/sig-storage/csi-provisioner:v3.2.1 diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index 7453099ab..490bceda7 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -514,16 +514,11 @@ data: # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster. # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs. ROOK_ENABLE_DISCOVERY_DAEMON: "false" - # Enable the volume replication controller - # Before enabling, ensure the Volume Replication CRDs are created. - # See https://rook.io/docs/rook/latest/ceph-csi-drivers.html#rbd-mirroring - CSI_ENABLE_VOLUME_REPLICATION: "false" # The timeout value (in seconds) of Ceph commands. It should be >= 1. If this variable is not set or is an invalid value, it's default to 15. ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS: "15" - # CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.3.0" # Enable the csi addons sidecar. CSI_ENABLE_CSIADDONS: "false" - # ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.4.0" + # ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.5.0" # The GCSI RPC timeout value (in seconds). It should be >= 120. If this variable is not set or is an invalid value, it's default to 150. CSI_GRPC_TIMEOUT_SECONDS: "150" --- diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index e928f1284..c9450aab2 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -441,14 +441,9 @@ data: ROOK_ENABLE_DISCOVERY_DAEMON: "false" # The timeout value (in seconds) of Ceph commands. It should be >= 1. If this variable is not set or is an invalid value, it's default to 15. ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS: "15" - # Enable the volume replication controller. - # Before enabling, ensure the Volume Replication CRDs are created. - # See https://rook.io/docs/rook/latest/ceph-csi-drivers.html#rbd-mirroring - CSI_ENABLE_VOLUME_REPLICATION: "false" - # CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.3.0" # Enable the csi addons sidecar. CSI_ENABLE_CSIADDONS: "false" - # ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.4.0" + # ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.5.0" # The CSI GRPC timeout value (in seconds). It should be >= 120. If this variable is not set or is an invalid value, it's default to 150. CSI_GRPC_TIMEOUT_SECONDS: "150" --- diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index bb12c4592..5c5c9750d 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -164,11 +164,6 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error { CSIParam.EnableCephFSSnapshotter = false } - CSIParam.EnableVolumeReplicationSideCar = false - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_VOLUME_REPLICATION", "false"), "true") { - CSIParam.EnableVolumeReplicationSideCar = true - } - CSIParam.EnableCSIAddonsSideCar = false if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_CSIADDONS", "false"), "true") { CSIParam.EnableCSIAddonsSideCar = true @@ -254,7 +249,6 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error { CSIParam.AttacherImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ATTACHER_IMAGE", DefaultAttacherImage) CSIParam.SnapshotterImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_SNAPSHOTTER_IMAGE", DefaultSnapshotterImage) CSIParam.KubeletDirPath = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_KUBELET_DIR_PATH", DefaultKubeletDirPath) - CSIParam.VolumeReplicationImage = k8sutil.GetValue(r.opConfig.Parameters, "CSI_VOLUME_REPLICATION_IMAGE", DefaultVolumeReplicationImage) CSIParam.CSIAddonsImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSIADDONS_IMAGE", DefaultCSIAddonsImage) csiCephFSPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_CEPHFS_POD_LABELS", "") CSIParam.CSICephFSPodLabels = k8sutil.ParseStringToLabels(csiCephFSPodLabels) diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 3b9a478ef..fb99b0964 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -39,48 +39,47 @@ import ( ) type Param struct { - CSIPluginImage string - RegistrarImage string - ProvisionerImage string - AttacherImage string - SnapshotterImage string - ResizerImage string - DriverNamePrefix string - EnableCSIGRPCMetrics string - KubeletDirPath string - ForceCephFSKernelClient string - CephFSPluginUpdateStrategy string - NFSPluginUpdateStrategy string - RBDPluginUpdateStrategy string - PluginPriorityClassName string - ProvisionerPriorityClassName string - VolumeReplicationImage string - CSIAddonsImage string - CSIClusterName string - GRPCTimeout time.Duration - CSIEnableMetadata bool - EnablePluginSelinuxHostMount bool - EnableCSIHostNetwork bool - EnableOMAPGenerator bool - EnableRBDSnapshotter bool - EnableCephFSSnapshotter bool - EnableVolumeReplicationSideCar bool - EnableCSIAddonsSideCar bool - MountCustomCephConf bool - EnableOIDCTokenProjection bool - EnableCSIEncryption bool - EnableLiveness bool - LogLevel uint8 - SidecarLogLevel uint8 - CephFSGRPCMetricsPort uint16 - CephFSLivenessMetricsPort uint16 - RBDGRPCMetricsPort uint16 - CSIAddonsPort uint16 - RBDLivenessMetricsPort uint16 - ProvisionerReplicas int32 - CSICephFSPodLabels map[string]string - CSINFSPodLabels map[string]string - CSIRBDPodLabels map[string]string + CSIPluginImage string + RegistrarImage string + ProvisionerImage string + AttacherImage string + SnapshotterImage string + ResizerImage string + DriverNamePrefix string + EnableCSIGRPCMetrics string + KubeletDirPath string + ForceCephFSKernelClient string + CephFSPluginUpdateStrategy string + NFSPluginUpdateStrategy string + RBDPluginUpdateStrategy string + PluginPriorityClassName string + ProvisionerPriorityClassName string + VolumeReplicationImage string + CSIAddonsImage string + CSIClusterName string + GRPCTimeout time.Duration + CSIEnableMetadata bool + EnablePluginSelinuxHostMount bool + EnableCSIHostNetwork bool + EnableOMAPGenerator bool + EnableRBDSnapshotter bool + EnableCephFSSnapshotter bool + EnableCSIAddonsSideCar bool + MountCustomCephConf bool + EnableOIDCTokenProjection bool + EnableCSIEncryption bool + EnableLiveness bool + LogLevel uint8 + SidecarLogLevel uint8 + CephFSGRPCMetricsPort uint16 + CephFSLivenessMetricsPort uint16 + RBDGRPCMetricsPort uint16 + CSIAddonsPort uint16 + RBDLivenessMetricsPort uint16 + ProvisionerReplicas int32 + CSICephFSPodLabels map[string]string + CSINFSPodLabels map[string]string + CSIRBDPodLabels map[string]string } type templateParam struct { @@ -126,14 +125,13 @@ var ( // manually challenging. var ( // image names - DefaultCSIPluginImage = "quay.io/cephcsi/cephcsi:v3.7.0" - DefaultRegistrarImage = "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1" - DefaultProvisionerImage = "registry.k8s.io/sig-storage/csi-provisioner:v3.2.1" - DefaultAttacherImage = "registry.k8s.io/sig-storage/csi-attacher:v3.5.0" - DefaultSnapshotterImage = "registry.k8s.io/sig-storage/csi-snapshotter:v6.0.1" - DefaultResizerImage = "registry.k8s.io/sig-storage/csi-resizer:v1.5.0" - DefaultVolumeReplicationImage = "quay.io/csiaddons/volumereplication-operator:v0.3.0" - DefaultCSIAddonsImage = "quay.io/csiaddons/k8s-sidecar:v0.4.0" + DefaultCSIPluginImage = "quay.io/cephcsi/cephcsi:v3.7.0" + DefaultRegistrarImage = "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1" + DefaultProvisionerImage = "registry.k8s.io/sig-storage/csi-provisioner:v3.2.1" + DefaultAttacherImage = "registry.k8s.io/sig-storage/csi-attacher:v3.5.0" + DefaultSnapshotterImage = "registry.k8s.io/sig-storage/csi-snapshotter:v6.0.1" + DefaultResizerImage = "registry.k8s.io/sig-storage/csi-resizer:v1.5.0" + DefaultCSIAddonsImage = "quay.io/csiaddons/k8s-sidecar:v0.5.0" // Local package template path for RBD //go:embed template/rbd/csi-rbdplugin.yaml diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 243f5a7cb..1680ee2b3 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -121,25 +121,6 @@ spec: - name: keys-tmp-dir mountPath: /tmp/csi/keys {{ end }} - {{ if .EnableVolumeReplicationSideCar }} - - name: volume-replication - image: {{ .VolumeReplicationImage }} - args : - - "--metrics-bind-address=0" - - "--leader-election-namespace={{ .Namespace }}" - - "--driver-name={{ .DriverNamePrefix }}rbd.csi.ceph.com" - - "--csi-address=$(ADDRESS)" - - "--rpc-timeout={{ .GRPCTimeout }}" - - "--health-probe-bind-address=:9998" - - "--leader-elect=true" - env: - - name: ADDRESS - value: unix:///csi/csi-provisioner.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /csi - {{ end }} {{ if .EnableCSIAddonsSideCar }} - name: csi-addons image: {{ .CSIAddonsImage }} diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 11c4441d8..ca37d7d0c 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -60,7 +60,7 @@ const ( osd_pool_default_size = 1 bdev_flock_retry = 20 ` - volumeReplicationVersion = "v0.3.0" + volumeReplicationVersion = "v0.5.0" ) var ( @@ -69,7 +69,7 @@ var ( QuincyVersion = cephv1.CephVersionSpec{Image: quincyTestImage} QuincyDevelVersion = cephv1.CephVersionSpec{Image: quincyDevelTestImage} MasterVersion = cephv1.CephVersionSpec{Image: masterTestImage, AllowUnsupported: true} - volumeReplicationBaseURL = fmt.Sprintf("https://raw.githubusercontent.com/csi-addons/volume-replication-operator/%s/config/crd/bases/", volumeReplicationVersion) + volumeReplicationBaseURL = fmt.Sprintf("https://raw.githubusercontent.com/csi-addons/kubernetes-csi-addons/%s/config/crd/bases/", volumeReplicationVersion) volumeReplicationCRDURL = volumeReplicationBaseURL + "replication.storage.openshift.io_volumereplications.yaml" volumeReplicationClassCRDURL = volumeReplicationBaseURL + "replication.storage.openshift.io_volumereplicationclasses.yaml" ) From cc4fd0c5472ce1f1a38aa4bab67661c5293d7871 Mon Sep 17 00:00:00 2001 From: yati1998 Date: Fri, 2 Sep 2022 12:16:04 +0530 Subject: [PATCH 2/3] csi: remove controller-ip from csi-addons This commit removes controller-ip parameter from csi-addons sidecar as it is no longer required to be passed. Signed-off-by: yati1998 --- .../ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml | 5 ----- pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml | 5 ----- 2 files changed, 10 deletions(-) diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 1680ee2b3..0d66bb89e 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -128,7 +128,6 @@ spec: - "--node-id=$(NODE_ID)" - "--v={{ .LogLevel }}" - "--csi-addons-address=$(CSIADDONS_ENDPOINT)" - - "--controller-ip=$(POD_IP)" - "--controller-port={{ .CSIAddonsPort }}" - "--pod=$(POD_NAME)" - "--namespace=$(POD_NAMESPACE)" @@ -137,10 +136,6 @@ spec: ports: - containerPort: {{ .CSIAddonsPort }} env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - name: NODE_ID valueFrom: fieldRef: diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml index c387abba2..d66e025ef 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -141,7 +141,6 @@ spec: - "--node-id=$(NODE_ID)" - "--v={{ .LogLevel }}" - "--csi-addons-address=$(CSIADDONS_ENDPOINT)" - - "--controller-ip=$(POD_IP)" - "--controller-port={{ .CSIAddonsPort }}" - "--pod=$(POD_NAME)" - "--namespace=$(POD_NAMESPACE)" @@ -150,10 +149,6 @@ spec: ports: - containerPort: {{ .CSIAddonsPort }} env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - name: NODE_ID valueFrom: fieldRef: From 69a233be4e3d264ac93bae01d360f785870ee868 Mon Sep 17 00:00:00 2001 From: yati1998 Date: Wed, 7 Sep 2022 12:36:09 +0530 Subject: [PATCH 3/3] build: remove volume replication crd from build This commit removes the creation of volume replication crds from build. Signed-off-by: yati1998 --- build/crds/build-crds.sh | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/build/crds/build-crds.sh b/build/crds/build-crds.sh index 6eb31b1dc..750914257 100755 --- a/build/crds/build-crds.sh +++ b/build/crds/build-crds.sh @@ -51,21 +51,6 @@ copy_ob_obc_crds() { cp -f "${SCRIPT_ROOT}/deploy/olm/assemble/objectbucket.io_objectbuckets.yaml" "$OLM_CATALOG_DIR" } -copy_volume_replication_crds() { - volume_replication_dir=${SCRIPT_ROOT}/.cache/crds/volumereplication - volume_replication_crd=replication.storage.openshift.io_volumereplications.yaml - volume_replication_class_crd=replication.storage.openshift.io_volumereplicationclasses.yaml - if [[ ! -d "${volume_replication_dir}" ]]; then - mkdir -p "${volume_replication_dir}" - VOLUME_REPLICATION_URL=${VOLUME_REPLICATION_URL:-https://raw.githubusercontent.com/csi-addons/volume-replication-operator/v0.3.0/config/crd/bases} - curl -L ${VOLUME_REPLICATION_URL}/${volume_replication_crd} -o ${volume_replication_dir}/${volume_replication_crd} - curl -L ${VOLUME_REPLICATION_URL}/${volume_replication_class_crd} -o ${volume_replication_dir}/${volume_replication_class_crd} - fi - - cp -f "${volume_replication_dir}/${volume_replication_crd}" "$OLM_CATALOG_DIR" - cp -f "${volume_replication_dir}/${volume_replication_class_crd}" "$OLM_CATALOG_DIR" -} - generating_crds_v1() { echo "Generating ceph crds" "$CONTROLLER_GEN_BIN_PATH" "$CRD_OPTIONS" paths="./pkg/apis/ceph.rook.io/v1" output:crd:artifacts:config="$OLM_CATALOG_DIR" @@ -126,7 +111,3 @@ echo "---" >> "$CEPH_CRDS_FILE_PATH" # yq doesn't output the first doc separator $YQ_BIN_PATH eval-all '.' "${CRD_FILES[@]}" >> "$CEPH_CRDS_FILE_PATH" build_helm_resources - -# copy the volume replication crds last so they won't be included in the helm chart or crds.yaml, -# we just want them to end up owned by the csv -copy_volume_replication_crds