diff --git a/Documentation/ceph-csi-drivers.md b/Documentation/ceph-csi-drivers.md index 21dd586a0..7a126133e 100644 --- a/Documentation/ceph-csi-drivers.md +++ b/Documentation/ceph-csi-drivers.md @@ -65,3 +65,12 @@ storageclass. Now expand the PVC by editing the PVC Once PVC is expanded on backend and same is reflected size is reflected on application mountpoint, the status capacity `pvc.status.capacity.storage` of PVC will be updated to new size. + +## RBD Mirroring + +To support RBD Mirroring, the [Volume Replication Operator](https://github.com/csi-addons/volume-replication-operator/blob/main/README.md) will be started in the RBD provisioner pod. +Volume Replication Operator is a kubernetes operator that provides common and reusable APIs for storage disaster recovery. It is based on [csi-addons/spec](https://github.com/csi-addons/spec) specification and can be used by any storage provider. +It follows controller pattern and provides extended APIs for storage disaster recovery. The extended APIs are provided via Custom Resource Definition (CRD). +To enable volume replication: +- For Helm deployments see the [helm settings](helm-operator.md#configuration). +- For non-Helm deployments set `CSI_ENABLE_VOLUME_REPLICATION: "true"` in the operator.yaml diff --git a/Documentation/helm-operator.md b/Documentation/helm-operator.md index 79b024261..1734878b7 100644 --- a/Documentation/helm-operator.md +++ b/Documentation/helm-operator.md @@ -134,6 +134,8 @@ The following tables lists the configurable parameters of the rook-operator char | `csi.attacher.image` | Kubernetes CSI Attacher image. | `k8s.gcr.io/sig-storage/csi-attacher:v3.0.0` | | `csi.cephfsPodLabels` | Labels to add to the CSI CephFS Pods. | | | `csi.rbdPodLabels` | Labels to add to the CSI RBD Pods. | | +| `csi.volumeReplication.enabled` | Enable Volume Replication. | `false` | +| `csi.volumeReplication.image` | Volume Replication Controller image. | `quay.io/csiaddons/volumereplication-operator:v0.1.0` | | `agent.flexVolumeDirPath` | Path where the Rook agent discovers the flex volume plugins (*) | `/usr/libexec/kubernetes/kubelet-plugins/volume/exec/` | | `agent.libModulesDirPath` | Path where the Rook agent should look for kernel modules (*) | `/lib/modules` | | `agent.mounts` | Additional paths to be mounted in the agent container (**) | | diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 9c1f8d9c2..c2de057cb 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -36,3 +36,4 @@ v1.6... * Update CephCSI to v3.3.0 * Monitors failover can be [disabled](Documentation/ceph-mon-health.md#failing-over-a-monitor), this is useful for monitor going under planned maintenance where automatic failover is not desired * RGW: Rook has started using deployment's replicatset functionality instead of deploying multiple deployments for each rgw +* Support running Volume Replication Controller on the RBD provisioner pod. Depends on Cephcsi v3.3.0 diff --git a/cluster/charts/rook-ceph/templates/clusterrole.yaml b/cluster/charts/rook-ceph/templates/clusterrole.yaml index 52f4e9ee8..b52840489 100644 --- a/cluster/charts/rook-ceph/templates/clusterrole.yaml +++ b/cluster/charts/rook-ceph/templates/clusterrole.yaml @@ -404,6 +404,18 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplications", "volumereplicationclasses"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplications/finalizers"] + verbs: ["update"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplications/status"] + verbs: ["get", "patch", "update"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplicationclasses/status"] + verbs: ["get"] {{- end }} {{- if .Values.pspEnable }} --- diff --git a/cluster/charts/rook-ceph/templates/deployment.yaml b/cluster/charts/rook-ceph/templates/deployment.yaml index 023b1654e..29bf59637 100644 --- a/cluster/charts/rook-ceph/templates/deployment.yaml +++ b/cluster/charts/rook-ceph/templates/deployment.yaml @@ -120,6 +120,8 @@ spec: value: {{ .Values.csi.provisionerPriorityClassName | quote }} - name: CSI_ENABLE_OMAP_GENERATOR value: {{ .Values.csi.enableOMAPGenerator | quote }} + - name: CSI_ENABLE_VOLUME_REPLICATION + value: {{ .Values.csi.volumeReplication.enabled | quote }} {{- if .Values.csi.cephFSPluginUpdateStrategy }} - name: CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY value: {{ .Values.csi.cephFSPluginUpdateStrategy | quote }} @@ -178,6 +180,12 @@ spec: value: {{ .Values.csi.resizer.image | quote }} {{- end }} {{- end }} +{{- if .Values.csi.volumeReplication }} +{{- if .Values.csi.volumeReplication.image }} + - name: CSI_VOLUME_REPLICATION_IMAGE + value: {{ .Values.csi.volumeReplication.image | quote }} +{{- end }} +{{- end }} {{- if .Values.csi.cephfsPodLabels }} - name: ROOK_CSI_CEPHFS_POD_LABELS value: {{ .Values.csi.cephfsPodLabels | quote }} diff --git a/cluster/charts/rook-ceph/values.yaml b/cluster/charts/rook-ceph/values.yaml index 0b0764ab6..a5f5fecec 100644 --- a/cluster/charts/rook-ceph/values.yaml +++ b/cluster/charts/rook-ceph/values.yaml @@ -279,6 +279,10 @@ csi: #cephfsPodLabels: "key1=value1,key2=value2" # Labels to add to the CSI RBD Deployments and DaemonSets Pods. #rbdPodLabels: "key1=value1,key2=value2" + # Enable volume replication controller + volumeReplication: + enabled: false + #image: "quay.io/csiaddons/volumereplication-operator:v0.1.0" enableFlexDriver: false enableDiscoveryDaemon: false diff --git a/cluster/examples/kubernetes/ceph/common.yaml b/cluster/examples/kubernetes/ceph/common.yaml index 66e23ceb4..9142378a9 100644 --- a/cluster/examples/kubernetes/ceph/common.yaml +++ b/cluster/examples/kubernetes/ceph/common.yaml @@ -1082,6 +1082,18 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplications", "volumereplicationclasses"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplications/finalizers"] + verbs: ["update"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplications/status"] + verbs: ["get", "patch", "update"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplicationclasses/status"] + verbs: ["get"] # OLM: END CSI RBD CLUSTER ROLE # OLM: BEGIN CSI RBD CLUSTER ROLEBINDING --- diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index ecf2c284a..df35b8f1b 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -108,6 +108,25 @@ spec: - name: keys-tmp-dir mountPath: /tmp/csi/keys {{ end }} + {{ if .EnableVolumeReplicationSideCar }} + - name: volume-replication + image: {{ .VolumeReplicationImage }} + args : + - "--metrics-bind-address=0" + - "--leader-election-namespace={{ .Namespace }}" + - "--driver-name={{ .DriverNamePrefix }}rbd.csi.ceph.com" + - "--csi-address=$(ADDRESS)" + - "--rpc-timeout=150s" + - "--health-probe-bind-address=:9998" + - "--leader-elect=true" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + {{ end }} - name: csi-rbdplugin image: {{ .CSIPluginImage }} args : diff --git a/cluster/examples/kubernetes/ceph/operator-openshift.yaml b/cluster/examples/kubernetes/ceph/operator-openshift.yaml index 6d6205a5c..49456446d 100644 --- a/cluster/examples/kubernetes/ceph/operator-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/operator-openshift.yaml @@ -356,6 +356,9 @@ data: # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster. # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs. ROOK_ENABLE_DISCOVERY_DAEMON: "false" + # Enable volume replication controller + CSI_ENABLE_VOLUME_REPLICATION: "false" + # CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0" # (Optional) Admission controller NodeAffinity. # ADMISSION_CONTROLLER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index 35bdeb7b8..cf2933580 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -280,6 +280,9 @@ data: # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster. # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs. ROOK_ENABLE_DISCOVERY_DAEMON: "false" + # Enable volume replication controller + CSI_ENABLE_VOLUME_REPLICATION: "false" + # CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0" # (Optional) Admission controller NodeAffinity. # ADMISSION_CONTROLLER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index 99c83a125..b0364a171 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -108,6 +108,10 @@ func SetParams(clientset kubernetes.Interface) error { if err != nil { return errors.Wrap(err, "unable to configure CSI kubelet directory path") } + CSIParam.VolumeReplicationImage, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_VOLUME_REPLICATION_IMAGE", DefaultVolumeReplicationImage) + if err != nil { + return errors.Wrap(err, "unable to configure Volume replication controller image") + } csiCephFSPodLabels, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_CEPHFS_POD_LABELS", "") if err != nil { diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 6e3fa1de5..19bf420dd 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -40,31 +40,33 @@ import ( ) type Param struct { - CSIPluginImage string - RegistrarImage string - ProvisionerImage string - AttacherImage string - SnapshotterImage string - ResizerImage string - DriverNamePrefix string - EnableCSIGRPCMetrics string - KubeletDirPath string - ForceCephFSKernelClient string - CephFSPluginUpdateStrategy string - RBDPluginUpdateStrategy string - PluginPriorityClassName string - ProvisionerPriorityClassName string - EnableOMAPGenerator bool - EnableRBDSnapshotter bool - EnableCephFSSnapshotter bool - LogLevel uint8 - CephFSGRPCMetricsPort uint16 - CephFSLivenessMetricsPort uint16 - RBDGRPCMetricsPort uint16 - RBDLivenessMetricsPort uint16 - ProvisionerReplicas uint8 - CSICephFSPodLabels map[string]string - CSIRBDPodLabels map[string]string + CSIPluginImage string + RegistrarImage string + ProvisionerImage string + AttacherImage string + SnapshotterImage string + ResizerImage string + DriverNamePrefix string + EnableCSIGRPCMetrics string + KubeletDirPath string + ForceCephFSKernelClient string + CephFSPluginUpdateStrategy string + RBDPluginUpdateStrategy string + PluginPriorityClassName string + ProvisionerPriorityClassName string + VolumeReplicationImage string + EnableOMAPGenerator bool + EnableRBDSnapshotter bool + EnableCephFSSnapshotter bool + EnableVolumeReplicationSideCar bool + LogLevel uint8 + CephFSGRPCMetricsPort uint16 + CephFSLivenessMetricsPort uint16 + RBDGRPCMetricsPort uint16 + RBDLivenessMetricsPort uint16 + ProvisionerReplicas uint8 + CSICephFSPodLabels map[string]string + CSIRBDPodLabels map[string]string } type templateParam struct { @@ -105,12 +107,13 @@ var ( // manually challenging. var ( // image names - DefaultCSIPluginImage = "quay.io/cephcsi/cephcsi:v3.3.0" - DefaultRegistrarImage = "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1" - DefaultProvisionerImage = "k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4" - DefaultAttacherImage = "k8s.gcr.io/sig-storage/csi-attacher:v3.0.2" - DefaultSnapshotterImage = "k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2" - DefaultResizerImage = "k8s.gcr.io/sig-storage/csi-resizer:v1.0.1" + DefaultCSIPluginImage = "quay.io/cephcsi/cephcsi:v3.3.0" + DefaultRegistrarImage = "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1" + DefaultProvisionerImage = "k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4" + DefaultAttacherImage = "k8s.gcr.io/sig-storage/csi-attacher:v3.0.2" + DefaultSnapshotterImage = "k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2" + DefaultResizerImage = "k8s.gcr.io/sig-storage/csi-resizer:v1.0.1" + DefaultVolumeReplicationImage = "quay.io/csiaddons/volumereplication-operator:v0.1.0" ) const ( @@ -305,6 +308,15 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter tp.EnableCephFSSnapshotter = false } + tp.EnableVolumeReplicationSideCar = false + enableVolumeReplicationSideCar, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_ENABLE_VOLUME_REPLICATION", "false") + if err != nil { + return errors.Wrap(err, "failed to load CSI_ENABLE_VOLUME_REPLICATION setting") + } + if strings.EqualFold(enableVolumeReplicationSideCar, "true") { + tp.EnableVolumeReplicationSideCar = true + } + updateStrategy, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate) if err != nil { return errors.Wrap(err, "failed to load CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY setting") diff --git a/tests/framework/installer/ceph_settings.go b/tests/framework/installer/ceph_settings.go index 5fdbd9fc6..a483d9438 100644 --- a/tests/framework/installer/ceph_settings.go +++ b/tests/framework/installer/ceph_settings.go @@ -49,6 +49,7 @@ type TestCephSettings struct { SkipClusterCleanup bool SkipCleanupPolicy bool DirectMountToolbox bool + EnableVolumeReplication bool RookVersion string CephVersion cephv1.CephVersionSpec } @@ -106,6 +107,7 @@ func (s *TestCephSettings) replaceOperatorSettings(manifest string) string { manifest = strings.ReplaceAll(manifest, `ROOK_ENABLE_FLEX_DRIVER: "false"`, fmt.Sprintf(`ROOK_ENABLE_FLEX_DRIVER: "%t"`, !s.UseCSI)) manifest = strings.ReplaceAll(manifest, `ROOK_CSI_ENABLE_CEPHFS: "true"`, fmt.Sprintf(`ROOK_CSI_ENABLE_CEPHFS: "%t"`, s.UseCSI)) manifest = strings.ReplaceAll(manifest, `ROOK_CSI_ENABLE_RBD: "true"`, fmt.Sprintf(`ROOK_CSI_ENABLE_RBD: "%t"`, s.UseCSI)) + manifest = strings.ReplaceAll(manifest, `CSI_ENABLE_VOLUME_REPLICATION: "false"`, fmt.Sprintf(`CSI_ENABLE_VOLUME_REPLICATION: "%t"`, s.EnableVolumeReplication)) return manifest } diff --git a/tests/integration/ceph_smoke_test.go b/tests/integration/ceph_smoke_test.go index 086dfc459..e10a247f2 100644 --- a/tests/integration/ceph_smoke_test.go +++ b/tests/integration/ceph_smoke_test.go @@ -99,6 +99,7 @@ func (s *SmokeSuite) SetupSuite() { UseCSI: true, EnableAdmissionController: true, UseCrashPruner: true, + EnableVolumeReplication: true, RookVersion: installer.VersionMaster, CephVersion: installer.PacificVersion, }