feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

ceph: adds a container to support volume replication controller

Browse Source 
this commit adds a new container inside of rbd-provisioner.

Signed-off-by: bipuladh <badhikar@redhat.com>
This commit is contained in:
bipuladh committed 2021-04-15 21:39:26 +05:30
1 parent 78bfcacacd
commit 7d5cc8d06b
14 files changed
+123 -31
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Documentation/ceph-csi-drivers.md
+9
View File
@@ -65,3 +65,12 @@ storageclass. Now expand the PVC by editing the PVC
Once PVC is expanded on backend and same is reflected size is reflected on
application mountpoint, the status capacity `pvc.status.capacity.storage` of
PVC will be updated to new size.
## RBD Mirroring
To support RBD Mirroring, the [Volume Replication Operator](https://github.com/csi-addons/volume-replication-operator/blob/main/README.md) will be started in the RBD provisioner pod.
Volume Replication Operator is a kubernetes operator that provides common and reusable APIs for storage disaster recovery. It is based on [csi-addons/spec](https://github.com/csi-addons/spec) specification and can be used by any storage provider.
It follows controller pattern and provides extended APIs for storage disaster recovery. The extended APIs are provided via Custom Resource Definition (CRD).
To enable volume replication:
- For Helm deployments see the [helm settings](helm-operator.md#configuration).
- For non-Helm deployments set `CSI_ENABLE_VOLUME_REPLICATION: "true"` in the operator.yaml
Documentation/helm-operator.md
+2
View File
@@ -134,6 +134,8 @@ The following tables lists the configurable parameters of the rook-operator char
| `csi.attacher.image` | Kubernetes CSI Attacher image. | `k8s.gcr.io/sig-storage/csi-attacher:v3.0.0` |
| `csi.cephfsPodLabels` | Labels to add to the CSI CephFS Pods. | <none> |
| `csi.rbdPodLabels` | Labels to add to the CSI RBD Pods. | <none> |
| `csi.volumeReplication.enabled` | Enable Volume Replication. | `false` |
| `csi.volumeReplication.image` | Volume Replication Controller image. | `quay.io/csiaddons/volumereplication-operator:v0.1.0` |
| `agent.flexVolumeDirPath` | Path where the Rook agent discovers the flex volume plugins (*) | `/usr/libexec/kubernetes/kubelet-plugins/volume/exec/` |
| `agent.libModulesDirPath` | Path where the Rook agent should look for kernel modules (*) | `/lib/modules` |
| `agent.mounts` | Additional paths to be mounted in the agent container (**) | <none> |
PendingReleaseNotes.md
+1
View File
@@ -36,3 +36,4 @@ v1.6...
* Update CephCSI to v3.3.0
* Monitors failover can be [disabled](Documentation/ceph-mon-health.md#failing-over-a-monitor), this is useful for monitor going under planned maintenance where automatic failover is not desired
* RGW: Rook has started using deployment's replicatset functionality instead of deploying multiple deployments for each rgw
* Support running Volume Replication Controller on the RBD provisioner pod. Depends on Cephcsi v3.3.0
cluster/charts/rook-ceph/templates/clusterrole.yaml
+12
View File
@@ -404,6 +404,18 @@ rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get"]
- apiGroups: ["replication.storage.openshift.io"]
resources: ["volumereplications", "volumereplicationclasses"]
verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
- apiGroups: ["replication.storage.openshift.io"]
resources: ["volumereplications/finalizers"]
verbs: ["update"]
- apiGroups: ["replication.storage.openshift.io"]
resources: ["volumereplications/status"]
verbs: ["get", "patch", "update"]
- apiGroups: ["replication.storage.openshift.io"]
resources: ["volumereplicationclasses/status"]
verbs: ["get"]
{{- end }}
{{- if .Values.pspEnable }}
---
cluster/charts/rook-ceph/templates/deployment.yaml
+8
View File
@@ -120,6 +120,8 @@ spec:
value: {{ .Values.csi.provisionerPriorityClassName | quote }}
- name: CSI_ENABLE_OMAP_GENERATOR
value: {{ .Values.csi.enableOMAPGenerator | quote }}
- name: CSI_ENABLE_VOLUME_REPLICATION
value: {{ .Values.csi.volumeReplication.enabled | quote }}
{{- if .Values.csi.cephFSPluginUpdateStrategy }}
- name: CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY
value: {{ .Values.csi.cephFSPluginUpdateStrategy | quote }}
@@ -178,6 +180,12 @@ spec:
value: {{ .Values.csi.resizer.image | quote }}
{{- end }}
{{- end }}
{{- if .Values.csi.volumeReplication }}
{{- if .Values.csi.volumeReplication.image }}
- name: CSI_VOLUME_REPLICATION_IMAGE
value: {{ .Values.csi.volumeReplication.image | quote }}
{{- end }}
{{- end }}
{{- if .Values.csi.cephfsPodLabels }}
- name: ROOK_CSI_CEPHFS_POD_LABELS
value: {{ .Values.csi.cephfsPodLabels | quote }}
cluster/charts/rook-ceph/values.yaml
+4
View File
@@ -279,6 +279,10 @@ csi:
#cephfsPodLabels: "key1=value1,key2=value2"
# Labels to add to the CSI RBD Deployments and DaemonSets Pods.
#rbdPodLabels: "key1=value1,key2=value2"
# Enable volume replication controller
volumeReplication:
enabled: false
#image: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
enableFlexDriver: false
enableDiscoveryDaemon: false
cluster/examples/kubernetes/ceph/common.yaml
+12
View File
@@ -1082,6 +1082,18 @@ rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get"]
- apiGroups: ["replication.storage.openshift.io"]
resources: ["volumereplications", "volumereplicationclasses"]
verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
- apiGroups: ["replication.storage.openshift.io"]
resources: ["volumereplications/finalizers"]
verbs: ["update"]
- apiGroups: ["replication.storage.openshift.io"]
resources: ["volumereplications/status"]
verbs: ["get", "patch", "update"]
- apiGroups: ["replication.storage.openshift.io"]
resources: ["volumereplicationclasses/status"]
verbs: ["get"]
# OLM: END CSI RBD CLUSTER ROLE
# OLM: BEGIN CSI RBD CLUSTER ROLEBINDING
---
cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
+19
View File
@@ -108,6 +108,25 @@ spec:
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
{{ end }}
{{ if .EnableVolumeReplicationSideCar }}
- name: volume-replication
image: {{ .VolumeReplicationImage }}
args :
- "--metrics-bind-address=0"
- "--leader-election-namespace={{ .Namespace }}"
- "--driver-name={{ .DriverNamePrefix }}rbd.csi.ceph.com"
- "--csi-address=$(ADDRESS)"
- "--rpc-timeout=150s"
- "--health-probe-bind-address=:9998"
- "--leader-elect=true"
env:
- name: ADDRESS
value: unix:///csi/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /csi
{{ end }}
- name: csi-rbdplugin
image: {{ .CSIPluginImage }}
args :
cluster/examples/kubernetes/ceph/operator-openshift.yaml
+3
View File
@@ -356,6 +356,9 @@ data:
# Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster.
# This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs.
ROOK_ENABLE_DISCOVERY_DAEMON: "false"
# Enable volume replication controller
CSI_ENABLE_VOLUME_REPLICATION: "false"
# CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
# (Optional) Admission controller NodeAffinity.
# ADMISSION_CONTROLLER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
cluster/examples/kubernetes/ceph/operator.yaml
+3
View File
@@ -280,6 +280,9 @@ data:
# Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster.
# This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs.
ROOK_ENABLE_DISCOVERY_DAEMON: "false"
# Enable volume replication controller
CSI_ENABLE_VOLUME_REPLICATION: "false"
# CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
# (Optional) Admission controller NodeAffinity.
# ADMISSION_CONTROLLER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
pkg/operator/ceph/csi/csi.go
+4
View File
@@ -108,6 +108,10 @@ func SetParams(clientset kubernetes.Interface) error {
if err != nil {
return errors.Wrap(err, "unable to configure CSI kubelet directory path")
}
CSIParam.VolumeReplicationImage, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_VOLUME_REPLICATION_IMAGE", DefaultVolumeReplicationImage)
if err != nil {
return errors.Wrap(err, "unable to configure Volume replication controller image")
}
csiCephFSPodLabels, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_CEPHFS_POD_LABELS", "")
if err != nil {
pkg/operator/ceph/csi/spec.go
+43 -31
View File
@@ -40,31 +40,33 @@ import (
)
type Param struct {
CSIPluginImage string
RegistrarImage string
ProvisionerImage string
AttacherImage string
SnapshotterImage string
ResizerImage string
DriverNamePrefix string
EnableCSIGRPCMetrics string
KubeletDirPath string
ForceCephFSKernelClient string
CephFSPluginUpdateStrategy string
RBDPluginUpdateStrategy string
PluginPriorityClassName string
ProvisionerPriorityClassName string
EnableOMAPGenerator bool
EnableRBDSnapshotter bool
EnableCephFSSnapshotter bool
LogLevel uint8
CephFSGRPCMetricsPort uint16
CephFSLivenessMetricsPort uint16
RBDGRPCMetricsPort uint16
RBDLivenessMetricsPort uint16
ProvisionerReplicas uint8
CSICephFSPodLabels map[string]string
CSIRBDPodLabels map[string]string
CSIPluginImage string
RegistrarImage string
ProvisionerImage string
AttacherImage string
SnapshotterImage string
ResizerImage string
DriverNamePrefix string
EnableCSIGRPCMetrics string
KubeletDirPath string
ForceCephFSKernelClient string
CephFSPluginUpdateStrategy string
RBDPluginUpdateStrategy string
PluginPriorityClassName string
ProvisionerPriorityClassName string
VolumeReplicationImage string
EnableOMAPGenerator bool
EnableRBDSnapshotter bool
EnableCephFSSnapshotter bool
EnableVolumeReplicationSideCar bool
LogLevel uint8
CephFSGRPCMetricsPort uint16
CephFSLivenessMetricsPort uint16
RBDGRPCMetricsPort uint16
RBDLivenessMetricsPort uint16
ProvisionerReplicas uint8
CSICephFSPodLabels map[string]string
CSIRBDPodLabels map[string]string
}
type templateParam struct {
@@ -105,12 +107,13 @@ var (
// manually challenging.
var (
// image names
DefaultCSIPluginImage = "quay.io/cephcsi/cephcsi:v3.3.0"
DefaultRegistrarImage = "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1"
DefaultProvisionerImage = "k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4"
DefaultAttacherImage = "k8s.gcr.io/sig-storage/csi-attacher:v3.0.2"
DefaultSnapshotterImage = "k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2"
DefaultResizerImage = "k8s.gcr.io/sig-storage/csi-resizer:v1.0.1"
DefaultCSIPluginImage = "quay.io/cephcsi/cephcsi:v3.3.0"
DefaultRegistrarImage = "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1"
DefaultProvisionerImage = "k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4"
DefaultAttacherImage = "k8s.gcr.io/sig-storage/csi-attacher:v3.0.2"
DefaultSnapshotterImage = "k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2"
DefaultResizerImage = "k8s.gcr.io/sig-storage/csi-resizer:v1.0.1"
DefaultVolumeReplicationImage = "quay.io/csiaddons/volumereplication-operator:v0.1.0"
)
const (
@@ -305,6 +308,15 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
tp.EnableCephFSSnapshotter = false
}
tp.EnableVolumeReplicationSideCar = false
enableVolumeReplicationSideCar, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_ENABLE_VOLUME_REPLICATION", "false")
if err != nil {
return errors.Wrap(err, "failed to load CSI_ENABLE_VOLUME_REPLICATION setting")
}
if strings.EqualFold(enableVolumeReplicationSideCar, "true") {
tp.EnableVolumeReplicationSideCar = true
}
updateStrategy, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate)
if err != nil {
return errors.Wrap(err, "failed to load CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY setting")
tests/framework/installer/ceph_settings.go
+2
View File
@@ -49,6 +49,7 @@ type TestCephSettings struct {
SkipClusterCleanup bool
SkipCleanupPolicy bool
DirectMountToolbox bool
EnableVolumeReplication bool
RookVersion string
CephVersion cephv1.CephVersionSpec
}
@@ -106,6 +107,7 @@ func (s *TestCephSettings) replaceOperatorSettings(manifest string) string {
manifest = strings.ReplaceAll(manifest, `ROOK_ENABLE_FLEX_DRIVER: "false"`, fmt.Sprintf(`ROOK_ENABLE_FLEX_DRIVER: "%t"`, !s.UseCSI))
manifest = strings.ReplaceAll(manifest, `ROOK_CSI_ENABLE_CEPHFS: "true"`, fmt.Sprintf(`ROOK_CSI_ENABLE_CEPHFS: "%t"`, s.UseCSI))
manifest = strings.ReplaceAll(manifest, `ROOK_CSI_ENABLE_RBD: "true"`, fmt.Sprintf(`ROOK_CSI_ENABLE_RBD: "%t"`, s.UseCSI))
manifest = strings.ReplaceAll(manifest, `CSI_ENABLE_VOLUME_REPLICATION: "false"`, fmt.Sprintf(`CSI_ENABLE_VOLUME_REPLICATION: "%t"`, s.EnableVolumeReplication))
return manifest
}
tests/integration/ceph_smoke_test.go
+1
View File
@@ -99,6 +99,7 @@ func (s *SmokeSuite) SetupSuite() {
UseCSI: true,
EnableAdmissionController: true,
UseCrashPruner: true,
EnableVolumeReplication: true,
RookVersion: installer.VersionMaster,
CephVersion: installer.PacificVersion,
}