diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md index 8ec872db7..4ed28dad6 100644 --- a/Documentation/Helm-Charts/operator-chart.md +++ b/Documentation/Helm-Charts/operator-chart.md @@ -133,6 +133,10 @@ The following tables lists the configurable parameters of the rook-operator char | `admissionController.tolerations` | Array of tolerations in YAML format which will be added to admission controller deployment. | `` | | `admissionController.nodeAffinity` | The node labels for affinity of the admission controller deployment [^1] | `` | | `monitoring.enabled` | Create necessary RBAC rules for Rook to integrate with Prometheus monitoring in the operator namespace. Requires Prometheus to be pre-installed. | `false` | +| `csi.csiRBDPluginVolume` | The volume of the CephCSI RBD plugin DaemonSet | `` | +| `csi.csiRBDPluginVolumeMount` | The volume mounts of the CephCSI RBD plugin DaemonSet | `` | +| `csi.csiCephFSPluginVolume` | The volume of the CephCSI CephFS plugin DaemonSet | `` | +| `csi.csiCephFSPluginVolumeMount` | The volume mounts of the CephCSI CephFS plugin DaemonSet | `` | [^1]: `nodeAffinity` and `*NodeAffinity` options should have the format `"role=storage,rook; storage=ceph"` or `storage=;role=rook-example` or `storage=;` (_checks only for presence of key_) diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml index 27b09e723..6a004b026 100644 --- a/deploy/charts/rook-ceph/templates/configmap.yaml +++ b/deploy/charts/rook-ceph/templates/configmap.yaml @@ -203,4 +203,16 @@ data: {{- if .Values.csi.csiNFSPluginResource }} CSI_NFS_PLUGIN_RESOURCE: {{ .Values.csi.csiNFSPluginResource | quote }} {{- end }} +{{- if .Values.csi.csiRBDPluginVolume }} + CSI_RBD_PLUGIN_VOLUME: {{ toYaml .Values.csi.csiRBDPluginVolume | quote }} +{{- end }} +{{- if .Values.csi.csiRBDPluginVolumeMount }} + CSI_RBD_PLUGIN_VOLUME_MOUNT: {{ toYaml .Values.csi.csiRBDPluginVolumeMount | quote }} +{{- end }} +{{- if .Values.csi.csiCephFSPluginVolume }} + CSI_CEPHFS_PLUGIN_VOLUME: {{ toYaml .Values.csi.csiRBDPluginVolumes | quote }} +{{- end }} +{{- if .Values.csi.csiCephFSPluginVolumeMount }} + CSI_CEPHFS_PLUGIN_VOLUME_MOUNT: {{ toYaml .Values.csi.csiCephFSPluginVolumeMount | quote }} +{{- end }} {{- end }} diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index 5863bae28..3c7ecaece 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -131,6 +131,37 @@ csi: # Allow starting unsupported ceph-csi image allowUnsupportedVersion: false + + # CephCSI RBD plugin Volumes + # csiRBDPluginVolumes: | + # - name: lib-modules + # hostPath: + # path: /run/current-system/kernel-modules/lib/modules/ + # - name: host-nix + # hostPath: + # path: /nix + + # CephCSI RBD plugin Volume mounts + # csiRBDPluginVolumeMounts: | + # - name: host-nix + # mountPath: /nix + # readOnly: true + + # CephCSI CephFS plugin Volumes + # csiCephFSPluginVolumes: | + # - name: lib-modules + # hostPath: + # path: /run/current-system/kernel-modules/lib/modules/ + # - name: host-nix + # hostPath: + # path: /nix + + # CephCSI CephFS plugin Volume mounts + # csiCephFSPluginVolumeMounts: | + # - name: host-nix + # mountPath: /nix + # readOnly: true + # CEPH CSI RBD provisioner resource requirement list, Put here list of resource # requests and limits you want to apply for provisioner pod # csi-omap-generator resources will be applied only if enableOMAPGenerator is set to true diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index fb01447cf..d939d19b0 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -221,6 +221,36 @@ data: # Labels to add to the CSI NFS Deployments and DaemonSets Pods. # ROOK_CSI_NFS_POD_LABELS: "key1=value1,key2=value2" + # (Optional) CephCSI CephFS plugin Volumes + # CSI_CEPHFS_PLUGIN_VOLUME: | + # - name: lib-modules + # hostPath: + # path: /run/current-system/kernel-modules/lib/modules/ + # - name: host-nix + # hostPath: + # path: /nix + + # (Optional) CephCSI CephFS plugin Volume mounts + # CSI_CEPHFS_PLUGIN_VOLUME_MOUNT: | + # - name: host-nix + # mountPath: /nix + # readOnly: true + + # (Optional) CephCSI RBD plugin Volumes + # CSI_RBD_PLUGIN_VOLUME: | + # - name: lib-modules + # hostPath: + # path: /run/current-system/kernel-modules/lib/modules/ + # - name: host-nix + # hostPath: + # path: /nix + + # (Optional) CephCSI RBD plugin Volume mounts + # CSI_RBD_PLUGIN_VOLUME_MOUNT: | + # - name: host-nix + # mountPath: /nix + # readOnly: true + # (Optional) CephCSI provisioner NodeAffinity(applied to both CephFS and RBD provisioner). # CSI_PROVISIONER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" # (Optional) CephCSI provisioner tolerations list(applied to both CephFS and RBD provisioner). diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index cb69e7f03..356d382c3 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -138,6 +138,36 @@ data: # Labels to add to the CSI NFS Deployments and DaemonSets Pods. # ROOK_CSI_NFS_POD_LABELS: "key1=value1,key2=value2" + # (Optional) CephCSI CephFS plugin Volumes + # CSI_CEPHFS_PLUGIN_VOLUME: | + # - name: lib-modules + # hostPath: + # path: /run/current-system/kernel-modules/lib/modules/ + # - name: host-nix + # hostPath: + # path: /nix + + # (Optional) CephCSI CephFS plugin Volume mounts + # CSI_CEPHFS_PLUGIN_VOLUME_MOUNT: | + # - name: host-nix + # mountPath: /nix + # readOnly: true + + # (Optional) CephCSI RBD plugin Volumes + # CSI_RBD_PLUGIN_VOLUME: | + # - name: lib-modules + # hostPath: + # path: /run/current-system/kernel-modules/lib/modules/ + # - name: host-nix + # hostPath: + # path: /nix + + # (Optional) CephCSI RBD plugin Volume mounts + # CSI_RBD_PLUGIN_VOLUME_MOUNT: | + # - name: host-nix + # mountPath: /nix + # readOnly: true + # (Optional) CephCSI provisioner NodeAffinity (applied to both CephFS and RBD provisioner). # CSI_PROVISIONER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" # (Optional) CephCSI provisioner tolerations list(applied to both CephFS and RBD provisioner). diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 9416ff4d1..f76938021 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -204,6 +204,15 @@ const ( nfsProvisionerResource = "CSI_NFS_PROVISIONER_RESOURCE" nfsPluginResource = "CSI_NFS_PLUGIN_RESOURCE" + cephFSPluginVolume = "CSI_CEPHFS_PLUGIN_VOLUME" + cephFSPluginVolumeMount = "CSI_CEPHFS_PLUGIN_VOLUME_MOUNT" + + rbdPluginVolume = "CSI_RBD_PLUGIN_VOLUME" + rbdPluginVolumeMount = "CSI_RBD_PLUGIN_VOLUME_MOUNT" + + nfsPluginVolume = "CSI_NFS_PLUGIN_VOLUME" + nfsPluginVolumeMount = "CSI_NFS_PLUGIN_VOLUME_MOUNT" + // kubelet directory path DefaultKubeletDirPath = "/var/lib/kubelet" @@ -422,6 +431,10 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI applyToPodSpec(&rbdPlugin.Spec.Template.Spec, rbdPluginNodeAffinity, rbdPluginTolerations) // apply resource request and limit to rbdplugin containers applyResourcesToContainers(r.opConfig.Parameters, rbdPluginResource, &rbdPlugin.Spec.Template.Spec) + // apply custom mounts to volumes + applyVolumeToPodSpec(r.opConfig.Parameters, rbdPluginVolume, &rbdPlugin.Spec.Template.Spec) + // apply custom mounts to volume mounts + applyVolumeMountToContainer(r.opConfig.Parameters, rbdPluginVolumeMount, "csi-rbdplugin", &rbdPlugin.Spec.Template.Spec) err = ownerInfo.SetControllerReference(rbdPlugin) if err != nil { return errors.Wrapf(err, "failed to set owner reference to rbd plugin daemonset %q", rbdPlugin.Name) @@ -490,6 +503,10 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI applyToPodSpec(&cephfsPlugin.Spec.Template.Spec, cephFSPluginNodeAffinity, cephFSPluginTolerations) // apply resource request and limit to cephfs plugin containers applyResourcesToContainers(r.opConfig.Parameters, cephFSPluginResource, &cephfsPlugin.Spec.Template.Spec) + // apply custom mounts to volumes + applyVolumeToPodSpec(r.opConfig.Parameters, cephFSPluginVolume, &cephfsPlugin.Spec.Template.Spec) + // apply custom mounts to volume mounts + applyVolumeMountToContainer(r.opConfig.Parameters, cephFSPluginVolumeMount, "csi-cephfsplugin", &cephfsPlugin.Spec.Template.Spec) err = ownerInfo.SetControllerReference(cephfsPlugin) if err != nil { return errors.Wrapf(err, "failed to set owner reference to cephfs plugin daemonset %q", cephfsPlugin.Name) @@ -561,6 +578,10 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI applyToPodSpec(&nfsPlugin.Spec.Template.Spec, nfsPluginNodeAffinity, nfsPluginTolerations) // apply resource request and limit to nfs plugin containers applyResourcesToContainers(r.opConfig.Parameters, nfsPluginResource, &nfsPlugin.Spec.Template.Spec) + // apply custom mounts to volumes + applyVolumeToPodSpec(r.opConfig.Parameters, nfsPluginVolume, &nfsPlugin.Spec.Template.Spec) + // apply custom mounts to volume mounts + applyVolumeMountToContainer(r.opConfig.Parameters, nfsPluginVolumeMount, "csi-nfsplugin", &nfsPlugin.Spec.Template.Spec) err = ownerInfo.SetControllerReference(nfsPlugin) if err != nil { return errors.Wrapf(err, "failed to set owner reference to nfs plugin daemonset %q", nfsPlugin.Name) diff --git a/pkg/operator/ceph/csi/util.go b/pkg/operator/ceph/csi/util.go index 44d2b42d0..0066ec996 100644 --- a/pkg/operator/ceph/csi/util.go +++ b/pkg/operator/ceph/csi/util.go @@ -188,3 +188,67 @@ func GetPodAntiAffinity(key, value string) corev1.PodAntiAffinity { }, } } + +func applyVolumeToPodSpec(opConfig map[string]string, configName string, podspec *corev1.PodSpec) { + volumesRaw := k8sutil.GetValue(opConfig, configName, "") + if volumesRaw == "" { + return + } + volumes, err := k8sutil.YamlToVolumes(volumesRaw) + if err != nil { + logger.Warningf("failed to parse %q for %q. %v", volumesRaw, configName, err) + return + } + if len(volumes) > 0 { + for i := range volumes { + found := false + for j := range podspec.Volumes { + // check do we need to override any existing volumes + if volumes[i].Name == podspec.Volumes[j].Name { + podspec.Volumes[j] = volumes[i] + found = true + break + } + } + if !found { + // if not found add volume to volumes list + podspec.Volumes = append(podspec.Volumes, volumes[i]) + } + } + } +} + +func applyVolumeMountToContainer(opConfig map[string]string, configName, containerName string, podspec *corev1.PodSpec) { + volumeMountsRaw := k8sutil.GetValue(opConfig, configName, "") + if volumeMountsRaw == "" { + return + } + volumeMounts, err := k8sutil.YamlToVolumeMounts(volumeMountsRaw) + if err != nil { + logger.Warningf("failed to parse %q for %q. %v", volumeMountsRaw, configName, err) + return + } + if len(volumeMounts) > 0 { + for i, c := range podspec.Containers { + if c.Name == containerName { + for j := range volumeMounts { + found := false + for k := range podspec.Containers[i].VolumeMounts { + // override if the name is matching + if volumeMounts[j].Name == podspec.Containers[i].VolumeMounts[k].Name { + found = true + podspec.Containers[i].VolumeMounts[k] = volumeMounts[j] + break + } + } + if !found { + // if not found append it to the exiting volumes + podspec.Containers[i].VolumeMounts = append(podspec.Containers[i].VolumeMounts, volumeMounts[j]) + } + } + // return as we finished with found container + return + } + } + } +} diff --git a/pkg/operator/ceph/csi/util_test.go b/pkg/operator/ceph/csi/util_test.go index 707852bf6..241e0fdfe 100644 --- a/pkg/operator/ceph/csi/util_test.go +++ b/pkg/operator/ceph/csi/util_test.go @@ -22,6 +22,8 @@ import ( "github.com/stretchr/testify/assert" "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" ) func TestDaemonSetTemplate(t *testing.T) { @@ -109,3 +111,143 @@ func TestApplyingResourcesToRBDPlugin(t *testing.T) { assert.Equal(t, rbdPlugin.Spec.Template.Spec.Containers[0].Resources.Requests.Cpu().String(), "100m") assert.Equal(t, rbdPlugin.Spec.Template.Spec.Containers[0].Resources.Limits.Cpu().String(), "200m") } + +func Test_applyVolumeToPodSpec(t *testing.T) { + // when no volumes specified + config := make(map[string]string) + configKey := "TEST_CSI_PLUGIN_VOLUME" + dsName := "test-ds" + tp := templateParam{ + Param: CSIParam, + Namespace: "foo", + } + // rbdplugin has 11 volumes by default + defaultVolumes := 11 + ds, err := templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) + assert.Nil(t, err) + applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec) + + assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes) + // add new volume + volumes := []corev1.Volume{ + { + Name: "test-host-dev", + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/dev"}}, + }, + } + volumeRaw, err := yaml.Marshal(volumes) + assert.Nil(t, err) + config[configKey] = string(volumeRaw) + ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) + assert.Nil(t, err) + applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec) + + assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+1) + // add one more volume + volumes = append(volumes, corev1.Volume{ + Name: "test-host-run", + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run"}}, + }) + volumeRaw, err = yaml.Marshal(volumes) + assert.Nil(t, err) + config[configKey] = string(volumeRaw) + ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) + assert.Nil(t, err) + applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec) + assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+2) + // override existing volume configuration + volumes[1].VolumeSource = corev1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/run/test/run"}} + volumeRaw, err = yaml.Marshal(volumes) + assert.Nil(t, err) + config[configKey] = string(volumeRaw) + ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) + assert.Nil(t, err) + applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec) + assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+2) + // remove existing volume configuration + volumes = []corev1.Volume{ + { + Name: "test-host-dev", + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/dev"}}, + }, + } + volumeRaw, err = yaml.Marshal(volumes) + assert.Nil(t, err) + config[configKey] = string(volumeRaw) + ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) + assert.Nil(t, err) + applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec) + assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+1) + +} + +func Test_applyVolumeMountToContainer(t *testing.T) { + // when no volumes specified + config := make(map[string]string) + configKey := "TEST_CSI_PLUGIN_VOLUME_MOUNT" + dsName := "test-ds" + rbdContainerName := "csi-rbdplugin" + tp := templateParam{ + Param: CSIParam, + Namespace: "foo", + } + // csi-rbdplugin has 10 volumes by default + defaultVolumes := 10 + ds, err := templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) + assert.Nil(t, err) + applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec) + + assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes) + // add new volume mount + volumeMounts := []corev1.VolumeMount{ + { + Name: "test-host-dev", + MountPath: "/dev/test", + }, + } + volumeMountsRaw, err := yaml.Marshal(volumeMounts) + assert.Nil(t, err) + config[configKey] = string(volumeMountsRaw) + ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) + assert.Nil(t, err) + applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec) + + assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+1) + // add one more volumemount + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "test-host-run", + MountPath: "/run/test", + }) + volumeMountsRaw, err = yaml.Marshal(volumeMounts) + assert.Nil(t, err) + config[configKey] = string(volumeMountsRaw) + ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) + assert.Nil(t, err) + applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec) + assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+2) + // override existing volume configuration + volumeMounts[1].MountPath = "/run/test/run" + volumeMountsRaw, err = yaml.Marshal(volumeMounts) + assert.Nil(t, err) + config[configKey] = string(volumeMountsRaw) + ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) + assert.Nil(t, err) + applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec) + assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+2) + // remove existing volume configuration + volumeMounts = []corev1.VolumeMount{ + { + Name: "test-host-dev", + MountPath: "/dev/test", + }, + } + volumeMountsRaw, err = yaml.Marshal(volumeMounts) + assert.Nil(t, err) + config[configKey] = string(volumeMountsRaw) + ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) + assert.Nil(t, err) + applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec) + assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+1) + +} diff --git a/pkg/operator/k8sutil/volume.go b/pkg/operator/k8sutil/volume.go index 22f67be44..ea4cdfc3e 100644 --- a/pkg/operator/k8sutil/volume.go +++ b/pkg/operator/k8sutil/volume.go @@ -18,12 +18,14 @@ limitations under the License. package k8sutil import ( + "encoding/json" "fmt" "os" "strings" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/apimachinery/pkg/util/yaml" ) const ( @@ -90,3 +92,44 @@ func BinariesMountInfo() (v1.EnvVar, v1.Volume, v1.VolumeMount) { m := v1.VolumeMount{Name: volumeName, MountPath: BinariesMountPath} return e, v, m } + +// This function takes raw YAML string and converts it to Kubernetes Volume array. +func YamlToVolumes(raw string) ([]v1.Volume, error) { + if raw == "" { + return []v1.Volume{}, nil + } + + rawJSON, err := yaml.ToJSON([]byte(raw)) + if err != nil { + return []v1.Volume{}, err + } + + var volume []v1.Volume + err = json.Unmarshal(rawJSON, &volume) + if err != nil { + return []v1.Volume{}, err + } + + return volume, nil +} + +// This function takes raw YAML string and converts it to Kubernetes Volume +// mount array. +func YamlToVolumeMounts(raw string) ([]v1.VolumeMount, error) { + if raw == "" { + return []v1.VolumeMount{}, nil + } + + rawJSON, err := yaml.ToJSON([]byte(raw)) + if err != nil { + return []v1.VolumeMount{}, err + } + + var volumeMount []v1.VolumeMount + err = json.Unmarshal(rawJSON, &volumeMount) + if err != nil { + return []v1.VolumeMount{}, err + } + + return volumeMount, nil +}