Merge pull request #10942 from Madhu-1/fix-10932

csi: customize plugin volumes and volumemounts
This commit is contained in:
Travis Nielsen authored and GitHub committed 2022-09-16 11:33:31 -06:00
commit 85b079105b
9 files changed
+377

No files matched your search

@@ -133,6 +133,10 @@ The following tables lists the configurable parameters of the rook-operator char
| `admissionController.tolerations` | Array of tolerations in YAML format which will be added to admission controller deployment. | `<none>` |
| `admissionController.nodeAffinity` | The node labels for affinity of the admission controller deployment [^1] | `<none>` |
| `monitoring.enabled` | Create necessary RBAC rules for Rook to integrate with Prometheus monitoring in the operator namespace. Requires Prometheus to be pre-installed. | `false` |
| `csi.csiRBDPluginVolume` | The volume of the CephCSI RBD plugin DaemonSet | `<none>` |
| `csi.csiRBDPluginVolumeMount` | The volume mounts of the CephCSI RBD plugin DaemonSet | `<none>` |
| `csi.csiCephFSPluginVolume` | The volume of the CephCSI CephFS plugin DaemonSet | `<none>` |
| `csi.csiCephFSPluginVolumeMount` | The volume mounts of the CephCSI CephFS plugin DaemonSet | `<none>` |
[^1]: `nodeAffinity` and `*NodeAffinity` options should have the format `"role=storage,rook; storage=ceph"` or `storage=;role=rook-example` or `storage=;` (_checks only for presence of key_)
@@ -203,4 +203,16 @@ data:
{{- if .Values.csi.csiNFSPluginResource }}
CSI_NFS_PLUGIN_RESOURCE: {{ .Values.csi.csiNFSPluginResource | quote }}
{{- end }}
{{- if .Values.csi.csiRBDPluginVolume }}
CSI_RBD_PLUGIN_VOLUME: {{ toYaml .Values.csi.csiRBDPluginVolume | quote }}
{{- end }}
{{- if .Values.csi.csiRBDPluginVolumeMount }}
CSI_RBD_PLUGIN_VOLUME_MOUNT: {{ toYaml .Values.csi.csiRBDPluginVolumeMount | quote }}
{{- end }}
{{- if .Values.csi.csiCephFSPluginVolume }}
CSI_CEPHFS_PLUGIN_VOLUME: {{ toYaml .Values.csi.csiRBDPluginVolumes | quote }}
{{- end }}
{{- if .Values.csi.csiCephFSPluginVolumeMount }}
CSI_CEPHFS_PLUGIN_VOLUME_MOUNT: {{ toYaml .Values.csi.csiCephFSPluginVolumeMount | quote }}
{{- end }}
{{- end }}
+31
View File
@@ -131,6 +131,37 @@ csi:
# Allow starting unsupported ceph-csi image
allowUnsupportedVersion: false
# CephCSI RBD plugin Volumes
# csiRBDPluginVolumes: |
# - name: lib-modules
# hostPath:
# path: /run/current-system/kernel-modules/lib/modules/
# - name: host-nix
# hostPath:
# path: /nix
# CephCSI RBD plugin Volume mounts
# csiRBDPluginVolumeMounts: |
# - name: host-nix
# mountPath: /nix
# readOnly: true
# CephCSI CephFS plugin Volumes
# csiCephFSPluginVolumes: |
# - name: lib-modules
# hostPath:
# path: /run/current-system/kernel-modules/lib/modules/
# - name: host-nix
# hostPath:
# path: /nix
# CephCSI CephFS plugin Volume mounts
# csiCephFSPluginVolumeMounts: |
# - name: host-nix
# mountPath: /nix
# readOnly: true
# CEPH CSI RBD provisioner resource requirement list, Put here list of resource
# requests and limits you want to apply for provisioner pod
# csi-omap-generator resources will be applied only if enableOMAPGenerator is set to true
+30
View File
@@ -221,6 +221,36 @@ data:
# Labels to add to the CSI NFS Deployments and DaemonSets Pods.
# ROOK_CSI_NFS_POD_LABELS: "key1=value1,key2=value2"
# (Optional) CephCSI CephFS plugin Volumes
# CSI_CEPHFS_PLUGIN_VOLUME: |
# - name: lib-modules
# hostPath:
# path: /run/current-system/kernel-modules/lib/modules/
# - name: host-nix
# hostPath:
# path: /nix
# (Optional) CephCSI CephFS plugin Volume mounts
# CSI_CEPHFS_PLUGIN_VOLUME_MOUNT: |
# - name: host-nix
# mountPath: /nix
# readOnly: true
# (Optional) CephCSI RBD plugin Volumes
# CSI_RBD_PLUGIN_VOLUME: |
# - name: lib-modules
# hostPath:
# path: /run/current-system/kernel-modules/lib/modules/
# - name: host-nix
# hostPath:
# path: /nix
# (Optional) CephCSI RBD plugin Volume mounts
# CSI_RBD_PLUGIN_VOLUME_MOUNT: |
# - name: host-nix
# mountPath: /nix
# readOnly: true
# (Optional) CephCSI provisioner NodeAffinity(applied to both CephFS and RBD provisioner).
# CSI_PROVISIONER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
# (Optional) CephCSI provisioner tolerations list(applied to both CephFS and RBD provisioner).
+30
View File
@@ -138,6 +138,36 @@ data:
# Labels to add to the CSI NFS Deployments and DaemonSets Pods.
# ROOK_CSI_NFS_POD_LABELS: "key1=value1,key2=value2"
# (Optional) CephCSI CephFS plugin Volumes
# CSI_CEPHFS_PLUGIN_VOLUME: |
# - name: lib-modules
# hostPath:
# path: /run/current-system/kernel-modules/lib/modules/
# - name: host-nix
# hostPath:
# path: /nix
# (Optional) CephCSI CephFS plugin Volume mounts
# CSI_CEPHFS_PLUGIN_VOLUME_MOUNT: |
# - name: host-nix
# mountPath: /nix
# readOnly: true
# (Optional) CephCSI RBD plugin Volumes
# CSI_RBD_PLUGIN_VOLUME: |
# - name: lib-modules
# hostPath:
# path: /run/current-system/kernel-modules/lib/modules/
# - name: host-nix
# hostPath:
# path: /nix
# (Optional) CephCSI RBD plugin Volume mounts
# CSI_RBD_PLUGIN_VOLUME_MOUNT: |
# - name: host-nix
# mountPath: /nix
# readOnly: true
# (Optional) CephCSI provisioner NodeAffinity (applied to both CephFS and RBD provisioner).
# CSI_PROVISIONER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
# (Optional) CephCSI provisioner tolerations list(applied to both CephFS and RBD provisioner).
+21
View File
@@ -204,6 +204,15 @@ const (
nfsProvisionerResource = "CSI_NFS_PROVISIONER_RESOURCE"
nfsPluginResource = "CSI_NFS_PLUGIN_RESOURCE"
cephFSPluginVolume = "CSI_CEPHFS_PLUGIN_VOLUME"
cephFSPluginVolumeMount = "CSI_CEPHFS_PLUGIN_VOLUME_MOUNT"
rbdPluginVolume = "CSI_RBD_PLUGIN_VOLUME"
rbdPluginVolumeMount = "CSI_RBD_PLUGIN_VOLUME_MOUNT"
nfsPluginVolume = "CSI_NFS_PLUGIN_VOLUME"
nfsPluginVolumeMount = "CSI_NFS_PLUGIN_VOLUME_MOUNT"
// kubelet directory path
DefaultKubeletDirPath = "/var/lib/kubelet"
@@ -422,6 +431,10 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
applyToPodSpec(&rbdPlugin.Spec.Template.Spec, rbdPluginNodeAffinity, rbdPluginTolerations)
// apply resource request and limit to rbdplugin containers
applyResourcesToContainers(r.opConfig.Parameters, rbdPluginResource, &rbdPlugin.Spec.Template.Spec)
// apply custom mounts to volumes
applyVolumeToPodSpec(r.opConfig.Parameters, rbdPluginVolume, &rbdPlugin.Spec.Template.Spec)
// apply custom mounts to volume mounts
applyVolumeMountToContainer(r.opConfig.Parameters, rbdPluginVolumeMount, "csi-rbdplugin", &rbdPlugin.Spec.Template.Spec)
err = ownerInfo.SetControllerReference(rbdPlugin)
if err != nil {
return errors.Wrapf(err, "failed to set owner reference to rbd plugin daemonset %q", rbdPlugin.Name)
@@ -490,6 +503,10 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
applyToPodSpec(&cephfsPlugin.Spec.Template.Spec, cephFSPluginNodeAffinity, cephFSPluginTolerations)
// apply resource request and limit to cephfs plugin containers
applyResourcesToContainers(r.opConfig.Parameters, cephFSPluginResource, &cephfsPlugin.Spec.Template.Spec)
// apply custom mounts to volumes
applyVolumeToPodSpec(r.opConfig.Parameters, cephFSPluginVolume, &cephfsPlugin.Spec.Template.Spec)
// apply custom mounts to volume mounts
applyVolumeMountToContainer(r.opConfig.Parameters, cephFSPluginVolumeMount, "csi-cephfsplugin", &cephfsPlugin.Spec.Template.Spec)
err = ownerInfo.SetControllerReference(cephfsPlugin)
if err != nil {
return errors.Wrapf(err, "failed to set owner reference to cephfs plugin daemonset %q", cephfsPlugin.Name)
@@ -561,6 +578,10 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
applyToPodSpec(&nfsPlugin.Spec.Template.Spec, nfsPluginNodeAffinity, nfsPluginTolerations)
// apply resource request and limit to nfs plugin containers
applyResourcesToContainers(r.opConfig.Parameters, nfsPluginResource, &nfsPlugin.Spec.Template.Spec)
// apply custom mounts to volumes
applyVolumeToPodSpec(r.opConfig.Parameters, nfsPluginVolume, &nfsPlugin.Spec.Template.Spec)
// apply custom mounts to volume mounts
applyVolumeMountToContainer(r.opConfig.Parameters, nfsPluginVolumeMount, "csi-nfsplugin", &nfsPlugin.Spec.Template.Spec)
err = ownerInfo.SetControllerReference(nfsPlugin)
if err != nil {
return errors.Wrapf(err, "failed to set owner reference to nfs plugin daemonset %q", nfsPlugin.Name)
+64
View File
@@ -188,3 +188,67 @@ func GetPodAntiAffinity(key, value string) corev1.PodAntiAffinity {
},
}
}
func applyVolumeToPodSpec(opConfig map[string]string, configName string, podspec *corev1.PodSpec) {
volumesRaw := k8sutil.GetValue(opConfig, configName, "")
if volumesRaw == "" {
return
}
volumes, err := k8sutil.YamlToVolumes(volumesRaw)
if err != nil {
logger.Warningf("failed to parse %q for %q. %v", volumesRaw, configName, err)
return
}
if len(volumes) > 0 {
for i := range volumes {
found := false
for j := range podspec.Volumes {
// check do we need to override any existing volumes
if volumes[i].Name == podspec.Volumes[j].Name {
podspec.Volumes[j] = volumes[i]
found = true
break
}
}
if !found {
// if not found add volume to volumes list
podspec.Volumes = append(podspec.Volumes, volumes[i])
}
}
}
}
func applyVolumeMountToContainer(opConfig map[string]string, configName, containerName string, podspec *corev1.PodSpec) {
volumeMountsRaw := k8sutil.GetValue(opConfig, configName, "")
if volumeMountsRaw == "" {
return
}
volumeMounts, err := k8sutil.YamlToVolumeMounts(volumeMountsRaw)
if err != nil {
logger.Warningf("failed to parse %q for %q. %v", volumeMountsRaw, configName, err)
return
}
if len(volumeMounts) > 0 {
for i, c := range podspec.Containers {
if c.Name == containerName {
for j := range volumeMounts {
found := false
for k := range podspec.Containers[i].VolumeMounts {
// override if the name is matching
if volumeMounts[j].Name == podspec.Containers[i].VolumeMounts[k].Name {
found = true
podspec.Containers[i].VolumeMounts[k] = volumeMounts[j]
break
}
}
if !found {
// if not found append it to the exiting volumes
podspec.Containers[i].VolumeMounts = append(podspec.Containers[i].VolumeMounts, volumeMounts[j])
}
}
// return as we finished with found container
return
}
}
}
}
+142
View File
@@ -22,6 +22,8 @@ import (
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v2"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
)
func TestDaemonSetTemplate(t *testing.T) {
@@ -109,3 +111,143 @@ func TestApplyingResourcesToRBDPlugin(t *testing.T) {
assert.Equal(t, rbdPlugin.Spec.Template.Spec.Containers[0].Resources.Requests.Cpu().String(), "100m")
assert.Equal(t, rbdPlugin.Spec.Template.Spec.Containers[0].Resources.Limits.Cpu().String(), "200m")
}
func Test_applyVolumeToPodSpec(t *testing.T) {
// when no volumes specified
config := make(map[string]string)
configKey := "TEST_CSI_PLUGIN_VOLUME"
dsName := "test-ds"
tp := templateParam{
Param: CSIParam,
Namespace: "foo",
}
// rbdplugin has 11 volumes by default
defaultVolumes := 11
ds, err := templateToDaemonSet(dsName, RBDPluginTemplatePath, tp)
assert.Nil(t, err)
applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec)
assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes)
// add new volume
volumes := []corev1.Volume{
{
Name: "test-host-dev",
VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/dev"}},
},
}
volumeRaw, err := yaml.Marshal(volumes)
assert.Nil(t, err)
config[configKey] = string(volumeRaw)
ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp)
assert.Nil(t, err)
applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec)
assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+1)
// add one more volume
volumes = append(volumes, corev1.Volume{
Name: "test-host-run",
VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run"}},
})
volumeRaw, err = yaml.Marshal(volumes)
assert.Nil(t, err)
config[configKey] = string(volumeRaw)
ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp)
assert.Nil(t, err)
applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec)
assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+2)
// override existing volume configuration
volumes[1].VolumeSource = corev1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{Path: "/run/test/run"}}
volumeRaw, err = yaml.Marshal(volumes)
assert.Nil(t, err)
config[configKey] = string(volumeRaw)
ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp)
assert.Nil(t, err)
applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec)
assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+2)
// remove existing volume configuration
volumes = []corev1.Volume{
{
Name: "test-host-dev",
VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/dev"}},
},
}
volumeRaw, err = yaml.Marshal(volumes)
assert.Nil(t, err)
config[configKey] = string(volumeRaw)
ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp)
assert.Nil(t, err)
applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec)
assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+1)
}
func Test_applyVolumeMountToContainer(t *testing.T) {
// when no volumes specified
config := make(map[string]string)
configKey := "TEST_CSI_PLUGIN_VOLUME_MOUNT"
dsName := "test-ds"
rbdContainerName := "csi-rbdplugin"
tp := templateParam{
Param: CSIParam,
Namespace: "foo",
}
// csi-rbdplugin has 10 volumes by default
defaultVolumes := 10
ds, err := templateToDaemonSet(dsName, RBDPluginTemplatePath, tp)
assert.Nil(t, err)
applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec)
assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes)
// add new volume mount
volumeMounts := []corev1.VolumeMount{
{
Name: "test-host-dev",
MountPath: "/dev/test",
},
}
volumeMountsRaw, err := yaml.Marshal(volumeMounts)
assert.Nil(t, err)
config[configKey] = string(volumeMountsRaw)
ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp)
assert.Nil(t, err)
applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec)
assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+1)
// add one more volumemount
volumeMounts = append(volumeMounts, corev1.VolumeMount{
Name: "test-host-run",
MountPath: "/run/test",
})
volumeMountsRaw, err = yaml.Marshal(volumeMounts)
assert.Nil(t, err)
config[configKey] = string(volumeMountsRaw)
ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp)
assert.Nil(t, err)
applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec)
assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+2)
// override existing volume configuration
volumeMounts[1].MountPath = "/run/test/run"
volumeMountsRaw, err = yaml.Marshal(volumeMounts)
assert.Nil(t, err)
config[configKey] = string(volumeMountsRaw)
ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp)
assert.Nil(t, err)
applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec)
assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+2)
// remove existing volume configuration
volumeMounts = []corev1.VolumeMount{
{
Name: "test-host-dev",
MountPath: "/dev/test",
},
}
volumeMountsRaw, err = yaml.Marshal(volumeMounts)
assert.Nil(t, err)
config[configKey] = string(volumeMountsRaw)
ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp)
assert.Nil(t, err)
applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec)
assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+1)
}
+43
View File
@@ -18,12 +18,14 @@ limitations under the License.
package k8sutil
import (
"encoding/json"
"fmt"
"os"
"strings"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/validation"
"k8s.io/apimachinery/pkg/util/yaml"
)
const (
@@ -90,3 +92,44 @@ func BinariesMountInfo() (v1.EnvVar, v1.Volume, v1.VolumeMount) {
m := v1.VolumeMount{Name: volumeName, MountPath: BinariesMountPath}
return e, v, m
}
// This function takes raw YAML string and converts it to Kubernetes Volume array.
func YamlToVolumes(raw string) ([]v1.Volume, error) {
if raw == "" {
return []v1.Volume{}, nil
}
rawJSON, err := yaml.ToJSON([]byte(raw))
if err != nil {
return []v1.Volume{}, err
}
var volume []v1.Volume
err = json.Unmarshal(rawJSON, &volume)
if err != nil {
return []v1.Volume{}, err
}
return volume, nil
}
// This function takes raw YAML string and converts it to Kubernetes Volume
// mount array.
func YamlToVolumeMounts(raw string) ([]v1.VolumeMount, error) {
if raw == "" {
return []v1.VolumeMount{}, nil
}
rawJSON, err := yaml.ToJSON([]byte(raw))
if err != nil {
return []v1.VolumeMount{}, err
}
var volumeMount []v1.VolumeMount
err = json.Unmarshal(rawJSON, &volumeMount)
if err != nil {
return []v1.VolumeMount{}, err
}
return volumeMount, nil
}