From 7b201b8dae9ac514da96052ddbee95797cd79a15 Mon Sep 17 00:00:00 2001 From: Umanga Chapagain Date: Wed, 12 Feb 2020 14:07:03 +0530 Subject: [PATCH] Ceph: adds "rook-ceph-operator-config" configmap "rook-ceph-operator-config" can be used to override Env Var initialized during operator deployment. This commit makes use of the ConfigMap to determine NodeAffinity and Tolerations for ceph-csi drivers. Closes #3239 Signed-off-by: Umanga Chapagain --- .../kubernetes/ceph/operator-openshift.yaml | 35 +++++++++++ .../examples/kubernetes/ceph/operator.yaml | 35 +++++++++++ pkg/operator/ceph/csi/spec.go | 8 +-- pkg/operator/ceph/csi/util.go | 42 ++++++++----- pkg/operator/k8sutil/configmap.go | 26 ++++++++ pkg/operator/k8sutil/configmap_test.go | 62 ++++++++++++++++++- 6 files changed, 187 insertions(+), 21 deletions(-) diff --git a/cluster/examples/kubernetes/ceph/operator-openshift.yaml b/cluster/examples/kubernetes/ceph/operator-openshift.yaml index 11acd8f9a..50da83f02 100644 --- a/cluster/examples/kubernetes/ceph/operator-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/operator-openshift.yaml @@ -88,6 +88,41 @@ users: - system:serviceaccount:rook-ceph:rook-csi-cephfs-plugin-sa - system:serviceaccount:rook-ceph:rook-csi-cephfs-provisioner-sa --- +# Rook Ceph Operator Config +# Use this ConfigMap to override operator configurations +# Precedence will be given to this config in case +# Env Var also exists for the same +# +kind: ConfigMap +apiVersion: v1 +metadata: + name: rook-ceph-operator-config + # should be in the namespace of the operator + namespace: rook-ceph +data: + # # (Optional) Ceph Provisioner NodeAffinity. + # CSI_PROVISIONER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" + # # (Optional) CEPH CSI provisioner tolerations list. Put here list of taints you want to tolerate in YAML format. + # # CSI provisioner would be best to start on the same nodes as other ceph daemons. + # CSI_PROVISIONER_TOLERATIONS: | + # - effect: NoSchedule + # key: node-role.kubernetes.io/controlplane + # operator: Exists + # - effect: NoExecute + # key: node-role.kubernetes.io/etcd + # operator: Exists + # # (Optional) Ceph CSI plugin NodeAffinity. + # CSI_PLUGIN_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" + # # (Optional) CEPH CSI plugin tolerations list. Put here list of taints you want to tolerate in YAML format. + # # CSI plugins need to be started on all the nodes where the clients need to mount the storage. + # CSI_PLUGIN_TOLERATIONS: | + # - effect: NoSchedule + # key: node-role.kubernetes.io/controlplane + # operator: Exists + # - effect: NoExecute + # key: node-role.kubernetes.io/etcd + # operator: Exists +--- # The deployment for the rook operator # OLM: BEGIN OPERATOR DEPLOYMENT apiVersion: apps/v1 diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index 66c68108e..8a11d029a 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -9,6 +9,41 @@ # Also see other operator sample files for variations of operator.yaml: # - operator-openshift.yaml: Common settings for running in OpenShift ################################################################################################################# +# Rook Ceph Operator Config +# Use this ConfigMap to override operator configurations +# Precedence will be given to this config in case +# Env Var also exists for the same +# +kind: ConfigMap +apiVersion: v1 +metadata: + name: rook-ceph-operator-config + # should be in the namespace of the operator + namespace: rook-ceph +data: + # # (Optional) Ceph Provisioner NodeAffinity. + # CSI_PROVISIONER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" + # # (Optional) CEPH CSI provisioner tolerations list. Put here list of taints you want to tolerate in YAML format. + # # CSI provisioner would be best to start on the same nodes as other ceph daemons. + # CSI_PROVISIONER_TOLERATIONS: | + # - effect: NoSchedule + # key: node-role.kubernetes.io/controlplane + # operator: Exists + # - effect: NoExecute + # key: node-role.kubernetes.io/etcd + # operator: Exists + # # (Optional) Ceph CSI plugin NodeAffinity. + # CSI_PLUGIN_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" + # # (Optional) CEPH CSI plugin tolerations list. Put here list of taints you want to tolerate in YAML format. + # # CSI plugins need to be started on all the nodes where the clients need to mount the storage. + # CSI_PLUGIN_TOLERATIONS: | + # - effect: NoSchedule + # key: node-role.kubernetes.io/controlplane + # operator: Exists + # - effect: NoExecute + # key: node-role.kubernetes.io/etcd + # operator: Exists +--- # OLM: BEGIN OPERATOR DEPLOYMENT apiVersion: apps/v1 kind: Deployment diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 1c7cc1d85..1604e0e21 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -312,11 +312,11 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers } } // get provisioner toleration and node affinity - provisionerTolerations := getToleration(true) - provisionerNodeAffinity := getNodeAffinity(true) + provisionerTolerations := getToleration(clientset, true) + provisionerNodeAffinity := getNodeAffinity(clientset, true) // get plugin toleration and node affinity - pluginTolerations := getToleration(false) - pluginNodeAffinity := getNodeAffinity(false) + pluginTolerations := getToleration(clientset, false) + pluginNodeAffinity := getNodeAffinity(clientset, false) if rbdPlugin != nil { applyToPodSpec(&rbdPlugin.Spec.Template.Spec, pluginNodeAffinity, pluginTolerations) err = k8sutil.CreateDaemonSet("csi-rbdplugin", namespace, clientset, rbdPlugin) diff --git a/pkg/operator/ceph/csi/util.go b/pkg/operator/ceph/csi/util.go index 97c5ed4c6..422c44c72 100644 --- a/pkg/operator/ceph/csi/util.go +++ b/pkg/operator/ceph/csi/util.go @@ -24,12 +24,12 @@ import ( "strings" "text/template" + "github.com/ghodss/yaml" "github.com/pkg/errors" k8sutil "github.com/rook/rook/pkg/operator/k8sutil" - - "github.com/ghodss/yaml" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" ) func loadTemplate(name, templatePath string, p templateParam) (string, error) { @@ -104,21 +104,25 @@ func templateToDeployment(name, templatePath string, p templateParam) (*apps.Dep return &ds, nil } -func getToleration(provisioner bool) []corev1.Toleration { +func getToleration(clientset kubernetes.Interface, provisioner bool) []corev1.Toleration { // Add toleration if any tolerations := []corev1.Toleration{} var err error tolerationsRaw := "" if provisioner { - tolerationsRaw = os.Getenv(provisionerTolerationsEnv) + tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, provisionerTolerationsEnv, "") } else { - tolerationsRaw = os.Getenv(pluginTolerationsEnv) + tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, pluginTolerationsEnv, "") } - if tolerationsRaw != "" { - tolerations, err = k8sutil.YamlToTolerations(tolerationsRaw) - if err != nil { - logger.Warningf("failed to parse %q. %v", tolerationsRaw, err) - } + if err != nil { + // tolerationsRaw is empty + logger.Warningf("tolerations will not be applied. %v", err) + return tolerations + } + tolerations, err = k8sutil.YamlToTolerations(tolerationsRaw) + if err != nil { + logger.Warningf("failed to parse %q. %v", tolerationsRaw, err) + return tolerations } for i := range tolerations { if tolerations[i].Key == "" { @@ -132,22 +136,28 @@ func getToleration(provisioner bool) []corev1.Toleration { return tolerations } -func getNodeAffinity(provisioner bool) *corev1.NodeAffinity { +func getNodeAffinity(clientset kubernetes.Interface, provisioner bool) *corev1.NodeAffinity { // Add NodeAffinity if any nodeAffinity := "" + v1NodeAffinity := &corev1.NodeAffinity{} + var err error if provisioner { - nodeAffinity = os.Getenv(provisionerNodeAffinityEnv) + nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, provisionerNodeAffinityEnv, "") } else { - nodeAffinity = os.Getenv(pluginNodeAffinityEnv) + nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, pluginNodeAffinityEnv, "") + } + if err != nil { + logger.Warningf("node affinity will not be applied. %v", err) + // nodeAffinity will be empty by default in case of error + return v1NodeAffinity } if nodeAffinity != "" { - v1NodeAffinity, err := k8sutil.GenerateNodeAffinity(nodeAffinity) + v1NodeAffinity, err = k8sutil.GenerateNodeAffinity(nodeAffinity) if err != nil { logger.Warningf("failed to parse %q. %v", nodeAffinity, err) } - return v1NodeAffinity } - return nil + return v1NodeAffinity } func applyToPodSpec(pod *corev1.PodSpec, n *corev1.NodeAffinity, t []corev1.Toleration) { diff --git a/pkg/operator/k8sutil/configmap.go b/pkg/operator/k8sutil/configmap.go index cd33fda9c..79ff7188f 100644 --- a/pkg/operator/k8sutil/configmap.go +++ b/pkg/operator/k8sutil/configmap.go @@ -18,8 +18,10 @@ package k8sutil import ( "fmt" + "os" "time" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -36,3 +38,27 @@ func DeleteConfigMap(clientset kubernetes.Interface, cmName, namespace string, o defaultWaitOptions := &WaitOptions{RetryCount: 20, RetryInterval: 2 * time.Second} return DeleteResource(delete, verify, resource, opts, defaultWaitOptions) } + +// GetOperatorSetting gets the operator setting from ConfigMap or Env Var +// returns defaultValue if setting is not found +func GetOperatorSetting(clientset kubernetes.Interface, settingName, defaultValue string) (string, error) { + // config must be in operator pod namespace + const rookCephOperatorSetting string = "rook-ceph-operator-config" + namespace := os.Getenv("POD_NAMESPACE") + cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(rookCephOperatorSetting, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + if settingValue, ok := os.LookupEnv(settingName); ok { + return settingValue, nil + } + return defaultValue, nil + } + return defaultValue, fmt.Errorf("error reading ConfigMap %q. %v", rookCephOperatorSetting, err) + } + if settingValue, ok := cm.Data[settingName]; ok { + return settingValue, nil + } else if settingValue, ok := os.LookupEnv(settingName); ok { + return settingValue, nil + } + return defaultValue, nil +} diff --git a/pkg/operator/k8sutil/configmap_test.go b/pkg/operator/k8sutil/configmap_test.go index 273452128..600b99725 100644 --- a/pkg/operator/k8sutil/configmap_test.go +++ b/pkg/operator/k8sutil/configmap_test.go @@ -17,10 +17,11 @@ limitations under the License. package k8sutil import ( + "os" "testing" "github.com/stretchr/testify/assert" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" @@ -55,3 +56,62 @@ func TestDeleteConfigMap(t *testing.T) { assert.Error(t, err) assert.True(t, errors.IsNotFound(err)) } + +func TestGetOperatorSetting(t *testing.T) { + k8s := fake.NewSimpleClientset() + + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-ceph-operator-config", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "NODE_AFFINITY": "storage=rook, ceph", + }, + } + + nodeAffinity := "NODE_AFFINITY" + podAffinity := "POD_AFFINITY" + envSettingValue := "role=storage-node" + cmSettingValue := "storage=rook, ceph" + defaultValue := "" + + // ConfigMap is not found + setting, err := GetOperatorSetting(k8s, nodeAffinity, defaultValue) + assert.NoError(t, err) + + // Env Var doesn't exist + assert.Equal(t, defaultValue, setting) + // Env Var exists + err = os.Setenv(nodeAffinity, envSettingValue) + assert.NoError(t, err) + setting, err = GetOperatorSetting(k8s, nodeAffinity, defaultValue) + assert.NoError(t, err) + assert.Equal(t, envSettingValue, setting) + + // ConfigMap is found + _, err = k8s.CoreV1().ConfigMaps("test-namespace").Create(cm) + assert.NoError(t, err) + + // Setting exists in ConfigMap + setting, err = GetOperatorSetting(k8s, nodeAffinity, defaultValue) + assert.NoError(t, err) + // Env Var exists + assert.Equal(t, cmSettingValue, setting) + // Env Var doesn't exist + err = os.Unsetenv(nodeAffinity) + assert.NoError(t, err) + assert.Equal(t, cmSettingValue, setting) + + // Setting doesn't exist in ConfigMap + setting, err = GetOperatorSetting(k8s, podAffinity, defaultValue) + assert.NoError(t, err) + // Env Var doesn't exist + assert.Equal(t, defaultValue, setting) + // Env Var exists + err = os.Setenv(podAffinity, envSettingValue) + assert.NoError(t, err) + setting, err = GetOperatorSetting(k8s, podAffinity, defaultValue) + assert.NoError(t, err) + assert.Equal(t, envSettingValue, setting) +}