Ceph: adds "rook-ceph-operator-config" configmap

"rook-ceph-operator-config" can be used to override Env Var
initialized during operator deployment.

This commit makes use of the ConfigMap to determine NodeAffinity
and Tolerations for ceph-csi drivers.

Closes #3239
Signed-off-by: Umanga Chapagain <chapagainumanga@gmail.com>
This commit is contained in:
Umanga Chapagain committed 2020-02-12 14:07:03 +05:30
1 parent 7877a4b10e
commit 7b201b8dae
6 files changed
+187 -21

No files matched your search

@@ -88,6 +88,41 @@ users:
- system:serviceaccount:rook-ceph:rook-csi-cephfs-plugin-sa
- system:serviceaccount:rook-ceph:rook-csi-cephfs-provisioner-sa
---
# Rook Ceph Operator Config
# Use this ConfigMap to override operator configurations
# Precedence will be given to this config in case
# Env Var also exists for the same
#
kind: ConfigMap
apiVersion: v1
metadata:
name: rook-ceph-operator-config
# should be in the namespace of the operator
namespace: rook-ceph
data:
# # (Optional) Ceph Provisioner NodeAffinity.
# CSI_PROVISIONER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
# # (Optional) CEPH CSI provisioner tolerations list. Put here list of taints you want to tolerate in YAML format.
# # CSI provisioner would be best to start on the same nodes as other ceph daemons.
# CSI_PROVISIONER_TOLERATIONS: |
# - effect: NoSchedule
# key: node-role.kubernetes.io/controlplane
# operator: Exists
# - effect: NoExecute
# key: node-role.kubernetes.io/etcd
# operator: Exists
# # (Optional) Ceph CSI plugin NodeAffinity.
# CSI_PLUGIN_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
# # (Optional) CEPH CSI plugin tolerations list. Put here list of taints you want to tolerate in YAML format.
# # CSI plugins need to be started on all the nodes where the clients need to mount the storage.
# CSI_PLUGIN_TOLERATIONS: |
# - effect: NoSchedule
# key: node-role.kubernetes.io/controlplane
# operator: Exists
# - effect: NoExecute
# key: node-role.kubernetes.io/etcd
# operator: Exists
---
# The deployment for the rook operator
# OLM: BEGIN OPERATOR DEPLOYMENT
apiVersion: apps/v1
@@ -9,6 +9,41 @@
# Also see other operator sample files for variations of operator.yaml:
# - operator-openshift.yaml: Common settings for running in OpenShift
#################################################################################################################
# Rook Ceph Operator Config
# Use this ConfigMap to override operator configurations
# Precedence will be given to this config in case
# Env Var also exists for the same
#
kind: ConfigMap
apiVersion: v1
metadata:
name: rook-ceph-operator-config
# should be in the namespace of the operator
namespace: rook-ceph
data:
# # (Optional) Ceph Provisioner NodeAffinity.
# CSI_PROVISIONER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
# # (Optional) CEPH CSI provisioner tolerations list. Put here list of taints you want to tolerate in YAML format.
# # CSI provisioner would be best to start on the same nodes as other ceph daemons.
# CSI_PROVISIONER_TOLERATIONS: |
# - effect: NoSchedule
# key: node-role.kubernetes.io/controlplane
# operator: Exists
# - effect: NoExecute
# key: node-role.kubernetes.io/etcd
# operator: Exists
# # (Optional) Ceph CSI plugin NodeAffinity.
# CSI_PLUGIN_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
# # (Optional) CEPH CSI plugin tolerations list. Put here list of taints you want to tolerate in YAML format.
# # CSI plugins need to be started on all the nodes where the clients need to mount the storage.
# CSI_PLUGIN_TOLERATIONS: |
# - effect: NoSchedule
# key: node-role.kubernetes.io/controlplane
# operator: Exists
# - effect: NoExecute
# key: node-role.kubernetes.io/etcd
# operator: Exists
---
# OLM: BEGIN OPERATOR DEPLOYMENT
apiVersion: apps/v1
kind: Deployment
+4 -4
View File
@@ -312,11 +312,11 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers
}
}
// get provisioner toleration and node affinity
provisionerTolerations := getToleration(true)
provisionerNodeAffinity := getNodeAffinity(true)
provisionerTolerations := getToleration(clientset, true)
provisionerNodeAffinity := getNodeAffinity(clientset, true)
// get plugin toleration and node affinity
pluginTolerations := getToleration(false)
pluginNodeAffinity := getNodeAffinity(false)
pluginTolerations := getToleration(clientset, false)
pluginNodeAffinity := getNodeAffinity(clientset, false)
if rbdPlugin != nil {
applyToPodSpec(&rbdPlugin.Spec.Template.Spec, pluginNodeAffinity, pluginTolerations)
err = k8sutil.CreateDaemonSet("csi-rbdplugin", namespace, clientset, rbdPlugin)
+26 -16
View File
@@ -24,12 +24,12 @@ import (
"strings"
"text/template"
"github.com/ghodss/yaml"
"github.com/pkg/errors"
k8sutil "github.com/rook/rook/pkg/operator/k8sutil"
"github.com/ghodss/yaml"
apps "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
)
func loadTemplate(name, templatePath string, p templateParam) (string, error) {
@@ -104,21 +104,25 @@ func templateToDeployment(name, templatePath string, p templateParam) (*apps.Dep
return &ds, nil
}
func getToleration(provisioner bool) []corev1.Toleration {
func getToleration(clientset kubernetes.Interface, provisioner bool) []corev1.Toleration {
// Add toleration if any
tolerations := []corev1.Toleration{}
var err error
tolerationsRaw := ""
if provisioner {
tolerationsRaw = os.Getenv(provisionerTolerationsEnv)
tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, provisionerTolerationsEnv, "")
} else {
tolerationsRaw = os.Getenv(pluginTolerationsEnv)
tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, pluginTolerationsEnv, "")
}
if tolerationsRaw != "" {
tolerations, err = k8sutil.YamlToTolerations(tolerationsRaw)
if err != nil {
logger.Warningf("failed to parse %q. %v", tolerationsRaw, err)
}
if err != nil {
// tolerationsRaw is empty
logger.Warningf("tolerations will not be applied. %v", err)
return tolerations
}
tolerations, err = k8sutil.YamlToTolerations(tolerationsRaw)
if err != nil {
logger.Warningf("failed to parse %q. %v", tolerationsRaw, err)
return tolerations
}
for i := range tolerations {
if tolerations[i].Key == "" {
@@ -132,22 +136,28 @@ func getToleration(provisioner bool) []corev1.Toleration {
return tolerations
}
func getNodeAffinity(provisioner bool) *corev1.NodeAffinity {
func getNodeAffinity(clientset kubernetes.Interface, provisioner bool) *corev1.NodeAffinity {
// Add NodeAffinity if any
nodeAffinity := ""
v1NodeAffinity := &corev1.NodeAffinity{}
var err error
if provisioner {
nodeAffinity = os.Getenv(provisionerNodeAffinityEnv)
nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, provisionerNodeAffinityEnv, "")
} else {
nodeAffinity = os.Getenv(pluginNodeAffinityEnv)
nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, pluginNodeAffinityEnv, "")
}
if err != nil {
logger.Warningf("node affinity will not be applied. %v", err)
// nodeAffinity will be empty by default in case of error
return v1NodeAffinity
}
if nodeAffinity != "" {
v1NodeAffinity, err := k8sutil.GenerateNodeAffinity(nodeAffinity)
v1NodeAffinity, err = k8sutil.GenerateNodeAffinity(nodeAffinity)
if err != nil {
logger.Warningf("failed to parse %q. %v", nodeAffinity, err)
}
return v1NodeAffinity
}
return nil
return v1NodeAffinity
}
func applyToPodSpec(pod *corev1.PodSpec, n *corev1.NodeAffinity, t []corev1.Toleration) {
+26
View File
@@ -18,8 +18,10 @@ package k8sutil
import (
"fmt"
"os"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
@@ -36,3 +38,27 @@ func DeleteConfigMap(clientset kubernetes.Interface, cmName, namespace string, o
defaultWaitOptions := &WaitOptions{RetryCount: 20, RetryInterval: 2 * time.Second}
return DeleteResource(delete, verify, resource, opts, defaultWaitOptions)
}
// GetOperatorSetting gets the operator setting from ConfigMap or Env Var
// returns defaultValue if setting is not found
func GetOperatorSetting(clientset kubernetes.Interface, settingName, defaultValue string) (string, error) {
// config must be in operator pod namespace
const rookCephOperatorSetting string = "rook-ceph-operator-config"
namespace := os.Getenv("POD_NAMESPACE")
cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(rookCephOperatorSetting, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
if settingValue, ok := os.LookupEnv(settingName); ok {
return settingValue, nil
}
return defaultValue, nil
}
return defaultValue, fmt.Errorf("error reading ConfigMap %q. %v", rookCephOperatorSetting, err)
}
if settingValue, ok := cm.Data[settingName]; ok {
return settingValue, nil
} else if settingValue, ok := os.LookupEnv(settingName); ok {
return settingValue, nil
}
return defaultValue, nil
}
+61 -1
View File
@@ -17,10 +17,11 @@ limitations under the License.
package k8sutil
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
@@ -55,3 +56,62 @@ func TestDeleteConfigMap(t *testing.T) {
assert.Error(t, err)
assert.True(t, errors.IsNotFound(err))
}
func TestGetOperatorSetting(t *testing.T) {
k8s := fake.NewSimpleClientset()
cm := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "rook-ceph-operator-config",
Namespace: "test-namespace",
},
Data: map[string]string{
"NODE_AFFINITY": "storage=rook, ceph",
},
}
nodeAffinity := "NODE_AFFINITY"
podAffinity := "POD_AFFINITY"
envSettingValue := "role=storage-node"
cmSettingValue := "storage=rook, ceph"
defaultValue := ""
// ConfigMap is not found
setting, err := GetOperatorSetting(k8s, nodeAffinity, defaultValue)
assert.NoError(t, err)
// Env Var doesn't exist
assert.Equal(t, defaultValue, setting)
// Env Var exists
err = os.Setenv(nodeAffinity, envSettingValue)
assert.NoError(t, err)
setting, err = GetOperatorSetting(k8s, nodeAffinity, defaultValue)
assert.NoError(t, err)
assert.Equal(t, envSettingValue, setting)
// ConfigMap is found
_, err = k8s.CoreV1().ConfigMaps("test-namespace").Create(cm)
assert.NoError(t, err)
// Setting exists in ConfigMap
setting, err = GetOperatorSetting(k8s, nodeAffinity, defaultValue)
assert.NoError(t, err)
// Env Var exists
assert.Equal(t, cmSettingValue, setting)
// Env Var doesn't exist
err = os.Unsetenv(nodeAffinity)
assert.NoError(t, err)
assert.Equal(t, cmSettingValue, setting)
// Setting doesn't exist in ConfigMap
setting, err = GetOperatorSetting(k8s, podAffinity, defaultValue)
assert.NoError(t, err)
// Env Var doesn't exist
assert.Equal(t, defaultValue, setting)
// Env Var exists
err = os.Setenv(podAffinity, envSettingValue)
assert.NoError(t, err)
setting, err = GetOperatorSetting(k8s, podAffinity, defaultValue)
assert.NoError(t, err)
assert.Equal(t, envSettingValue, setting)
}