Ceph: adds "rook-ceph-operator-config" configmap
"rook-ceph-operator-config" can be used to override Env Var initialized during operator deployment. This commit makes use of the ConfigMap to determine NodeAffinity and Tolerations for ceph-csi drivers. Closes #3239 Signed-off-by: Umanga Chapagain <chapagainumanga@gmail.com>
This commit is contained in:
6 files changed
+187
-21
No files matched your search
@@ -88,6 +88,41 @@ users:
|
||||
- system:serviceaccount:rook-ceph:rook-csi-cephfs-plugin-sa
|
||||
- system:serviceaccount:rook-ceph:rook-csi-cephfs-provisioner-sa
|
||||
---
|
||||
# Rook Ceph Operator Config
|
||||
# Use this ConfigMap to override operator configurations
|
||||
# Precedence will be given to this config in case
|
||||
# Env Var also exists for the same
|
||||
#
|
||||
kind: ConfigMap
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: rook-ceph-operator-config
|
||||
# should be in the namespace of the operator
|
||||
namespace: rook-ceph
|
||||
data:
|
||||
# # (Optional) Ceph Provisioner NodeAffinity.
|
||||
# CSI_PROVISIONER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
|
||||
# # (Optional) CEPH CSI provisioner tolerations list. Put here list of taints you want to tolerate in YAML format.
|
||||
# # CSI provisioner would be best to start on the same nodes as other ceph daemons.
|
||||
# CSI_PROVISIONER_TOLERATIONS: |
|
||||
# - effect: NoSchedule
|
||||
# key: node-role.kubernetes.io/controlplane
|
||||
# operator: Exists
|
||||
# - effect: NoExecute
|
||||
# key: node-role.kubernetes.io/etcd
|
||||
# operator: Exists
|
||||
# # (Optional) Ceph CSI plugin NodeAffinity.
|
||||
# CSI_PLUGIN_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
|
||||
# # (Optional) CEPH CSI plugin tolerations list. Put here list of taints you want to tolerate in YAML format.
|
||||
# # CSI plugins need to be started on all the nodes where the clients need to mount the storage.
|
||||
# CSI_PLUGIN_TOLERATIONS: |
|
||||
# - effect: NoSchedule
|
||||
# key: node-role.kubernetes.io/controlplane
|
||||
# operator: Exists
|
||||
# - effect: NoExecute
|
||||
# key: node-role.kubernetes.io/etcd
|
||||
# operator: Exists
|
||||
---
|
||||
# The deployment for the rook operator
|
||||
# OLM: BEGIN OPERATOR DEPLOYMENT
|
||||
apiVersion: apps/v1
|
||||
|
||||
@@ -9,6 +9,41 @@
|
||||
# Also see other operator sample files for variations of operator.yaml:
|
||||
# - operator-openshift.yaml: Common settings for running in OpenShift
|
||||
#################################################################################################################
|
||||
# Rook Ceph Operator Config
|
||||
# Use this ConfigMap to override operator configurations
|
||||
# Precedence will be given to this config in case
|
||||
# Env Var also exists for the same
|
||||
#
|
||||
kind: ConfigMap
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: rook-ceph-operator-config
|
||||
# should be in the namespace of the operator
|
||||
namespace: rook-ceph
|
||||
data:
|
||||
# # (Optional) Ceph Provisioner NodeAffinity.
|
||||
# CSI_PROVISIONER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
|
||||
# # (Optional) CEPH CSI provisioner tolerations list. Put here list of taints you want to tolerate in YAML format.
|
||||
# # CSI provisioner would be best to start on the same nodes as other ceph daemons.
|
||||
# CSI_PROVISIONER_TOLERATIONS: |
|
||||
# - effect: NoSchedule
|
||||
# key: node-role.kubernetes.io/controlplane
|
||||
# operator: Exists
|
||||
# - effect: NoExecute
|
||||
# key: node-role.kubernetes.io/etcd
|
||||
# operator: Exists
|
||||
# # (Optional) Ceph CSI plugin NodeAffinity.
|
||||
# CSI_PLUGIN_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
|
||||
# # (Optional) CEPH CSI plugin tolerations list. Put here list of taints you want to tolerate in YAML format.
|
||||
# # CSI plugins need to be started on all the nodes where the clients need to mount the storage.
|
||||
# CSI_PLUGIN_TOLERATIONS: |
|
||||
# - effect: NoSchedule
|
||||
# key: node-role.kubernetes.io/controlplane
|
||||
# operator: Exists
|
||||
# - effect: NoExecute
|
||||
# key: node-role.kubernetes.io/etcd
|
||||
# operator: Exists
|
||||
---
|
||||
# OLM: BEGIN OPERATOR DEPLOYMENT
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
|
||||
@@ -312,11 +312,11 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers
|
||||
}
|
||||
}
|
||||
// get provisioner toleration and node affinity
|
||||
provisionerTolerations := getToleration(true)
|
||||
provisionerNodeAffinity := getNodeAffinity(true)
|
||||
provisionerTolerations := getToleration(clientset, true)
|
||||
provisionerNodeAffinity := getNodeAffinity(clientset, true)
|
||||
// get plugin toleration and node affinity
|
||||
pluginTolerations := getToleration(false)
|
||||
pluginNodeAffinity := getNodeAffinity(false)
|
||||
pluginTolerations := getToleration(clientset, false)
|
||||
pluginNodeAffinity := getNodeAffinity(clientset, false)
|
||||
if rbdPlugin != nil {
|
||||
applyToPodSpec(&rbdPlugin.Spec.Template.Spec, pluginNodeAffinity, pluginTolerations)
|
||||
err = k8sutil.CreateDaemonSet("csi-rbdplugin", namespace, clientset, rbdPlugin)
|
||||
|
||||
@@ -24,12 +24,12 @@ import (
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
"github.com/ghodss/yaml"
|
||||
"github.com/pkg/errors"
|
||||
k8sutil "github.com/rook/rook/pkg/operator/k8sutil"
|
||||
|
||||
"github.com/ghodss/yaml"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
func loadTemplate(name, templatePath string, p templateParam) (string, error) {
|
||||
@@ -104,21 +104,25 @@ func templateToDeployment(name, templatePath string, p templateParam) (*apps.Dep
|
||||
return &ds, nil
|
||||
}
|
||||
|
||||
func getToleration(provisioner bool) []corev1.Toleration {
|
||||
func getToleration(clientset kubernetes.Interface, provisioner bool) []corev1.Toleration {
|
||||
// Add toleration if any
|
||||
tolerations := []corev1.Toleration{}
|
||||
var err error
|
||||
tolerationsRaw := ""
|
||||
if provisioner {
|
||||
tolerationsRaw = os.Getenv(provisionerTolerationsEnv)
|
||||
tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, provisionerTolerationsEnv, "")
|
||||
} else {
|
||||
tolerationsRaw = os.Getenv(pluginTolerationsEnv)
|
||||
tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, pluginTolerationsEnv, "")
|
||||
}
|
||||
if tolerationsRaw != "" {
|
||||
tolerations, err = k8sutil.YamlToTolerations(tolerationsRaw)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to parse %q. %v", tolerationsRaw, err)
|
||||
}
|
||||
if err != nil {
|
||||
// tolerationsRaw is empty
|
||||
logger.Warningf("tolerations will not be applied. %v", err)
|
||||
return tolerations
|
||||
}
|
||||
tolerations, err = k8sutil.YamlToTolerations(tolerationsRaw)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to parse %q. %v", tolerationsRaw, err)
|
||||
return tolerations
|
||||
}
|
||||
for i := range tolerations {
|
||||
if tolerations[i].Key == "" {
|
||||
@@ -132,22 +136,28 @@ func getToleration(provisioner bool) []corev1.Toleration {
|
||||
return tolerations
|
||||
}
|
||||
|
||||
func getNodeAffinity(provisioner bool) *corev1.NodeAffinity {
|
||||
func getNodeAffinity(clientset kubernetes.Interface, provisioner bool) *corev1.NodeAffinity {
|
||||
// Add NodeAffinity if any
|
||||
nodeAffinity := ""
|
||||
v1NodeAffinity := &corev1.NodeAffinity{}
|
||||
var err error
|
||||
if provisioner {
|
||||
nodeAffinity = os.Getenv(provisionerNodeAffinityEnv)
|
||||
nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, provisionerNodeAffinityEnv, "")
|
||||
} else {
|
||||
nodeAffinity = os.Getenv(pluginNodeAffinityEnv)
|
||||
nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, pluginNodeAffinityEnv, "")
|
||||
}
|
||||
if err != nil {
|
||||
logger.Warningf("node affinity will not be applied. %v", err)
|
||||
// nodeAffinity will be empty by default in case of error
|
||||
return v1NodeAffinity
|
||||
}
|
||||
if nodeAffinity != "" {
|
||||
v1NodeAffinity, err := k8sutil.GenerateNodeAffinity(nodeAffinity)
|
||||
v1NodeAffinity, err = k8sutil.GenerateNodeAffinity(nodeAffinity)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to parse %q. %v", nodeAffinity, err)
|
||||
}
|
||||
return v1NodeAffinity
|
||||
}
|
||||
return nil
|
||||
return v1NodeAffinity
|
||||
}
|
||||
|
||||
func applyToPodSpec(pod *corev1.PodSpec, n *corev1.NodeAffinity, t []corev1.Toleration) {
|
||||
|
||||
@@ -18,8 +18,10 @@ package k8sutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
@@ -36,3 +38,27 @@ func DeleteConfigMap(clientset kubernetes.Interface, cmName, namespace string, o
|
||||
defaultWaitOptions := &WaitOptions{RetryCount: 20, RetryInterval: 2 * time.Second}
|
||||
return DeleteResource(delete, verify, resource, opts, defaultWaitOptions)
|
||||
}
|
||||
|
||||
// GetOperatorSetting gets the operator setting from ConfigMap or Env Var
|
||||
// returns defaultValue if setting is not found
|
||||
func GetOperatorSetting(clientset kubernetes.Interface, settingName, defaultValue string) (string, error) {
|
||||
// config must be in operator pod namespace
|
||||
const rookCephOperatorSetting string = "rook-ceph-operator-config"
|
||||
namespace := os.Getenv("POD_NAMESPACE")
|
||||
cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(rookCephOperatorSetting, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
if settingValue, ok := os.LookupEnv(settingName); ok {
|
||||
return settingValue, nil
|
||||
}
|
||||
return defaultValue, nil
|
||||
}
|
||||
return defaultValue, fmt.Errorf("error reading ConfigMap %q. %v", rookCephOperatorSetting, err)
|
||||
}
|
||||
if settingValue, ok := cm.Data[settingName]; ok {
|
||||
return settingValue, nil
|
||||
} else if settingValue, ok := os.LookupEnv(settingName); ok {
|
||||
return settingValue, nil
|
||||
}
|
||||
return defaultValue, nil
|
||||
}
|
||||
@@ -17,10 +17,11 @@ limitations under the License.
|
||||
package k8sutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/api/core/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
@@ -55,3 +56,62 @@ func TestDeleteConfigMap(t *testing.T) {
|
||||
assert.Error(t, err)
|
||||
assert.True(t, errors.IsNotFound(err))
|
||||
}
|
||||
|
||||
func TestGetOperatorSetting(t *testing.T) {
|
||||
k8s := fake.NewSimpleClientset()
|
||||
|
||||
cm := &v1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "rook-ceph-operator-config",
|
||||
Namespace: "test-namespace",
|
||||
},
|
||||
Data: map[string]string{
|
||||
"NODE_AFFINITY": "storage=rook, ceph",
|
||||
},
|
||||
}
|
||||
|
||||
nodeAffinity := "NODE_AFFINITY"
|
||||
podAffinity := "POD_AFFINITY"
|
||||
envSettingValue := "role=storage-node"
|
||||
cmSettingValue := "storage=rook, ceph"
|
||||
defaultValue := ""
|
||||
|
||||
// ConfigMap is not found
|
||||
setting, err := GetOperatorSetting(k8s, nodeAffinity, defaultValue)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Env Var doesn't exist
|
||||
assert.Equal(t, defaultValue, setting)
|
||||
// Env Var exists
|
||||
err = os.Setenv(nodeAffinity, envSettingValue)
|
||||
assert.NoError(t, err)
|
||||
setting, err = GetOperatorSetting(k8s, nodeAffinity, defaultValue)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, envSettingValue, setting)
|
||||
|
||||
// ConfigMap is found
|
||||
_, err = k8s.CoreV1().ConfigMaps("test-namespace").Create(cm)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Setting exists in ConfigMap
|
||||
setting, err = GetOperatorSetting(k8s, nodeAffinity, defaultValue)
|
||||
assert.NoError(t, err)
|
||||
// Env Var exists
|
||||
assert.Equal(t, cmSettingValue, setting)
|
||||
// Env Var doesn't exist
|
||||
err = os.Unsetenv(nodeAffinity)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, cmSettingValue, setting)
|
||||
|
||||
// Setting doesn't exist in ConfigMap
|
||||
setting, err = GetOperatorSetting(k8s, podAffinity, defaultValue)
|
||||
assert.NoError(t, err)
|
||||
// Env Var doesn't exist
|
||||
assert.Equal(t, defaultValue, setting)
|
||||
// Env Var exists
|
||||
err = os.Setenv(podAffinity, envSettingValue)
|
||||
assert.NoError(t, err)
|
||||
setting, err = GetOperatorSetting(k8s, podAffinity, defaultValue)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, envSettingValue, setting)
|
||||
}
|
||||
Reference in new issue
Block a user