From 06ba93e629171619e27a952befd7913753ac8fb7 Mon Sep 17 00:00:00 2001 From: Umanga Chapagain Date: Thu, 5 Mar 2020 11:50:22 +0530 Subject: [PATCH 1/2] Ceph: refactor GetOperatorSetting for reuse Signed-off-by: Umanga Chapagain --- pkg/operator/ceph/controller/controller_utils.go | 3 +++ pkg/operator/ceph/csi/util.go | 9 +++++---- pkg/operator/k8sutil/configmap.go | 7 +++---- pkg/operator/k8sutil/configmap_test.go | 13 +++++++------ 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index 5097d6eac..5708c3373 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -31,6 +31,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) +// OperatorSettingConfigMapName refers to ConfigMap that configures rook ceph operator +const OperatorSettingConfigMapName string = "rook-ceph-operator-config" + var ( // ImmediateRetryResult Return this for a immediate retry of the reconciliation loop with the same request object. ImmediateRetryResult = reconcile.Result{Requeue: true} diff --git a/pkg/operator/ceph/csi/util.go b/pkg/operator/ceph/csi/util.go index 705415bd6..31bd271da 100644 --- a/pkg/operator/ceph/csi/util.go +++ b/pkg/operator/ceph/csi/util.go @@ -27,6 +27,7 @@ import ( "github.com/ghodss/yaml" "github.com/pkg/errors" + "github.com/rook/rook/pkg/operator/ceph/controller" k8sutil "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -111,9 +112,9 @@ func getToleration(clientset kubernetes.Interface, provisioner bool) []corev1.To var err error tolerationsRaw := "" if provisioner { - tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, provisionerTolerationsEnv, "") + tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, controller.OperatorSettingConfigMapName, provisionerTolerationsEnv, "") } else { - tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, pluginTolerationsEnv, "") + tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, controller.OperatorSettingConfigMapName, pluginTolerationsEnv, "") } if err != nil { // tolerationsRaw is empty @@ -143,9 +144,9 @@ func getNodeAffinity(clientset kubernetes.Interface, provisioner bool) *corev1.N v1NodeAffinity := &corev1.NodeAffinity{} var err error if provisioner { - nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, provisionerNodeAffinityEnv, "") + nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, controller.OperatorSettingConfigMapName, provisionerNodeAffinityEnv, "") } else { - nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, pluginNodeAffinityEnv, "") + nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, controller.OperatorSettingConfigMapName, pluginNodeAffinityEnv, "") } if err != nil { logger.Warningf("node affinity will not be applied. %v", err) diff --git a/pkg/operator/k8sutil/configmap.go b/pkg/operator/k8sutil/configmap.go index 79ff7188f..015e1872c 100644 --- a/pkg/operator/k8sutil/configmap.go +++ b/pkg/operator/k8sutil/configmap.go @@ -41,11 +41,10 @@ func DeleteConfigMap(clientset kubernetes.Interface, cmName, namespace string, o // GetOperatorSetting gets the operator setting from ConfigMap or Env Var // returns defaultValue if setting is not found -func GetOperatorSetting(clientset kubernetes.Interface, settingName, defaultValue string) (string, error) { +func GetOperatorSetting(clientset kubernetes.Interface, configMapName, settingName, defaultValue string) (string, error) { // config must be in operator pod namespace - const rookCephOperatorSetting string = "rook-ceph-operator-config" namespace := os.Getenv("POD_NAMESPACE") - cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(rookCephOperatorSetting, metav1.GetOptions{}) + cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(configMapName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { if settingValue, ok := os.LookupEnv(settingName); ok { @@ -53,7 +52,7 @@ func GetOperatorSetting(clientset kubernetes.Interface, settingName, defaultValu } return defaultValue, nil } - return defaultValue, fmt.Errorf("error reading ConfigMap %q. %v", rookCephOperatorSetting, err) + return defaultValue, fmt.Errorf("error reading ConfigMap %q. %v", configMapName, err) } if settingValue, ok := cm.Data[settingName]; ok { return settingValue, nil diff --git a/pkg/operator/k8sutil/configmap_test.go b/pkg/operator/k8sutil/configmap_test.go index 600b99725..de204787b 100644 --- a/pkg/operator/k8sutil/configmap_test.go +++ b/pkg/operator/k8sutil/configmap_test.go @@ -60,9 +60,10 @@ func TestDeleteConfigMap(t *testing.T) { func TestGetOperatorSetting(t *testing.T) { k8s := fake.NewSimpleClientset() + operatorSettingConfigMapName := "rook-ceph-operator-config" cm := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: "rook-ceph-operator-config", + Name: operatorSettingConfigMapName, Namespace: "test-namespace", }, Data: map[string]string{ @@ -77,7 +78,7 @@ func TestGetOperatorSetting(t *testing.T) { defaultValue := "" // ConfigMap is not found - setting, err := GetOperatorSetting(k8s, nodeAffinity, defaultValue) + setting, err := GetOperatorSetting(k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue) assert.NoError(t, err) // Env Var doesn't exist @@ -85,7 +86,7 @@ func TestGetOperatorSetting(t *testing.T) { // Env Var exists err = os.Setenv(nodeAffinity, envSettingValue) assert.NoError(t, err) - setting, err = GetOperatorSetting(k8s, nodeAffinity, defaultValue) + setting, err = GetOperatorSetting(k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue) assert.NoError(t, err) assert.Equal(t, envSettingValue, setting) @@ -94,7 +95,7 @@ func TestGetOperatorSetting(t *testing.T) { assert.NoError(t, err) // Setting exists in ConfigMap - setting, err = GetOperatorSetting(k8s, nodeAffinity, defaultValue) + setting, err = GetOperatorSetting(k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue) assert.NoError(t, err) // Env Var exists assert.Equal(t, cmSettingValue, setting) @@ -104,14 +105,14 @@ func TestGetOperatorSetting(t *testing.T) { assert.Equal(t, cmSettingValue, setting) // Setting doesn't exist in ConfigMap - setting, err = GetOperatorSetting(k8s, podAffinity, defaultValue) + setting, err = GetOperatorSetting(k8s, operatorSettingConfigMapName, podAffinity, defaultValue) assert.NoError(t, err) // Env Var doesn't exist assert.Equal(t, defaultValue, setting) // Env Var exists err = os.Setenv(podAffinity, envSettingValue) assert.NoError(t, err) - setting, err = GetOperatorSetting(k8s, podAffinity, defaultValue) + setting, err = GetOperatorSetting(k8s, operatorSettingConfigMapName, podAffinity, defaultValue) assert.NoError(t, err) assert.Equal(t, envSettingValue, setting) } From c54c0d16cf661363b9ed06f4f626367e86a9a757 Mon Sep 17 00:00:00 2001 From: Umanga Chapagain Date: Mon, 24 Feb 2020 18:13:36 +0530 Subject: [PATCH 2/2] Ceph: adds watch to rook-ceph-operator-config ConfigMap watches the rook-ceph-operator-config ConfigMap and updates CSI driver Signed-off-by: Umanga Chapagain --- pkg/operator/ceph/cluster/controller.go | 66 ++++++++++++++------ pkg/operator/ceph/cluster/controller_test.go | 37 ++++++++--- pkg/operator/ceph/operator.go | 29 ++++++--- pkg/operator/k8sutil/configmap_test.go | 11 ++-- pkg/operator/k8sutil/k8sutil.go | 19 ++++++ 5 files changed, 122 insertions(+), 40 deletions(-) diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 6a037a70c..404b6dc0b 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -25,8 +25,6 @@ import ( "sync" "time" - "github.com/rook/rook/pkg/operator/ceph/csi" - "github.com/coreos/pkg/capnslog" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" @@ -40,6 +38,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/osd" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/ceph/csi" "github.com/rook/rook/pkg/operator/ceph/file" "github.com/rook/rook/pkg/operator/ceph/nfs" "github.com/rook/rook/pkg/operator/ceph/object" @@ -97,25 +96,27 @@ var ClusterResource = k8sutil.CustomResource{ // ClusterController controls an instance of a Rook cluster type ClusterController struct { - context *clusterd.Context - volumeAttachment attachment.Attachment - rookImage string - clusterMap map[string]*cluster - addClusterCallbacks []func(*cephv1.ClusterSpec) error - csiConfigMutex *sync.Mutex - nodeStore cache.Store - osdChecker *osd.Monitor + context *clusterd.Context + volumeAttachment attachment.Attachment + rookImage string + clusterMap map[string]*cluster + operatorConfigCallbacks []func() error + addClusterCallbacks []func() error + csiConfigMutex *sync.Mutex + nodeStore cache.Store + osdChecker *osd.Monitor } // NewClusterController create controller for watching cluster custom resources created -func NewClusterController(context *clusterd.Context, rookImage string, volumeAttachment attachment.Attachment, addClusterCallbacks []func(*cephv1.ClusterSpec) error) *ClusterController { +func NewClusterController(context *clusterd.Context, rookImage string, volumeAttachment attachment.Attachment, operatorConfigCallbacks []func() error, addClusterCallbacks []func() error) *ClusterController { return &ClusterController{ - context: context, - volumeAttachment: volumeAttachment, - rookImage: rookImage, - clusterMap: make(map[string]*cluster), - addClusterCallbacks: addClusterCallbacks, - csiConfigMutex: &sync.Mutex{}, + context: context, + volumeAttachment: volumeAttachment, + rookImage: rookImage, + clusterMap: make(map[string]*cluster), + operatorConfigCallbacks: operatorConfigCallbacks, + addClusterCallbacks: addClusterCallbacks, + csiConfigMutex: &sync.Mutex{}, } } @@ -149,10 +150,10 @@ func (c *ClusterController) StartWatch(namespace string, stopCh chan struct{}) { go nodeController.Run(stopCh) + operatorNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar) if disableVal := os.Getenv(disableHotplugEnv); disableVal != "true" { // watch for updates to the device discovery configmap logger.Infof("Enabling hotplug orchestration: %s=%s", disableHotplugEnv, disableVal) - operatorNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar) _, deviceCMController := cache.NewInformer( cache.NewFilteredListWatchFromClient(c.context.Clientset.CoreV1().RESTClient(), "configmaps", operatorNamespace, func(options *metav1.ListOptions) { @@ -172,6 +173,17 @@ func (c *ClusterController) StartWatch(namespace string, stopCh chan struct{}) { } else { logger.Infof("Disabling hotplug orchestration via %s", disableHotplugEnv) } + + // watch for "rook-ceph-operator-config" ConfigMap + k8sutil.StartOperatorSettingsWatch(c.context, operatorNamespace, controller.OperatorSettingConfigMapName, + c.operatorConfigChange, + func(oldObj, newObj interface{}) { + if reflect.DeepEqual(oldObj, newObj) { + return + } + c.operatorConfigChange(newObj) + return + }, nil, stopCh) } func (c *ClusterController) StopWatch() { @@ -188,6 +200,22 @@ func (c *ClusterController) GetClusterCount() int { // ************************************************************************************************ // Add event functions // ************************************************************************************************ +func (c *ClusterController) operatorConfigChange(obj interface{}) { + cm, ok := obj.(*v1.ConfigMap) + if !ok { + logger.Warningf("Expected ConfigMap but handler received %T. %#v", obj, obj) + return + } + + logger.Infof("ConfigMap %q changes detected. Updating configurations", cm.Name) + for _, callback := range c.operatorConfigCallbacks { + if err := callback(); err != nil { + logger.Errorf("%v", err) + } + } + return +} + func (c *ClusterController) onK8sNodeAdd(obj interface{}) { newNode, ok := obj.(*v1.Node) if !ok { @@ -251,7 +279,7 @@ func (c *ClusterController) onAdd(obj interface{}) { logger.Infof("starting cluster in namespace %s", cluster.Namespace) for _, callback := range c.addClusterCallbacks { - if err := callback(cluster.Spec); err != nil { + if err := callback(); err != nil { logger.Errorf("%v", err) } } diff --git a/pkg/operator/ceph/cluster/controller_test.go b/pkg/operator/ceph/cluster/controller_test.go index ae63c367d..f2556d28c 100644 --- a/pkg/operator/ceph/cluster/controller_test.go +++ b/pkg/operator/ceph/cluster/controller_test.go @@ -80,14 +80,20 @@ func TestClusterDeleteFlexEnabled(t *testing.T) { }, } - addCallbacks := []func(clusterSpec *cephv1.ClusterSpec) error{ - func(clusterSpec *cephv1.ClusterSpec) error { + operatorConfigCallbacks := []func() error{ + func() error { + logger.Infof("test success callback") + return nil + }, + } + addCallbacks := []func() error{ + func() error { logger.Infof("test success callback") return nil }, } // create the cluster controller and tell it that the cluster has been deleted - controller := NewClusterController(context, "", volumeAttachmentController, addCallbacks) + controller := NewClusterController(context, "", volumeAttachmentController, operatorConfigCallbacks, addCallbacks) clusterToDelete := &cephv1.CephCluster{ObjectMeta: metav1.ObjectMeta{Namespace: clusterName}} controller.handleDelete(clusterToDelete, time.Microsecond) @@ -117,8 +123,14 @@ func TestClusterDeleteFlexDisabled(t *testing.T) { }, } - addCallbacks := []func(clusterSpec *cephv1.ClusterSpec) error{ - func(clusterSpec *cephv1.ClusterSpec) error { + operatorConfigCallbacks := []func() error{ + func() error { + logger.Infof("test success callback") + return nil + }, + } + addCallbacks := []func() error{ + func() error { logger.Infof("test success callback") os.Setenv("ROOK_ENABLE_FLEX_DRIVER", "true") os.Setenv(k8sutil.PodNamespaceEnvVar, rookSystemNamespace) @@ -127,7 +139,7 @@ func TestClusterDeleteFlexDisabled(t *testing.T) { }, } // create the cluster controller and tell it that the cluster has been deleted - controller := NewClusterController(context, "", volumeAttachmentController, addCallbacks) + controller := NewClusterController(context, "", volumeAttachmentController, operatorConfigCallbacks, addCallbacks) clusterToDelete := &cephv1.CephCluster{ObjectMeta: metav1.ObjectMeta{Namespace: clusterName}} controller.handleDelete(clusterToDelete, time.Microsecond) @@ -198,13 +210,20 @@ func TestRemoveFinalizer(t *testing.T) { Clientset: clientset, RookClientset: rookfake.NewSimpleClientset(), } - addCallbacks := []func(clusterSpec *cephv1.ClusterSpec) error{ - func(clusterSpec *cephv1.ClusterSpec) error { + operatorConfigCallbacks := []func() error{ + func() error { + logger.Infof("test success callback") + return nil + }, + } + addCallbacks := []func() error{ + func() error { logger.Infof("test success callback") return errors.New("test failed callback") }, } - controller := NewClusterController(context, "", &attachment.MockAttachment{}, addCallbacks) + + controller := NewClusterController(context, "", &attachment.MockAttachment{}, operatorConfigCallbacks, addCallbacks) // ***************************************** // start with a current version ceph cluster diff --git a/pkg/operator/ceph/operator.go b/pkg/operator/ceph/operator.go index e49370073..dd07a04d9 100644 --- a/pkg/operator/ceph/operator.go +++ b/pkg/operator/ceph/operator.go @@ -25,7 +25,6 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume" "github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment" @@ -96,10 +95,13 @@ func New(context *clusterd.Context, volumeAttachmentWrapper attachment.Attachmen rookImage: rookImage, securityAccount: securityAccount, } - addCallbacks := []func(*cephv1.ClusterSpec) error{ - o.startSystemDaemons, + operatorConfigCallbacks := []func() error{ + o.updateDrivers, } - o.clusterController = cluster.NewClusterController(context, rookImage, volumeAttachmentWrapper, addCallbacks) + addCallbacks := []func() error{ + o.startDrivers, + } + o.clusterController = cluster.NewClusterController(context, rookImage, volumeAttachmentWrapper, operatorConfigCallbacks, addCallbacks) return o } @@ -165,11 +167,26 @@ func (o *Operator) Run() error { } } -func (o *Operator) startSystemDaemons(clusterSpec *cephv1.ClusterSpec) error { +func (o *Operator) startDrivers() error { if o.delayedDaemonsStarted { return nil } + o.delayedDaemonsStarted = true + if err := o.updateDrivers(); err != nil { + o.delayedDaemonsStarted = false // unset because failed to updateDrivers + return err + } + + return nil +} + +func (o *Operator) updateDrivers() error { + // Skipping CSI driver update since the first cluster hasn't been started yet + if !o.delayedDaemonsStarted { + return nil + } + if o.operatorNamespace == "" { return errors.Errorf("rook operator namespace is not provided. expose it via downward API in the rook operator manifest file using environment variable %s", k8sutil.PodNamespaceEnvVar) } @@ -211,7 +228,5 @@ func (o *Operator) startSystemDaemons(clusterSpec *cephv1.ClusterSpec) error { return errors.Wrapf(err, "failed to start Ceph csi drivers") } logger.Infof("successfully started Ceph CSI driver(s)") - - o.delayedDaemonsStarted = true return nil } diff --git a/pkg/operator/k8sutil/configmap_test.go b/pkg/operator/k8sutil/configmap_test.go index de204787b..6e2aa240a 100644 --- a/pkg/operator/k8sutil/configmap_test.go +++ b/pkg/operator/k8sutil/configmap_test.go @@ -60,21 +60,22 @@ func TestDeleteConfigMap(t *testing.T) { func TestGetOperatorSetting(t *testing.T) { k8s := fake.NewSimpleClientset() - operatorSettingConfigMapName := "rook-ceph-operator-config" + operatorSettingConfigMapName := "rook-operator-config" + testNamespace := "test-namespace" cm := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: operatorSettingConfigMapName, - Namespace: "test-namespace", + Namespace: testNamespace, }, Data: map[string]string{ - "NODE_AFFINITY": "storage=rook, ceph", + "NODE_AFFINITY": "storage=rook, worker", }, } nodeAffinity := "NODE_AFFINITY" podAffinity := "POD_AFFINITY" envSettingValue := "role=storage-node" - cmSettingValue := "storage=rook, ceph" + cmSettingValue := "storage=rook, worker" defaultValue := "" // ConfigMap is not found @@ -91,7 +92,7 @@ func TestGetOperatorSetting(t *testing.T) { assert.Equal(t, envSettingValue, setting) // ConfigMap is found - _, err = k8s.CoreV1().ConfigMaps("test-namespace").Create(cm) + _, err = k8s.CoreV1().ConfigMaps(testNamespace).Create(cm) assert.NoError(t, err) // Setting exists in ConfigMap diff --git a/pkg/operator/k8sutil/k8sutil.go b/pkg/operator/k8sutil/k8sutil.go index d2b39eb4d..790d5f794 100644 --- a/pkg/operator/k8sutil/k8sutil.go +++ b/pkg/operator/k8sutil/k8sutil.go @@ -26,12 +26,15 @@ import ( "time" "github.com/coreos/pkg/capnslog" + "github.com/rook/rook/pkg/clusterd" rookversion "github.com/rook/rook/pkg/version" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/version" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" ) var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-k8sutil") @@ -168,3 +171,19 @@ func validateLabelValue(value string) string { } return sanitized } + +// StartOperatorSettingsWatch starts the watch for Operator Settings ConfigMap +func StartOperatorSettingsWatch(context *clusterd.Context, operatorNamespace, operatorSettingConfigMapName string, + addFunc func(obj interface{}), updateFunc func(oldObj, newObj interface{}), deleteFunc func(obj interface{}), stopCh chan struct{}) { + _, cacheController := cache.NewInformer(cache.NewFilteredListWatchFromClient(context.Clientset.CoreV1().RESTClient(), + "configmaps", operatorNamespace, func(options *metav1.ListOptions) { + options.FieldSelector = fmt.Sprintf("%s=%s", "metadata.name", operatorSettingConfigMapName) + }), &v1.ConfigMap{}, + 0, + cache.ResourceEventHandlerFuncs{ + AddFunc: addFunc, + UpdateFunc: updateFunc, + DeleteFunc: deleteFunc, + }) + go cacheController.Run(stopCh) +}