Merge pull request #4909 from umangachapagain/config-watcher

Ceph: adds watch to rook-ceph-operator-config ConfigMap
This commit is contained in:
Sébastien Han authored and GitHub committed 2020-03-11 09:34:16 +01:00
commit 04979f8c60
8 files changed
+139 -53

No files matched your search

+47 -19
View File
@@ -25,8 +25,6 @@ import (
"sync"
"time"
"github.com/rook/rook/pkg/operator/ceph/csi"
"github.com/coreos/pkg/capnslog"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
@@ -40,6 +38,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/ceph/csi"
"github.com/rook/rook/pkg/operator/ceph/file"
"github.com/rook/rook/pkg/operator/ceph/nfs"
"github.com/rook/rook/pkg/operator/ceph/object"
@@ -97,25 +96,27 @@ var ClusterResource = k8sutil.CustomResource{
// ClusterController controls an instance of a Rook cluster
type ClusterController struct {
context *clusterd.Context
volumeAttachment attachment.Attachment
rookImage string
clusterMap map[string]*cluster
addClusterCallbacks []func(*cephv1.ClusterSpec) error
csiConfigMutex *sync.Mutex
nodeStore cache.Store
osdChecker *osd.Monitor
context *clusterd.Context
volumeAttachment attachment.Attachment
rookImage string
clusterMap map[string]*cluster
operatorConfigCallbacks []func() error
addClusterCallbacks []func() error
csiConfigMutex *sync.Mutex
nodeStore cache.Store
osdChecker *osd.Monitor
}
// NewClusterController create controller for watching cluster custom resources created
func NewClusterController(context *clusterd.Context, rookImage string, volumeAttachment attachment.Attachment, addClusterCallbacks []func(*cephv1.ClusterSpec) error) *ClusterController {
func NewClusterController(context *clusterd.Context, rookImage string, volumeAttachment attachment.Attachment, operatorConfigCallbacks []func() error, addClusterCallbacks []func() error) *ClusterController {
return &ClusterController{
context: context,
volumeAttachment: volumeAttachment,
rookImage: rookImage,
clusterMap: make(map[string]*cluster),
addClusterCallbacks: addClusterCallbacks,
csiConfigMutex: &sync.Mutex{},
context: context,
volumeAttachment: volumeAttachment,
rookImage: rookImage,
clusterMap: make(map[string]*cluster),
operatorConfigCallbacks: operatorConfigCallbacks,
addClusterCallbacks: addClusterCallbacks,
csiConfigMutex: &sync.Mutex{},
}
}
@@ -149,10 +150,10 @@ func (c *ClusterController) StartWatch(namespace string, stopCh chan struct{}) {
go nodeController.Run(stopCh)
operatorNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
if disableVal := os.Getenv(disableHotplugEnv); disableVal != "true" {
// watch for updates to the device discovery configmap
logger.Infof("Enabling hotplug orchestration: %s=%s", disableHotplugEnv, disableVal)
operatorNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
_, deviceCMController := cache.NewInformer(
cache.NewFilteredListWatchFromClient(c.context.Clientset.CoreV1().RESTClient(),
"configmaps", operatorNamespace, func(options *metav1.ListOptions) {
@@ -172,6 +173,17 @@ func (c *ClusterController) StartWatch(namespace string, stopCh chan struct{}) {
} else {
logger.Infof("Disabling hotplug orchestration via %s", disableHotplugEnv)
}
// watch for "rook-ceph-operator-config" ConfigMap
k8sutil.StartOperatorSettingsWatch(c.context, operatorNamespace, controller.OperatorSettingConfigMapName,
c.operatorConfigChange,
func(oldObj, newObj interface{}) {
if reflect.DeepEqual(oldObj, newObj) {
return
}
c.operatorConfigChange(newObj)
return
}, nil, stopCh)
}
func (c *ClusterController) StopWatch() {
@@ -188,6 +200,22 @@ func (c *ClusterController) GetClusterCount() int {
// ************************************************************************************************
// Add event functions
// ************************************************************************************************
func (c *ClusterController) operatorConfigChange(obj interface{}) {
cm, ok := obj.(*v1.ConfigMap)
if !ok {
logger.Warningf("Expected ConfigMap but handler received %T. %#v", obj, obj)
return
}
logger.Infof("ConfigMap %q changes detected. Updating configurations", cm.Name)
for _, callback := range c.operatorConfigCallbacks {
if err := callback(); err != nil {
logger.Errorf("%v", err)
}
}
return
}
func (c *ClusterController) onK8sNodeAdd(obj interface{}) {
newNode, ok := obj.(*v1.Node)
if !ok {
@@ -251,7 +279,7 @@ func (c *ClusterController) onAdd(obj interface{}) {
logger.Infof("starting cluster in namespace %s", cluster.Namespace)
for _, callback := range c.addClusterCallbacks {
if err := callback(cluster.Spec); err != nil {
if err := callback(); err != nil {
logger.Errorf("%v", err)
}
}
+28 -9
View File
@@ -80,14 +80,20 @@ func TestClusterDeleteFlexEnabled(t *testing.T) {
},
}
addCallbacks := []func(clusterSpec *cephv1.ClusterSpec) error{
func(clusterSpec *cephv1.ClusterSpec) error {
operatorConfigCallbacks := []func() error{
func() error {
logger.Infof("test success callback")
return nil
},
}
addCallbacks := []func() error{
func() error {
logger.Infof("test success callback")
return nil
},
}
// create the cluster controller and tell it that the cluster has been deleted
controller := NewClusterController(context, "", volumeAttachmentController, addCallbacks)
controller := NewClusterController(context, "", volumeAttachmentController, operatorConfigCallbacks, addCallbacks)
clusterToDelete := &cephv1.CephCluster{ObjectMeta: metav1.ObjectMeta{Namespace: clusterName}}
controller.handleDelete(clusterToDelete, time.Microsecond)
@@ -117,8 +123,14 @@ func TestClusterDeleteFlexDisabled(t *testing.T) {
},
}
addCallbacks := []func(clusterSpec *cephv1.ClusterSpec) error{
func(clusterSpec *cephv1.ClusterSpec) error {
operatorConfigCallbacks := []func() error{
func() error {
logger.Infof("test success callback")
return nil
},
}
addCallbacks := []func() error{
func() error {
logger.Infof("test success callback")
os.Setenv("ROOK_ENABLE_FLEX_DRIVER", "true")
os.Setenv(k8sutil.PodNamespaceEnvVar, rookSystemNamespace)
@@ -127,7 +139,7 @@ func TestClusterDeleteFlexDisabled(t *testing.T) {
},
}
// create the cluster controller and tell it that the cluster has been deleted
controller := NewClusterController(context, "", volumeAttachmentController, addCallbacks)
controller := NewClusterController(context, "", volumeAttachmentController, operatorConfigCallbacks, addCallbacks)
clusterToDelete := &cephv1.CephCluster{ObjectMeta: metav1.ObjectMeta{Namespace: clusterName}}
controller.handleDelete(clusterToDelete, time.Microsecond)
@@ -198,13 +210,20 @@ func TestRemoveFinalizer(t *testing.T) {
Clientset: clientset,
RookClientset: rookfake.NewSimpleClientset(),
}
addCallbacks := []func(clusterSpec *cephv1.ClusterSpec) error{
func(clusterSpec *cephv1.ClusterSpec) error {
operatorConfigCallbacks := []func() error{
func() error {
logger.Infof("test success callback")
return nil
},
}
addCallbacks := []func() error{
func() error {
logger.Infof("test success callback")
return errors.New("test failed callback")
},
}
controller := NewClusterController(context, "", &attachment.MockAttachment{}, addCallbacks)
controller := NewClusterController(context, "", &attachment.MockAttachment{}, operatorConfigCallbacks, addCallbacks)
// *****************************************
// start with a current version ceph cluster
@@ -31,6 +31,9 @@ import (
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)
// OperatorSettingConfigMapName refers to ConfigMap that configures rook ceph operator
const OperatorSettingConfigMapName string = "rook-ceph-operator-config"
var (
// ImmediateRetryResult Return this for a immediate retry of the reconciliation loop with the same request object.
ImmediateRetryResult = reconcile.Result{Requeue: true}
+5 -4
View File
@@ -27,6 +27,7 @@ import (
"github.com/ghodss/yaml"
"github.com/pkg/errors"
"github.com/rook/rook/pkg/operator/ceph/controller"
k8sutil "github.com/rook/rook/pkg/operator/k8sutil"
apps "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
@@ -111,9 +112,9 @@ func getToleration(clientset kubernetes.Interface, provisioner bool) []corev1.To
var err error
tolerationsRaw := ""
if provisioner {
tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, provisionerTolerationsEnv, "")
tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, controller.OperatorSettingConfigMapName, provisionerTolerationsEnv, "")
} else {
tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, pluginTolerationsEnv, "")
tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, controller.OperatorSettingConfigMapName, pluginTolerationsEnv, "")
}
if err != nil {
// tolerationsRaw is empty
@@ -143,9 +144,9 @@ func getNodeAffinity(clientset kubernetes.Interface, provisioner bool) *corev1.N
v1NodeAffinity := &corev1.NodeAffinity{}
var err error
if provisioner {
nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, provisionerNodeAffinityEnv, "")
nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, controller.OperatorSettingConfigMapName, provisionerNodeAffinityEnv, "")
} else {
nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, pluginNodeAffinityEnv, "")
nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, controller.OperatorSettingConfigMapName, pluginNodeAffinityEnv, "")
}
if err != nil {
logger.Warningf("node affinity will not be applied. %v", err)
+22 -7
View File
@@ -25,7 +25,6 @@ import (
"github.com/coreos/pkg/capnslog"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume"
"github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment"
@@ -96,10 +95,13 @@ func New(context *clusterd.Context, volumeAttachmentWrapper attachment.Attachmen
rookImage: rookImage,
securityAccount: securityAccount,
}
addCallbacks := []func(*cephv1.ClusterSpec) error{
o.startSystemDaemons,
operatorConfigCallbacks := []func() error{
o.updateDrivers,
}
o.clusterController = cluster.NewClusterController(context, rookImage, volumeAttachmentWrapper, addCallbacks)
addCallbacks := []func() error{
o.startDrivers,
}
o.clusterController = cluster.NewClusterController(context, rookImage, volumeAttachmentWrapper, operatorConfigCallbacks, addCallbacks)
return o
}
@@ -165,11 +167,26 @@ func (o *Operator) Run() error {
}
}
func (o *Operator) startSystemDaemons(clusterSpec *cephv1.ClusterSpec) error {
func (o *Operator) startDrivers() error {
if o.delayedDaemonsStarted {
return nil
}
o.delayedDaemonsStarted = true
if err := o.updateDrivers(); err != nil {
o.delayedDaemonsStarted = false // unset because failed to updateDrivers
return err
}
return nil
}
func (o *Operator) updateDrivers() error {
// Skipping CSI driver update since the first cluster hasn't been started yet
if !o.delayedDaemonsStarted {
return nil
}
if o.operatorNamespace == "" {
return errors.Errorf("rook operator namespace is not provided. expose it via downward API in the rook operator manifest file using environment variable %s", k8sutil.PodNamespaceEnvVar)
}
@@ -211,7 +228,5 @@ func (o *Operator) startSystemDaemons(clusterSpec *cephv1.ClusterSpec) error {
return errors.Wrapf(err, "failed to start Ceph csi drivers")
}
logger.Infof("successfully started Ceph CSI driver(s)")
o.delayedDaemonsStarted = true
return nil
}
+3 -4
View File
@@ -41,11 +41,10 @@ func DeleteConfigMap(clientset kubernetes.Interface, cmName, namespace string, o
// GetOperatorSetting gets the operator setting from ConfigMap or Env Var
// returns defaultValue if setting is not found
func GetOperatorSetting(clientset kubernetes.Interface, settingName, defaultValue string) (string, error) {
func GetOperatorSetting(clientset kubernetes.Interface, configMapName, settingName, defaultValue string) (string, error) {
// config must be in operator pod namespace
const rookCephOperatorSetting string = "rook-ceph-operator-config"
namespace := os.Getenv("POD_NAMESPACE")
cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(rookCephOperatorSetting, metav1.GetOptions{})
cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(configMapName, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
if settingValue, ok := os.LookupEnv(settingName); ok {
@@ -53,7 +52,7 @@ func GetOperatorSetting(clientset kubernetes.Interface, settingName, defaultValu
}
return defaultValue, nil
}
return defaultValue, fmt.Errorf("error reading ConfigMap %q. %v", rookCephOperatorSetting, err)
return defaultValue, fmt.Errorf("error reading ConfigMap %q. %v", configMapName, err)
}
if settingValue, ok := cm.Data[settingName]; ok {
return settingValue, nil
+12 -10
View File
@@ -60,24 +60,26 @@ func TestDeleteConfigMap(t *testing.T) {
func TestGetOperatorSetting(t *testing.T) {
k8s := fake.NewSimpleClientset()
operatorSettingConfigMapName := "rook-operator-config"
testNamespace := "test-namespace"
cm := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "rook-ceph-operator-config",
Namespace: "test-namespace",
Name: operatorSettingConfigMapName,
Namespace: testNamespace,
},
Data: map[string]string{
"NODE_AFFINITY": "storage=rook, ceph",
"NODE_AFFINITY": "storage=rook, worker",
},
}
nodeAffinity := "NODE_AFFINITY"
podAffinity := "POD_AFFINITY"
envSettingValue := "role=storage-node"
cmSettingValue := "storage=rook, ceph"
cmSettingValue := "storage=rook, worker"
defaultValue := ""
// ConfigMap is not found
setting, err := GetOperatorSetting(k8s, nodeAffinity, defaultValue)
setting, err := GetOperatorSetting(k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue)
assert.NoError(t, err)
// Env Var doesn't exist
@@ -85,16 +87,16 @@ func TestGetOperatorSetting(t *testing.T) {
// Env Var exists
err = os.Setenv(nodeAffinity, envSettingValue)
assert.NoError(t, err)
setting, err = GetOperatorSetting(k8s, nodeAffinity, defaultValue)
setting, err = GetOperatorSetting(k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue)
assert.NoError(t, err)
assert.Equal(t, envSettingValue, setting)
// ConfigMap is found
_, err = k8s.CoreV1().ConfigMaps("test-namespace").Create(cm)
_, err = k8s.CoreV1().ConfigMaps(testNamespace).Create(cm)
assert.NoError(t, err)
// Setting exists in ConfigMap
setting, err = GetOperatorSetting(k8s, nodeAffinity, defaultValue)
setting, err = GetOperatorSetting(k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue)
assert.NoError(t, err)
// Env Var exists
assert.Equal(t, cmSettingValue, setting)
@@ -104,14 +106,14 @@ func TestGetOperatorSetting(t *testing.T) {
assert.Equal(t, cmSettingValue, setting)
// Setting doesn't exist in ConfigMap
setting, err = GetOperatorSetting(k8s, podAffinity, defaultValue)
setting, err = GetOperatorSetting(k8s, operatorSettingConfigMapName, podAffinity, defaultValue)
assert.NoError(t, err)
// Env Var doesn't exist
assert.Equal(t, defaultValue, setting)
// Env Var exists
err = os.Setenv(podAffinity, envSettingValue)
assert.NoError(t, err)
setting, err = GetOperatorSetting(k8s, podAffinity, defaultValue)
setting, err = GetOperatorSetting(k8s, operatorSettingConfigMapName, podAffinity, defaultValue)
assert.NoError(t, err)
assert.Equal(t, envSettingValue, setting)
}
+19
View File
@@ -26,12 +26,15 @@ import (
"time"
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/pkg/clusterd"
rookversion "github.com/rook/rook/pkg/version"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/validation"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-k8sutil")
@@ -168,3 +171,19 @@ func validateLabelValue(value string) string {
}
return sanitized
}
// StartOperatorSettingsWatch starts the watch for Operator Settings ConfigMap
func StartOperatorSettingsWatch(context *clusterd.Context, operatorNamespace, operatorSettingConfigMapName string,
addFunc func(obj interface{}), updateFunc func(oldObj, newObj interface{}), deleteFunc func(obj interface{}), stopCh chan struct{}) {
_, cacheController := cache.NewInformer(cache.NewFilteredListWatchFromClient(context.Clientset.CoreV1().RESTClient(),
"configmaps", operatorNamespace, func(options *metav1.ListOptions) {
options.FieldSelector = fmt.Sprintf("%s=%s", "metadata.name", operatorSettingConfigMapName)
}), &v1.ConfigMap{},
0,
cache.ResourceEventHandlerFuncs{
AddFunc: addFunc,
UpdateFunc: updateFunc,
DeleteFunc: deleteFunc,
})
go cacheController.Run(stopCh)
}