Merge pull request #4909 from umangachapagain/config-watcher
Ceph: adds watch to rook-ceph-operator-config ConfigMap
This commit is contained in:
8 files changed
+139
-53
No files matched your search
@@ -25,8 +25,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/ceph/csi"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
@@ -40,6 +38,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/ceph/csi"
|
||||
"github.com/rook/rook/pkg/operator/ceph/file"
|
||||
"github.com/rook/rook/pkg/operator/ceph/nfs"
|
||||
"github.com/rook/rook/pkg/operator/ceph/object"
|
||||
@@ -97,25 +96,27 @@ var ClusterResource = k8sutil.CustomResource{
|
||||
|
||||
// ClusterController controls an instance of a Rook cluster
|
||||
type ClusterController struct {
|
||||
context *clusterd.Context
|
||||
volumeAttachment attachment.Attachment
|
||||
rookImage string
|
||||
clusterMap map[string]*cluster
|
||||
addClusterCallbacks []func(*cephv1.ClusterSpec) error
|
||||
csiConfigMutex *sync.Mutex
|
||||
nodeStore cache.Store
|
||||
osdChecker *osd.Monitor
|
||||
context *clusterd.Context
|
||||
volumeAttachment attachment.Attachment
|
||||
rookImage string
|
||||
clusterMap map[string]*cluster
|
||||
operatorConfigCallbacks []func() error
|
||||
addClusterCallbacks []func() error
|
||||
csiConfigMutex *sync.Mutex
|
||||
nodeStore cache.Store
|
||||
osdChecker *osd.Monitor
|
||||
}
|
||||
|
||||
// NewClusterController create controller for watching cluster custom resources created
|
||||
func NewClusterController(context *clusterd.Context, rookImage string, volumeAttachment attachment.Attachment, addClusterCallbacks []func(*cephv1.ClusterSpec) error) *ClusterController {
|
||||
func NewClusterController(context *clusterd.Context, rookImage string, volumeAttachment attachment.Attachment, operatorConfigCallbacks []func() error, addClusterCallbacks []func() error) *ClusterController {
|
||||
return &ClusterController{
|
||||
context: context,
|
||||
volumeAttachment: volumeAttachment,
|
||||
rookImage: rookImage,
|
||||
clusterMap: make(map[string]*cluster),
|
||||
addClusterCallbacks: addClusterCallbacks,
|
||||
csiConfigMutex: &sync.Mutex{},
|
||||
context: context,
|
||||
volumeAttachment: volumeAttachment,
|
||||
rookImage: rookImage,
|
||||
clusterMap: make(map[string]*cluster),
|
||||
operatorConfigCallbacks: operatorConfigCallbacks,
|
||||
addClusterCallbacks: addClusterCallbacks,
|
||||
csiConfigMutex: &sync.Mutex{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,10 +150,10 @@ func (c *ClusterController) StartWatch(namespace string, stopCh chan struct{}) {
|
||||
|
||||
go nodeController.Run(stopCh)
|
||||
|
||||
operatorNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
|
||||
if disableVal := os.Getenv(disableHotplugEnv); disableVal != "true" {
|
||||
// watch for updates to the device discovery configmap
|
||||
logger.Infof("Enabling hotplug orchestration: %s=%s", disableHotplugEnv, disableVal)
|
||||
operatorNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
|
||||
_, deviceCMController := cache.NewInformer(
|
||||
cache.NewFilteredListWatchFromClient(c.context.Clientset.CoreV1().RESTClient(),
|
||||
"configmaps", operatorNamespace, func(options *metav1.ListOptions) {
|
||||
@@ -172,6 +173,17 @@ func (c *ClusterController) StartWatch(namespace string, stopCh chan struct{}) {
|
||||
} else {
|
||||
logger.Infof("Disabling hotplug orchestration via %s", disableHotplugEnv)
|
||||
}
|
||||
|
||||
// watch for "rook-ceph-operator-config" ConfigMap
|
||||
k8sutil.StartOperatorSettingsWatch(c.context, operatorNamespace, controller.OperatorSettingConfigMapName,
|
||||
c.operatorConfigChange,
|
||||
func(oldObj, newObj interface{}) {
|
||||
if reflect.DeepEqual(oldObj, newObj) {
|
||||
return
|
||||
}
|
||||
c.operatorConfigChange(newObj)
|
||||
return
|
||||
}, nil, stopCh)
|
||||
}
|
||||
|
||||
func (c *ClusterController) StopWatch() {
|
||||
@@ -188,6 +200,22 @@ func (c *ClusterController) GetClusterCount() int {
|
||||
// ************************************************************************************************
|
||||
// Add event functions
|
||||
// ************************************************************************************************
|
||||
func (c *ClusterController) operatorConfigChange(obj interface{}) {
|
||||
cm, ok := obj.(*v1.ConfigMap)
|
||||
if !ok {
|
||||
logger.Warningf("Expected ConfigMap but handler received %T. %#v", obj, obj)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof("ConfigMap %q changes detected. Updating configurations", cm.Name)
|
||||
for _, callback := range c.operatorConfigCallbacks {
|
||||
if err := callback(); err != nil {
|
||||
logger.Errorf("%v", err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (c *ClusterController) onK8sNodeAdd(obj interface{}) {
|
||||
newNode, ok := obj.(*v1.Node)
|
||||
if !ok {
|
||||
@@ -251,7 +279,7 @@ func (c *ClusterController) onAdd(obj interface{}) {
|
||||
logger.Infof("starting cluster in namespace %s", cluster.Namespace)
|
||||
|
||||
for _, callback := range c.addClusterCallbacks {
|
||||
if err := callback(cluster.Spec); err != nil {
|
||||
if err := callback(); err != nil {
|
||||
logger.Errorf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,14 +80,20 @@ func TestClusterDeleteFlexEnabled(t *testing.T) {
|
||||
|
||||
},
|
||||
}
|
||||
addCallbacks := []func(clusterSpec *cephv1.ClusterSpec) error{
|
||||
func(clusterSpec *cephv1.ClusterSpec) error {
|
||||
operatorConfigCallbacks := []func() error{
|
||||
func() error {
|
||||
logger.Infof("test success callback")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
addCallbacks := []func() error{
|
||||
func() error {
|
||||
logger.Infof("test success callback")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
// create the cluster controller and tell it that the cluster has been deleted
|
||||
controller := NewClusterController(context, "", volumeAttachmentController, addCallbacks)
|
||||
controller := NewClusterController(context, "", volumeAttachmentController, operatorConfigCallbacks, addCallbacks)
|
||||
clusterToDelete := &cephv1.CephCluster{ObjectMeta: metav1.ObjectMeta{Namespace: clusterName}}
|
||||
controller.handleDelete(clusterToDelete, time.Microsecond)
|
||||
|
||||
@@ -117,8 +123,14 @@ func TestClusterDeleteFlexDisabled(t *testing.T) {
|
||||
|
||||
},
|
||||
}
|
||||
addCallbacks := []func(clusterSpec *cephv1.ClusterSpec) error{
|
||||
func(clusterSpec *cephv1.ClusterSpec) error {
|
||||
operatorConfigCallbacks := []func() error{
|
||||
func() error {
|
||||
logger.Infof("test success callback")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
addCallbacks := []func() error{
|
||||
func() error {
|
||||
logger.Infof("test success callback")
|
||||
os.Setenv("ROOK_ENABLE_FLEX_DRIVER", "true")
|
||||
os.Setenv(k8sutil.PodNamespaceEnvVar, rookSystemNamespace)
|
||||
@@ -127,7 +139,7 @@ func TestClusterDeleteFlexDisabled(t *testing.T) {
|
||||
},
|
||||
}
|
||||
// create the cluster controller and tell it that the cluster has been deleted
|
||||
controller := NewClusterController(context, "", volumeAttachmentController, addCallbacks)
|
||||
controller := NewClusterController(context, "", volumeAttachmentController, operatorConfigCallbacks, addCallbacks)
|
||||
clusterToDelete := &cephv1.CephCluster{ObjectMeta: metav1.ObjectMeta{Namespace: clusterName}}
|
||||
controller.handleDelete(clusterToDelete, time.Microsecond)
|
||||
|
||||
@@ -198,13 +210,20 @@ func TestRemoveFinalizer(t *testing.T) {
|
||||
Clientset: clientset,
|
||||
RookClientset: rookfake.NewSimpleClientset(),
|
||||
}
|
||||
addCallbacks := []func(clusterSpec *cephv1.ClusterSpec) error{
|
||||
func(clusterSpec *cephv1.ClusterSpec) error {
|
||||
operatorConfigCallbacks := []func() error{
|
||||
func() error {
|
||||
logger.Infof("test success callback")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
addCallbacks := []func() error{
|
||||
func() error {
|
||||
logger.Infof("test success callback")
|
||||
return errors.New("test failed callback")
|
||||
},
|
||||
}
|
||||
controller := NewClusterController(context, "", &attachment.MockAttachment{}, addCallbacks)
|
||||
|
||||
controller := NewClusterController(context, "", &attachment.MockAttachment{}, operatorConfigCallbacks, addCallbacks)
|
||||
|
||||
// *****************************************
|
||||
// start with a current version ceph cluster
|
||||
|
||||
@@ -31,6 +31,9 @@ import (
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
)
|
||||
|
||||
// OperatorSettingConfigMapName refers to ConfigMap that configures rook ceph operator
|
||||
const OperatorSettingConfigMapName string = "rook-ceph-operator-config"
|
||||
|
||||
var (
|
||||
// ImmediateRetryResult Return this for a immediate retry of the reconciliation loop with the same request object.
|
||||
ImmediateRetryResult = reconcile.Result{Requeue: true}
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
|
||||
"github.com/ghodss/yaml"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
k8sutil "github.com/rook/rook/pkg/operator/k8sutil"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
@@ -111,9 +112,9 @@ func getToleration(clientset kubernetes.Interface, provisioner bool) []corev1.To
|
||||
var err error
|
||||
tolerationsRaw := ""
|
||||
if provisioner {
|
||||
tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, provisionerTolerationsEnv, "")
|
||||
tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, controller.OperatorSettingConfigMapName, provisionerTolerationsEnv, "")
|
||||
} else {
|
||||
tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, pluginTolerationsEnv, "")
|
||||
tolerationsRaw, err = k8sutil.GetOperatorSetting(clientset, controller.OperatorSettingConfigMapName, pluginTolerationsEnv, "")
|
||||
}
|
||||
if err != nil {
|
||||
// tolerationsRaw is empty
|
||||
@@ -143,9 +144,9 @@ func getNodeAffinity(clientset kubernetes.Interface, provisioner bool) *corev1.N
|
||||
v1NodeAffinity := &corev1.NodeAffinity{}
|
||||
var err error
|
||||
if provisioner {
|
||||
nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, provisionerNodeAffinityEnv, "")
|
||||
nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, controller.OperatorSettingConfigMapName, provisionerNodeAffinityEnv, "")
|
||||
} else {
|
||||
nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, pluginNodeAffinityEnv, "")
|
||||
nodeAffinity, err = k8sutil.GetOperatorSetting(clientset, controller.OperatorSettingConfigMapName, pluginNodeAffinityEnv, "")
|
||||
}
|
||||
if err != nil {
|
||||
logger.Warningf("node affinity will not be applied. %v", err)
|
||||
|
||||
@@ -25,7 +25,6 @@ import (
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/pkg/errors"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment"
|
||||
@@ -96,10 +95,13 @@ func New(context *clusterd.Context, volumeAttachmentWrapper attachment.Attachmen
|
||||
rookImage: rookImage,
|
||||
securityAccount: securityAccount,
|
||||
}
|
||||
addCallbacks := []func(*cephv1.ClusterSpec) error{
|
||||
o.startSystemDaemons,
|
||||
operatorConfigCallbacks := []func() error{
|
||||
o.updateDrivers,
|
||||
}
|
||||
o.clusterController = cluster.NewClusterController(context, rookImage, volumeAttachmentWrapper, addCallbacks)
|
||||
addCallbacks := []func() error{
|
||||
o.startDrivers,
|
||||
}
|
||||
o.clusterController = cluster.NewClusterController(context, rookImage, volumeAttachmentWrapper, operatorConfigCallbacks, addCallbacks)
|
||||
return o
|
||||
}
|
||||
|
||||
@@ -165,11 +167,26 @@ func (o *Operator) Run() error {
|
||||
}
|
||||
}
|
||||
|
||||
func (o *Operator) startSystemDaemons(clusterSpec *cephv1.ClusterSpec) error {
|
||||
func (o *Operator) startDrivers() error {
|
||||
if o.delayedDaemonsStarted {
|
||||
return nil
|
||||
}
|
||||
|
||||
o.delayedDaemonsStarted = true
|
||||
if err := o.updateDrivers(); err != nil {
|
||||
o.delayedDaemonsStarted = false // unset because failed to updateDrivers
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *Operator) updateDrivers() error {
|
||||
// Skipping CSI driver update since the first cluster hasn't been started yet
|
||||
if !o.delayedDaemonsStarted {
|
||||
return nil
|
||||
}
|
||||
|
||||
if o.operatorNamespace == "" {
|
||||
return errors.Errorf("rook operator namespace is not provided. expose it via downward API in the rook operator manifest file using environment variable %s", k8sutil.PodNamespaceEnvVar)
|
||||
}
|
||||
@@ -211,7 +228,5 @@ func (o *Operator) startSystemDaemons(clusterSpec *cephv1.ClusterSpec) error {
|
||||
return errors.Wrapf(err, "failed to start Ceph csi drivers")
|
||||
}
|
||||
logger.Infof("successfully started Ceph CSI driver(s)")
|
||||
|
||||
o.delayedDaemonsStarted = true
|
||||
return nil
|
||||
}
|
||||
@@ -41,11 +41,10 @@ func DeleteConfigMap(clientset kubernetes.Interface, cmName, namespace string, o
|
||||
|
||||
// GetOperatorSetting gets the operator setting from ConfigMap or Env Var
|
||||
// returns defaultValue if setting is not found
|
||||
func GetOperatorSetting(clientset kubernetes.Interface, settingName, defaultValue string) (string, error) {
|
||||
func GetOperatorSetting(clientset kubernetes.Interface, configMapName, settingName, defaultValue string) (string, error) {
|
||||
// config must be in operator pod namespace
|
||||
const rookCephOperatorSetting string = "rook-ceph-operator-config"
|
||||
namespace := os.Getenv("POD_NAMESPACE")
|
||||
cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(rookCephOperatorSetting, metav1.GetOptions{})
|
||||
cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(configMapName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
if settingValue, ok := os.LookupEnv(settingName); ok {
|
||||
@@ -53,7 +52,7 @@ func GetOperatorSetting(clientset kubernetes.Interface, settingName, defaultValu
|
||||
}
|
||||
return defaultValue, nil
|
||||
}
|
||||
return defaultValue, fmt.Errorf("error reading ConfigMap %q. %v", rookCephOperatorSetting, err)
|
||||
return defaultValue, fmt.Errorf("error reading ConfigMap %q. %v", configMapName, err)
|
||||
}
|
||||
if settingValue, ok := cm.Data[settingName]; ok {
|
||||
return settingValue, nil
|
||||
|
||||
@@ -60,24 +60,26 @@ func TestDeleteConfigMap(t *testing.T) {
|
||||
func TestGetOperatorSetting(t *testing.T) {
|
||||
k8s := fake.NewSimpleClientset()
|
||||
|
||||
operatorSettingConfigMapName := "rook-operator-config"
|
||||
testNamespace := "test-namespace"
|
||||
cm := &v1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "rook-ceph-operator-config",
|
||||
Namespace: "test-namespace",
|
||||
Name: operatorSettingConfigMapName,
|
||||
Namespace: testNamespace,
|
||||
},
|
||||
Data: map[string]string{
|
||||
"NODE_AFFINITY": "storage=rook, ceph",
|
||||
"NODE_AFFINITY": "storage=rook, worker",
|
||||
},
|
||||
}
|
||||
|
||||
nodeAffinity := "NODE_AFFINITY"
|
||||
podAffinity := "POD_AFFINITY"
|
||||
envSettingValue := "role=storage-node"
|
||||
cmSettingValue := "storage=rook, ceph"
|
||||
cmSettingValue := "storage=rook, worker"
|
||||
defaultValue := ""
|
||||
|
||||
// ConfigMap is not found
|
||||
setting, err := GetOperatorSetting(k8s, nodeAffinity, defaultValue)
|
||||
setting, err := GetOperatorSetting(k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Env Var doesn't exist
|
||||
@@ -85,16 +87,16 @@ func TestGetOperatorSetting(t *testing.T) {
|
||||
// Env Var exists
|
||||
err = os.Setenv(nodeAffinity, envSettingValue)
|
||||
assert.NoError(t, err)
|
||||
setting, err = GetOperatorSetting(k8s, nodeAffinity, defaultValue)
|
||||
setting, err = GetOperatorSetting(k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, envSettingValue, setting)
|
||||
|
||||
// ConfigMap is found
|
||||
_, err = k8s.CoreV1().ConfigMaps("test-namespace").Create(cm)
|
||||
_, err = k8s.CoreV1().ConfigMaps(testNamespace).Create(cm)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Setting exists in ConfigMap
|
||||
setting, err = GetOperatorSetting(k8s, nodeAffinity, defaultValue)
|
||||
setting, err = GetOperatorSetting(k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue)
|
||||
assert.NoError(t, err)
|
||||
// Env Var exists
|
||||
assert.Equal(t, cmSettingValue, setting)
|
||||
@@ -104,14 +106,14 @@ func TestGetOperatorSetting(t *testing.T) {
|
||||
assert.Equal(t, cmSettingValue, setting)
|
||||
|
||||
// Setting doesn't exist in ConfigMap
|
||||
setting, err = GetOperatorSetting(k8s, podAffinity, defaultValue)
|
||||
setting, err = GetOperatorSetting(k8s, operatorSettingConfigMapName, podAffinity, defaultValue)
|
||||
assert.NoError(t, err)
|
||||
// Env Var doesn't exist
|
||||
assert.Equal(t, defaultValue, setting)
|
||||
// Env Var exists
|
||||
err = os.Setenv(podAffinity, envSettingValue)
|
||||
assert.NoError(t, err)
|
||||
setting, err = GetOperatorSetting(k8s, podAffinity, defaultValue)
|
||||
setting, err = GetOperatorSetting(k8s, operatorSettingConfigMapName, podAffinity, defaultValue)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, envSettingValue, setting)
|
||||
}
|
||||
@@ -26,12 +26,15 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
rookversion "github.com/rook/rook/pkg/version"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/validation"
|
||||
"k8s.io/apimachinery/pkg/util/version"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
)
|
||||
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-k8sutil")
|
||||
@@ -168,3 +171,19 @@ func validateLabelValue(value string) string {
|
||||
}
|
||||
return sanitized
|
||||
}
|
||||
|
||||
// StartOperatorSettingsWatch starts the watch for Operator Settings ConfigMap
|
||||
func StartOperatorSettingsWatch(context *clusterd.Context, operatorNamespace, operatorSettingConfigMapName string,
|
||||
addFunc func(obj interface{}), updateFunc func(oldObj, newObj interface{}), deleteFunc func(obj interface{}), stopCh chan struct{}) {
|
||||
_, cacheController := cache.NewInformer(cache.NewFilteredListWatchFromClient(context.Clientset.CoreV1().RESTClient(),
|
||||
"configmaps", operatorNamespace, func(options *metav1.ListOptions) {
|
||||
options.FieldSelector = fmt.Sprintf("%s=%s", "metadata.name", operatorSettingConfigMapName)
|
||||
}), &v1.ConfigMap{},
|
||||
0,
|
||||
cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: addFunc,
|
||||
UpdateFunc: updateFunc,
|
||||
DeleteFunc: deleteFunc,
|
||||
})
|
||||
go cacheController.Run(stopCh)
|
||||
}
|
||||
Reference in new issue
Block a user