diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 874b45173..7bb53ebb3 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -18,6 +18,7 @@ - Added the dashboard `port` configuration setting. - Added the dashboard `ssl` configuration setting. - Added Ceph CSI driver deployments on Kubernetes 1.13 and above. +- New Kubernetes nodes or nodes which are not tainted `NoSchedule` anymore get added automatically to the existing rook cluster if `useAllNodes` is set. [Issue #2208](https://github.com/rook/rook/issues/2208) ## Breaking Changes diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index c49185a80..221339211 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -23,6 +23,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -36,7 +37,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/rbd" "github.com/rook/rook/pkg/operator/k8sutil" batch "k8s.io/api/batch/v1" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -53,13 +54,17 @@ var ( ) type cluster struct { - Info *cephconfig.ClusterInfo - context *clusterd.Context - Namespace string - Spec *cephv1.ClusterSpec - mons *mon.Cluster - stopCh chan struct{} - ownerRef metav1.OwnerReference + Info *cephconfig.ClusterInfo + context *clusterd.Context + Namespace string + Spec *cephv1.ClusterSpec + mons *mon.Cluster + stopCh chan struct{} + ownerRef metav1.OwnerReference + orchestrationRunning bool + orchestrationPending bool + orchRunMux sync.Mutex + orchPenMux sync.Mutex } func newCluster(c *cephv1.CephCluster, context *clusterd.Context) *cluster { @@ -67,12 +72,14 @@ func newCluster(c *cephv1.CephCluster, context *clusterd.Context) *cluster { // at this phase of the cluster creation process, the identity components of the cluster are // not yet established. we reserve this struct which is filled in as soon as the cluster's // identity can be established. - Info: nil, - Namespace: c.Namespace, - Spec: &c.Spec, - context: context, - stopCh: make(chan struct{}), - ownerRef: ClusterOwnerRef(c.Namespace, string(c.UID)), + Info: nil, + Namespace: c.Namespace, + Spec: &c.Spec, + context: context, + stopCh: make(chan struct{}), + ownerRef: ClusterOwnerRef(c.Namespace, string(c.UID)), + orchestrationRunning: false, + orchestrationPending: false, } } @@ -133,71 +140,85 @@ func (c *cluster) detectCephMajorVersion(image string, timeout time.Duration) (s } func (c *cluster) createInstance(rookImage string) error { - - // Create a configmap for overriding ceph config settings - // These settings should only be modified by a user after they are initialized - placeholderConfig := map[string]string{ - k8sutil.ConfigOverrideVal: "", - } - cm := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: k8sutil.ConfigOverrideName, - }, - Data: placeholderConfig, - } - k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &cm.ObjectMeta, &c.ownerRef) - - _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(cm) - if err != nil && !errors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create override configmap %s. %+v", c.Namespace, err) + if c.checkSetOrchestrationRunning() == true { + logger.Debugf("As createInstance is currently running added this request as pending.") + c.setOrchestrationPending() + return nil } - // Start the mon pods - c.mons = mon.New(c.Info, c.context, c.Namespace, - c.Spec.DataDirHostPath, rookImage, c.Spec.CephVersion, c.Spec.Mon, - cephv1.GetMonPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork, - cephv1.GetMonResources(c.Spec.Resources), c.ownerRef) - clusterInfo, err := c.mons.Start() - if err != nil { - return fmt.Errorf("failed to start the mons. %+v", err) - } - c.Info = clusterInfo // mons return the cluster's info + defer c.unsetOrchestrationRunning() + initRun := true + for c.checkUnsetOrchestrationPending() == true || initRun == true { + initRun = false + // Use a DeepCopy of the spec to avoid using an inconsistent data-set + spec := c.Spec.DeepCopy() - // The cluster Identity must be established at this point - if !c.Info.IsInitialized() { - return fmt.Errorf("the cluster identity was not established: %+v", c.Info) + // Create a configmap for overriding ceph config settings + // These settings should only be modified by a user after they are initialized + placeholderConfig := map[string]string{ + k8sutil.ConfigOverrideVal: "", + } + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: k8sutil.ConfigOverrideName, + }, + Data: placeholderConfig, + } + k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &cm.ObjectMeta, &c.ownerRef) + + _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(cm) + if err != nil && !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create override configmap %s. %+v", c.Namespace, err) + } + + // Start the mon pods + c.mons = mon.New(c.Info, c.context, c.Namespace, + spec.DataDirHostPath, rookImage, spec.CephVersion, spec.Mon, + cephv1.GetMonPlacement(spec.Placement), spec.Network.HostNetwork, + cephv1.GetMonResources(spec.Resources), c.ownerRef) + clusterInfo, err := c.mons.Start() + if err != nil { + return fmt.Errorf("failed to start the mons. %+v", err) + } + c.Info = clusterInfo // mons return the cluster's info + + // The cluster Identity must be established at this point + if !c.Info.IsInitialized() { + return fmt.Errorf("the cluster identity was not established: %+v", c.Info) + } + + err = c.createInitialCrushMap() + if err != nil { + return fmt.Errorf("failed to create initial crushmap: %+v", err) + } + + mgrs := mgr.New(c.Info, c.context, c.Namespace, rookImage, + spec.CephVersion, cephv1.GetMgrPlacement(spec.Placement), spec.Network.HostNetwork, + spec.Dashboard, cephv1.GetMgrResources(spec.Resources), c.ownerRef) + err = mgrs.Start() + if err != nil { + return fmt.Errorf("failed to start the ceph mgr. %+v", err) + } + + // Start the OSDs + osds := osd.New(c.context, c.Namespace, rookImage, spec.CephVersion, spec.Storage, spec.DataDirHostPath, + cephv1.GetOSDPlacement(spec.Placement), spec.Network.HostNetwork, cephv1.GetOSDResources(spec.Resources), c.ownerRef) + err = osds.Start() + if err != nil { + return fmt.Errorf("failed to start the osds. %+v", err) + } + + // Start the rbd mirroring daemon(s) + rbdmirror := rbd.New(c.Info, c.context, c.Namespace, rookImage, spec.CephVersion, cephv1.GetRBDMirrorPlacement(spec.Placement), + spec.Network.HostNetwork, spec.RBDMirroring, cephv1.GetRBDMirrorResources(spec.Resources), c.ownerRef) + err = rbdmirror.Start() + if err != nil { + return fmt.Errorf("failed to start the rbd mirrors. %+v", err) + } + + logger.Infof("Done creating rook instance in namespace %s", c.Namespace) } - err = c.createInitialCrushMap() - if err != nil { - return fmt.Errorf("failed to create initial crushmap: %+v", err) - } - - mgrs := mgr.New(c.Info, c.context, c.Namespace, rookImage, - c.Spec.CephVersion, cephv1.GetMgrPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork, - c.Spec.Dashboard, cephv1.GetMgrResources(c.Spec.Resources), c.ownerRef) - err = mgrs.Start() - if err != nil { - return fmt.Errorf("failed to start the ceph mgr. %+v", err) - } - - // Start the OSDs - osds := osd.New(c.context, c.Namespace, rookImage, c.Spec.CephVersion, c.Spec.Storage, c.Spec.DataDirHostPath, - cephv1.GetOSDPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork, cephv1.GetOSDResources(c.Spec.Resources), c.ownerRef) - err = osds.Start() - if err != nil { - return fmt.Errorf("failed to start the osds. %+v", err) - } - - // Start the rbd mirroring daemon(s) - rbdmirror := rbd.New(c.Info, c.context, c.Namespace, rookImage, c.Spec.CephVersion, cephv1.GetRBDMirrorPlacement(c.Spec.Placement), - c.Spec.Network.HostNetwork, c.Spec.RBDMirroring, cephv1.GetRBDMirrorResources(c.Spec.Resources), c.ownerRef) - err = rbdmirror.Start() - if err != nil { - return fmt.Errorf("failed to start the rbd mirrors. %+v", err) - } - - logger.Infof("Done creating rook instance in namespace %s", c.Namespace) return nil } @@ -352,3 +373,49 @@ func versionSupported(version string) bool { } return false } + +func (c *cluster) unsetOrchestrationRunning() { + c.orchRunMux.Lock() + c.orchestrationRunning = false + c.orchRunMux.Unlock() +} + +func (c *cluster) setOrchestrationPending() { + c.orchPenMux.Lock() + c.orchestrationPending = true + c.orchPenMux.Unlock() +} + +func (c *cluster) getOrchestrationRunning() bool { + c.orchRunMux.Lock() + defer c.orchRunMux.Unlock() + return c.orchestrationRunning +} + +func (c *cluster) getOrchestrationPending() bool { + c.orchPenMux.Lock() + defer c.orchPenMux.Unlock() + return c.orchestrationPending +} + +// make checking and setting orchestrationRunning "atomic" +func (c *cluster) checkSetOrchestrationRunning() bool { + defer c.orchRunMux.Unlock() + c.orchRunMux.Lock() + if c.orchestrationRunning == false { + c.orchestrationRunning = true + return false + } + return true +} + +// make checking and unsetting orchestrationPending "atomic" +func (c *cluster) checkUnsetOrchestrationPending() bool { + defer c.orchPenMux.Unlock() + c.orchPenMux.Lock() + if c.orchestrationPending == true { + c.orchestrationPending = false + return true + } + return false +} diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 2f13bae93..0e997a0b3 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -29,20 +29,23 @@ import ( cephbeta "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment" - "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/cluster/osd" "github.com/rook/rook/pkg/operator/ceph/file" "github.com/rook/rook/pkg/operator/ceph/nfs" "github.com/rook/rook/pkg/operator/ceph/object" - "github.com/rook/rook/pkg/operator/ceph/object/user" + objectuser "github.com/rook/rook/pkg/operator/ceph/object/user" "github.com/rook/rook/pkg/operator/ceph/pool" "github.com/rook/rook/pkg/operator/k8sutil" + v1 "k8s.io/api/core/v1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/tools/cache" + "k8s.io/kubernetes/pkg/kubelet/apis" ) const ( @@ -116,6 +119,29 @@ func (c *ClusterController) StartWatch(namespace string, stopCh chan struct{}) e watcher := opkit.NewWatcher(ClusterResource, namespace, resourceHandlerFuncs, c.context.RookClientset.CephV1().RESTClient()) go watcher.Watch(&cephv1.CephCluster{}, stopCh) + // watch for events on new/updated K8s nodes, too + + lwNodes := &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return c.context.Clientset.CoreV1().Nodes().List(options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return c.context.Clientset.CoreV1().Nodes().Watch(options) + }, + } + + _, nodeController := cache.NewInformer( + lwNodes, + &v1.Node{}, + 0, + cache.ResourceEventHandlerFuncs{ + AddFunc: c.onK8sNodeAdd, + UpdateFunc: c.onK8sNodeUpdate, + DeleteFunc: nil, + }, + ) + go nodeController.Run(stopCh) + // watch for events on all legacy types too c.watchLegacyClusters(namespace, stopCh, resourceHandlerFuncs) @@ -132,6 +158,37 @@ func (c *ClusterController) StopWatch() { // ************************************************************************************************ // Add event functions // ************************************************************************************************ +func (c *ClusterController) onK8sNodeAdd(obj interface{}) { + newNode, ok := obj.(*v1.Node) + if !ok { + logger.Warningf("Expected NodeList but handler received %#v", obj) + } + + if k8sutil.GetNodeSchedulable(*newNode) == false { + logger.Debugf("Skipping cluster update. Added node %s is unschedulable", newNode.Labels[apis.LabelHostname]) + return + } + + for _, cluster := range c.clusterMap { + if cluster.Spec.Storage.UseAllNodes == false { + logger.Debugf("Skipping -> Do not use all Nodes") + continue + } + + if valid, _ := k8sutil.ValidNode(*newNode, cluster.Spec.Placement.All()); valid == true { + logger.Debugf("Adding %s to cluster %s", newNode.Labels[apis.LabelHostname], cluster.Namespace) + err := cluster.createInstance(c.rookImage) + if err != nil { + logger.Errorf("Failed to update cluster in namespace %s. Was not able to add %s. %+v", cluster.Namespace, newNode.Labels[apis.LabelHostname], err) + } + } else { + logger.Infof("Could not add host %s . It is not valid", newNode.Labels[apis.LabelHostname]) + continue + } + logger.Infof("Added %s to cluster %s", newNode.Labels[apis.LabelHostname], cluster.Namespace) + } +} + func (c *ClusterController) onAdd(obj interface{}) { clusterObj, migrationNeeded, err := getClusterObject(obj) if err != nil { @@ -262,6 +319,52 @@ func (c *ClusterController) onAdd(obj interface{}) { // ************************************************************************************************ // Update event functions // ************************************************************************************************ +func (c *ClusterController) onK8sNodeUpdate(oldObj, newObj interface{}) { + // skip forced resyncs + if reflect.DeepEqual(oldObj, newObj) { + return + } + + // Checking for nodes where NoSchedule-Taint got removed + newNode, ok := newObj.(*v1.Node) + if !ok { + logger.Warningf("Expected Node but handler received %#v", newObj) + return + } + + oldNode, ok := oldObj.(*v1.Node) + if !ok { + logger.Warningf("Expected Node but handler received %#v", oldObj) + return + } + + if k8sutil.GetNodeSchedulable(*newNode) == false { + logger.Debugf("Skipping cluster update. Updated node %s is unschedulable", newNode.Labels[apis.LabelHostname]) + return + } + + // Checking for nodes where NoSchedule-Taint got removed + if k8sutil.GetNodeSchedulable(*oldNode) == true { + logger.Debugf("Skipping cluster update. Updated node %s was and it is still schedulable", oldNode.Labels[apis.LabelHostname]) + return + } + + for _, cluster := range c.clusterMap { + if valid, _ := k8sutil.ValidNode(*newNode, cephv1.GetOSDPlacement(cluster.Spec.Placement)); valid == true { + logger.Debugf("Adding %s to cluster %s", newNode.Labels[apis.LabelHostname], cluster.Namespace) + err := cluster.createInstance(c.rookImage) + if err != nil { + logger.Errorf("Failed adding the updated node %s to cluster in namespace %s. %+v", newNode.Labels[apis.LabelHostname], cluster.Namespace, err) + continue + } + } else { + logger.Infof("Updated node %s is not valid and could not get added to cluster in namespace %s.", newNode.Labels[apis.LabelHostname], cluster.Namespace) + continue + } + logger.Infof("Added updated node %s to cluster %s", newNode.Labels[apis.LabelHostname], cluster.Namespace) + } +} + func (c *ClusterController) onUpdate(oldObj, newObj interface{}) { oldClust, _, err := getClusterObject(oldObj) if err != nil { diff --git a/pkg/operator/k8sutil/node.go b/pkg/operator/k8sutil/node.go index 9c933315d..97dcc2a33 100644 --- a/pkg/operator/k8sutil/node.go +++ b/pkg/operator/k8sutil/node.go @@ -21,7 +21,7 @@ import ( "fmt" rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" @@ -144,3 +144,16 @@ func GetNodeHostNames(clientset kubernetes.Interface) (map[string]string, error) } return nodeMap, nil } + +// GetNodeSchedulable returns a boolean if the node is tainted as Schedulable or not +// true -> Node is schedulable +// false -> Node is unschedulable +func GetNodeSchedulable(node v1.Node) bool { + for i := range node.Spec.Taints { + if node.Spec.Taints[i].Effect == "NoSchedule" { + logger.Debugf("Node %s is unschedulable", node.Labels[apis.LabelHostname]) + return false + } + } + return true +}