issue #2208 OSDs not automatically started when adding nodes to existing cluster

Signed-off-by: Stefan Haas <shaas@suse.com>
This commit is contained in:
Stefan Haas committed 2019-03-14 18:54:40 +01:00
1 parent 73c24ddf72
commit 1f1dbc7461
4 files changed
+261 -77

No files matched your search

+1
View File
@@ -18,6 +18,7 @@
- Added the dashboard `port` configuration setting.
- Added the dashboard `ssl` configuration setting.
- Added Ceph CSI driver deployments on Kubernetes 1.13 and above.
- New Kubernetes nodes or nodes which are not tainted `NoSchedule` anymore get added automatically to the existing rook cluster if `useAllNodes` is set. [Issue #2208](https://github.com/rook/rook/issues/2208)
## Breaking Changes
+141 -74
View File
@@ -23,6 +23,7 @@ import (
"sort"
"strconv"
"strings"
"sync"
"time"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
@@ -36,7 +37,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/cluster/rbd"
"github.com/rook/rook/pkg/operator/k8sutil"
batch "k8s.io/api/batch/v1"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -53,13 +54,17 @@ var (
)
type cluster struct {
Info *cephconfig.ClusterInfo
context *clusterd.Context
Namespace string
Spec *cephv1.ClusterSpec
mons *mon.Cluster
stopCh chan struct{}
ownerRef metav1.OwnerReference
Info *cephconfig.ClusterInfo
context *clusterd.Context
Namespace string
Spec *cephv1.ClusterSpec
mons *mon.Cluster
stopCh chan struct{}
ownerRef metav1.OwnerReference
orchestrationRunning bool
orchestrationPending bool
orchRunMux sync.Mutex
orchPenMux sync.Mutex
}
func newCluster(c *cephv1.CephCluster, context *clusterd.Context) *cluster {
@@ -67,12 +72,14 @@ func newCluster(c *cephv1.CephCluster, context *clusterd.Context) *cluster {
// at this phase of the cluster creation process, the identity components of the cluster are
// not yet established. we reserve this struct which is filled in as soon as the cluster's
// identity can be established.
Info: nil,
Namespace: c.Namespace,
Spec: &c.Spec,
context: context,
stopCh: make(chan struct{}),
ownerRef: ClusterOwnerRef(c.Namespace, string(c.UID)),
Info: nil,
Namespace: c.Namespace,
Spec: &c.Spec,
context: context,
stopCh: make(chan struct{}),
ownerRef: ClusterOwnerRef(c.Namespace, string(c.UID)),
orchestrationRunning: false,
orchestrationPending: false,
}
}
@@ -133,71 +140,85 @@ func (c *cluster) detectCephMajorVersion(image string, timeout time.Duration) (s
}
func (c *cluster) createInstance(rookImage string) error {
// Create a configmap for overriding ceph config settings
// These settings should only be modified by a user after they are initialized
placeholderConfig := map[string]string{
k8sutil.ConfigOverrideVal: "",
}
cm := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: k8sutil.ConfigOverrideName,
},
Data: placeholderConfig,
}
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &cm.ObjectMeta, &c.ownerRef)
_, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(cm)
if err != nil && !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create override configmap %s. %+v", c.Namespace, err)
if c.checkSetOrchestrationRunning() == true {
logger.Debugf("As createInstance is currently running added this request as pending.")
c.setOrchestrationPending()
return nil
}
// Start the mon pods
c.mons = mon.New(c.Info, c.context, c.Namespace,
c.Spec.DataDirHostPath, rookImage, c.Spec.CephVersion, c.Spec.Mon,
cephv1.GetMonPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork,
cephv1.GetMonResources(c.Spec.Resources), c.ownerRef)
clusterInfo, err := c.mons.Start()
if err != nil {
return fmt.Errorf("failed to start the mons. %+v", err)
}
c.Info = clusterInfo // mons return the cluster's info
defer c.unsetOrchestrationRunning()
initRun := true
for c.checkUnsetOrchestrationPending() == true || initRun == true {
initRun = false
// Use a DeepCopy of the spec to avoid using an inconsistent data-set
spec := c.Spec.DeepCopy()
// The cluster Identity must be established at this point
if !c.Info.IsInitialized() {
return fmt.Errorf("the cluster identity was not established: %+v", c.Info)
// Create a configmap for overriding ceph config settings
// These settings should only be modified by a user after they are initialized
placeholderConfig := map[string]string{
k8sutil.ConfigOverrideVal: "",
}
cm := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: k8sutil.ConfigOverrideName,
},
Data: placeholderConfig,
}
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &cm.ObjectMeta, &c.ownerRef)
_, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(cm)
if err != nil && !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create override configmap %s. %+v", c.Namespace, err)
}
// Start the mon pods
c.mons = mon.New(c.Info, c.context, c.Namespace,
spec.DataDirHostPath, rookImage, spec.CephVersion, spec.Mon,
cephv1.GetMonPlacement(spec.Placement), spec.Network.HostNetwork,
cephv1.GetMonResources(spec.Resources), c.ownerRef)
clusterInfo, err := c.mons.Start()
if err != nil {
return fmt.Errorf("failed to start the mons. %+v", err)
}
c.Info = clusterInfo // mons return the cluster's info
// The cluster Identity must be established at this point
if !c.Info.IsInitialized() {
return fmt.Errorf("the cluster identity was not established: %+v", c.Info)
}
err = c.createInitialCrushMap()
if err != nil {
return fmt.Errorf("failed to create initial crushmap: %+v", err)
}
mgrs := mgr.New(c.Info, c.context, c.Namespace, rookImage,
spec.CephVersion, cephv1.GetMgrPlacement(spec.Placement), spec.Network.HostNetwork,
spec.Dashboard, cephv1.GetMgrResources(spec.Resources), c.ownerRef)
err = mgrs.Start()
if err != nil {
return fmt.Errorf("failed to start the ceph mgr. %+v", err)
}
// Start the OSDs
osds := osd.New(c.context, c.Namespace, rookImage, spec.CephVersion, spec.Storage, spec.DataDirHostPath,
cephv1.GetOSDPlacement(spec.Placement), spec.Network.HostNetwork, cephv1.GetOSDResources(spec.Resources), c.ownerRef)
err = osds.Start()
if err != nil {
return fmt.Errorf("failed to start the osds. %+v", err)
}
// Start the rbd mirroring daemon(s)
rbdmirror := rbd.New(c.Info, c.context, c.Namespace, rookImage, spec.CephVersion, cephv1.GetRBDMirrorPlacement(spec.Placement),
spec.Network.HostNetwork, spec.RBDMirroring, cephv1.GetRBDMirrorResources(spec.Resources), c.ownerRef)
err = rbdmirror.Start()
if err != nil {
return fmt.Errorf("failed to start the rbd mirrors. %+v", err)
}
logger.Infof("Done creating rook instance in namespace %s", c.Namespace)
}
err = c.createInitialCrushMap()
if err != nil {
return fmt.Errorf("failed to create initial crushmap: %+v", err)
}
mgrs := mgr.New(c.Info, c.context, c.Namespace, rookImage,
c.Spec.CephVersion, cephv1.GetMgrPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork,
c.Spec.Dashboard, cephv1.GetMgrResources(c.Spec.Resources), c.ownerRef)
err = mgrs.Start()
if err != nil {
return fmt.Errorf("failed to start the ceph mgr. %+v", err)
}
// Start the OSDs
osds := osd.New(c.context, c.Namespace, rookImage, c.Spec.CephVersion, c.Spec.Storage, c.Spec.DataDirHostPath,
cephv1.GetOSDPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork, cephv1.GetOSDResources(c.Spec.Resources), c.ownerRef)
err = osds.Start()
if err != nil {
return fmt.Errorf("failed to start the osds. %+v", err)
}
// Start the rbd mirroring daemon(s)
rbdmirror := rbd.New(c.Info, c.context, c.Namespace, rookImage, c.Spec.CephVersion, cephv1.GetRBDMirrorPlacement(c.Spec.Placement),
c.Spec.Network.HostNetwork, c.Spec.RBDMirroring, cephv1.GetRBDMirrorResources(c.Spec.Resources), c.ownerRef)
err = rbdmirror.Start()
if err != nil {
return fmt.Errorf("failed to start the rbd mirrors. %+v", err)
}
logger.Infof("Done creating rook instance in namespace %s", c.Namespace)
return nil
}
@@ -352,3 +373,49 @@ func versionSupported(version string) bool {
}
return false
}
func (c *cluster) unsetOrchestrationRunning() {
c.orchRunMux.Lock()
c.orchestrationRunning = false
c.orchRunMux.Unlock()
}
func (c *cluster) setOrchestrationPending() {
c.orchPenMux.Lock()
c.orchestrationPending = true
c.orchPenMux.Unlock()
}
func (c *cluster) getOrchestrationRunning() bool {
c.orchRunMux.Lock()
defer c.orchRunMux.Unlock()
return c.orchestrationRunning
}
func (c *cluster) getOrchestrationPending() bool {
c.orchPenMux.Lock()
defer c.orchPenMux.Unlock()
return c.orchestrationPending
}
// make checking and setting orchestrationRunning "atomic"
func (c *cluster) checkSetOrchestrationRunning() bool {
defer c.orchRunMux.Unlock()
c.orchRunMux.Lock()
if c.orchestrationRunning == false {
c.orchestrationRunning = true
return false
}
return true
}
// make checking and unsetting orchestrationPending "atomic"
func (c *cluster) checkUnsetOrchestrationPending() bool {
defer c.orchPenMux.Unlock()
c.orchPenMux.Lock()
if c.orchestrationPending == true {
c.orchestrationPending = false
return true
}
return false
}
+105 -2
View File
@@ -29,20 +29,23 @@ import (
cephbeta "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment"
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
"github.com/rook/rook/pkg/operator/ceph/file"
"github.com/rook/rook/pkg/operator/ceph/nfs"
"github.com/rook/rook/pkg/operator/ceph/object"
"github.com/rook/rook/pkg/operator/ceph/object/user"
objectuser "github.com/rook/rook/pkg/operator/ceph/object/user"
"github.com/rook/rook/pkg/operator/ceph/pool"
"github.com/rook/rook/pkg/operator/k8sutil"
v1 "k8s.io/api/core/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
"k8s.io/kubernetes/pkg/kubelet/apis"
)
const (
@@ -116,6 +119,29 @@ func (c *ClusterController) StartWatch(namespace string, stopCh chan struct{}) e
watcher := opkit.NewWatcher(ClusterResource, namespace, resourceHandlerFuncs, c.context.RookClientset.CephV1().RESTClient())
go watcher.Watch(&cephv1.CephCluster{}, stopCh)
// watch for events on new/updated K8s nodes, too
lwNodes := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
return c.context.Clientset.CoreV1().Nodes().List(options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return c.context.Clientset.CoreV1().Nodes().Watch(options)
},
}
_, nodeController := cache.NewInformer(
lwNodes,
&v1.Node{},
0,
cache.ResourceEventHandlerFuncs{
AddFunc: c.onK8sNodeAdd,
UpdateFunc: c.onK8sNodeUpdate,
DeleteFunc: nil,
},
)
go nodeController.Run(stopCh)
// watch for events on all legacy types too
c.watchLegacyClusters(namespace, stopCh, resourceHandlerFuncs)
@@ -132,6 +158,37 @@ func (c *ClusterController) StopWatch() {
// ************************************************************************************************
// Add event functions
// ************************************************************************************************
func (c *ClusterController) onK8sNodeAdd(obj interface{}) {
newNode, ok := obj.(*v1.Node)
if !ok {
logger.Warningf("Expected NodeList but handler received %#v", obj)
}
if k8sutil.GetNodeSchedulable(*newNode) == false {
logger.Debugf("Skipping cluster update. Added node %s is unschedulable", newNode.Labels[apis.LabelHostname])
return
}
for _, cluster := range c.clusterMap {
if cluster.Spec.Storage.UseAllNodes == false {
logger.Debugf("Skipping -> Do not use all Nodes")
continue
}
if valid, _ := k8sutil.ValidNode(*newNode, cluster.Spec.Placement.All()); valid == true {
logger.Debugf("Adding %s to cluster %s", newNode.Labels[apis.LabelHostname], cluster.Namespace)
err := cluster.createInstance(c.rookImage)
if err != nil {
logger.Errorf("Failed to update cluster in namespace %s. Was not able to add %s. %+v", cluster.Namespace, newNode.Labels[apis.LabelHostname], err)
}
} else {
logger.Infof("Could not add host %s . It is not valid", newNode.Labels[apis.LabelHostname])
continue
}
logger.Infof("Added %s to cluster %s", newNode.Labels[apis.LabelHostname], cluster.Namespace)
}
}
func (c *ClusterController) onAdd(obj interface{}) {
clusterObj, migrationNeeded, err := getClusterObject(obj)
if err != nil {
@@ -262,6 +319,52 @@ func (c *ClusterController) onAdd(obj interface{}) {
// ************************************************************************************************
// Update event functions
// ************************************************************************************************
func (c *ClusterController) onK8sNodeUpdate(oldObj, newObj interface{}) {
// skip forced resyncs
if reflect.DeepEqual(oldObj, newObj) {
return
}
// Checking for nodes where NoSchedule-Taint got removed
newNode, ok := newObj.(*v1.Node)
if !ok {
logger.Warningf("Expected Node but handler received %#v", newObj)
return
}
oldNode, ok := oldObj.(*v1.Node)
if !ok {
logger.Warningf("Expected Node but handler received %#v", oldObj)
return
}
if k8sutil.GetNodeSchedulable(*newNode) == false {
logger.Debugf("Skipping cluster update. Updated node %s is unschedulable", newNode.Labels[apis.LabelHostname])
return
}
// Checking for nodes where NoSchedule-Taint got removed
if k8sutil.GetNodeSchedulable(*oldNode) == true {
logger.Debugf("Skipping cluster update. Updated node %s was and it is still schedulable", oldNode.Labels[apis.LabelHostname])
return
}
for _, cluster := range c.clusterMap {
if valid, _ := k8sutil.ValidNode(*newNode, cephv1.GetOSDPlacement(cluster.Spec.Placement)); valid == true {
logger.Debugf("Adding %s to cluster %s", newNode.Labels[apis.LabelHostname], cluster.Namespace)
err := cluster.createInstance(c.rookImage)
if err != nil {
logger.Errorf("Failed adding the updated node %s to cluster in namespace %s. %+v", newNode.Labels[apis.LabelHostname], cluster.Namespace, err)
continue
}
} else {
logger.Infof("Updated node %s is not valid and could not get added to cluster in namespace %s.", newNode.Labels[apis.LabelHostname], cluster.Namespace)
continue
}
logger.Infof("Added updated node %s to cluster %s", newNode.Labels[apis.LabelHostname], cluster.Namespace)
}
}
func (c *ClusterController) onUpdate(oldObj, newObj interface{}) {
oldClust, _, err := getClusterObject(oldObj)
if err != nil {
+14 -1
View File
@@ -21,7 +21,7 @@ import (
"fmt"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
@@ -144,3 +144,16 @@ func GetNodeHostNames(clientset kubernetes.Interface) (map[string]string, error)
}
return nodeMap, nil
}
// GetNodeSchedulable returns a boolean if the node is tainted as Schedulable or not
// true -> Node is schedulable
// false -> Node is unschedulable
func GetNodeSchedulable(node v1.Node) bool {
for i := range node.Spec.Taints {
if node.Spec.Taints[i].Effect == "NoSchedule" {
logger.Debugf("Node %s is unschedulable", node.Labels[apis.LabelHostname])
return false
}
}
return true
}