issue #2208 OSDs not automatically started when adding nodes to existing cluster
Signed-off-by: Stefan Haas <shaas@suse.com>
This commit is contained in:
4 files changed
+261
-77
No files matched your search
@@ -18,6 +18,7 @@
|
||||
- Added the dashboard `port` configuration setting.
|
||||
- Added the dashboard `ssl` configuration setting.
|
||||
- Added Ceph CSI driver deployments on Kubernetes 1.13 and above.
|
||||
- New Kubernetes nodes or nodes which are not tainted `NoSchedule` anymore get added automatically to the existing rook cluster if `useAllNodes` is set. [Issue #2208](https://github.com/rook/rook/issues/2208)
|
||||
|
||||
## Breaking Changes
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
@@ -36,7 +37,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/rbd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
batch "k8s.io/api/batch/v1"
|
||||
"k8s.io/api/core/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
@@ -53,13 +54,17 @@ var (
|
||||
)
|
||||
|
||||
type cluster struct {
|
||||
Info *cephconfig.ClusterInfo
|
||||
context *clusterd.Context
|
||||
Namespace string
|
||||
Spec *cephv1.ClusterSpec
|
||||
mons *mon.Cluster
|
||||
stopCh chan struct{}
|
||||
ownerRef metav1.OwnerReference
|
||||
Info *cephconfig.ClusterInfo
|
||||
context *clusterd.Context
|
||||
Namespace string
|
||||
Spec *cephv1.ClusterSpec
|
||||
mons *mon.Cluster
|
||||
stopCh chan struct{}
|
||||
ownerRef metav1.OwnerReference
|
||||
orchestrationRunning bool
|
||||
orchestrationPending bool
|
||||
orchRunMux sync.Mutex
|
||||
orchPenMux sync.Mutex
|
||||
}
|
||||
|
||||
func newCluster(c *cephv1.CephCluster, context *clusterd.Context) *cluster {
|
||||
@@ -67,12 +72,14 @@ func newCluster(c *cephv1.CephCluster, context *clusterd.Context) *cluster {
|
||||
// at this phase of the cluster creation process, the identity components of the cluster are
|
||||
// not yet established. we reserve this struct which is filled in as soon as the cluster's
|
||||
// identity can be established.
|
||||
Info: nil,
|
||||
Namespace: c.Namespace,
|
||||
Spec: &c.Spec,
|
||||
context: context,
|
||||
stopCh: make(chan struct{}),
|
||||
ownerRef: ClusterOwnerRef(c.Namespace, string(c.UID)),
|
||||
Info: nil,
|
||||
Namespace: c.Namespace,
|
||||
Spec: &c.Spec,
|
||||
context: context,
|
||||
stopCh: make(chan struct{}),
|
||||
ownerRef: ClusterOwnerRef(c.Namespace, string(c.UID)),
|
||||
orchestrationRunning: false,
|
||||
orchestrationPending: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,71 +140,85 @@ func (c *cluster) detectCephMajorVersion(image string, timeout time.Duration) (s
|
||||
}
|
||||
|
||||
func (c *cluster) createInstance(rookImage string) error {
|
||||
|
||||
// Create a configmap for overriding ceph config settings
|
||||
// These settings should only be modified by a user after they are initialized
|
||||
placeholderConfig := map[string]string{
|
||||
k8sutil.ConfigOverrideVal: "",
|
||||
}
|
||||
cm := &v1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: k8sutil.ConfigOverrideName,
|
||||
},
|
||||
Data: placeholderConfig,
|
||||
}
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &cm.ObjectMeta, &c.ownerRef)
|
||||
|
||||
_, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(cm)
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create override configmap %s. %+v", c.Namespace, err)
|
||||
if c.checkSetOrchestrationRunning() == true {
|
||||
logger.Debugf("As createInstance is currently running added this request as pending.")
|
||||
c.setOrchestrationPending()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Start the mon pods
|
||||
c.mons = mon.New(c.Info, c.context, c.Namespace,
|
||||
c.Spec.DataDirHostPath, rookImage, c.Spec.CephVersion, c.Spec.Mon,
|
||||
cephv1.GetMonPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork,
|
||||
cephv1.GetMonResources(c.Spec.Resources), c.ownerRef)
|
||||
clusterInfo, err := c.mons.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the mons. %+v", err)
|
||||
}
|
||||
c.Info = clusterInfo // mons return the cluster's info
|
||||
defer c.unsetOrchestrationRunning()
|
||||
initRun := true
|
||||
for c.checkUnsetOrchestrationPending() == true || initRun == true {
|
||||
initRun = false
|
||||
// Use a DeepCopy of the spec to avoid using an inconsistent data-set
|
||||
spec := c.Spec.DeepCopy()
|
||||
|
||||
// The cluster Identity must be established at this point
|
||||
if !c.Info.IsInitialized() {
|
||||
return fmt.Errorf("the cluster identity was not established: %+v", c.Info)
|
||||
// Create a configmap for overriding ceph config settings
|
||||
// These settings should only be modified by a user after they are initialized
|
||||
placeholderConfig := map[string]string{
|
||||
k8sutil.ConfigOverrideVal: "",
|
||||
}
|
||||
cm := &v1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: k8sutil.ConfigOverrideName,
|
||||
},
|
||||
Data: placeholderConfig,
|
||||
}
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &cm.ObjectMeta, &c.ownerRef)
|
||||
|
||||
_, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(cm)
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create override configmap %s. %+v", c.Namespace, err)
|
||||
}
|
||||
|
||||
// Start the mon pods
|
||||
c.mons = mon.New(c.Info, c.context, c.Namespace,
|
||||
spec.DataDirHostPath, rookImage, spec.CephVersion, spec.Mon,
|
||||
cephv1.GetMonPlacement(spec.Placement), spec.Network.HostNetwork,
|
||||
cephv1.GetMonResources(spec.Resources), c.ownerRef)
|
||||
clusterInfo, err := c.mons.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the mons. %+v", err)
|
||||
}
|
||||
c.Info = clusterInfo // mons return the cluster's info
|
||||
|
||||
// The cluster Identity must be established at this point
|
||||
if !c.Info.IsInitialized() {
|
||||
return fmt.Errorf("the cluster identity was not established: %+v", c.Info)
|
||||
}
|
||||
|
||||
err = c.createInitialCrushMap()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create initial crushmap: %+v", err)
|
||||
}
|
||||
|
||||
mgrs := mgr.New(c.Info, c.context, c.Namespace, rookImage,
|
||||
spec.CephVersion, cephv1.GetMgrPlacement(spec.Placement), spec.Network.HostNetwork,
|
||||
spec.Dashboard, cephv1.GetMgrResources(spec.Resources), c.ownerRef)
|
||||
err = mgrs.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the ceph mgr. %+v", err)
|
||||
}
|
||||
|
||||
// Start the OSDs
|
||||
osds := osd.New(c.context, c.Namespace, rookImage, spec.CephVersion, spec.Storage, spec.DataDirHostPath,
|
||||
cephv1.GetOSDPlacement(spec.Placement), spec.Network.HostNetwork, cephv1.GetOSDResources(spec.Resources), c.ownerRef)
|
||||
err = osds.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the osds. %+v", err)
|
||||
}
|
||||
|
||||
// Start the rbd mirroring daemon(s)
|
||||
rbdmirror := rbd.New(c.Info, c.context, c.Namespace, rookImage, spec.CephVersion, cephv1.GetRBDMirrorPlacement(spec.Placement),
|
||||
spec.Network.HostNetwork, spec.RBDMirroring, cephv1.GetRBDMirrorResources(spec.Resources), c.ownerRef)
|
||||
err = rbdmirror.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the rbd mirrors. %+v", err)
|
||||
}
|
||||
|
||||
logger.Infof("Done creating rook instance in namespace %s", c.Namespace)
|
||||
}
|
||||
|
||||
err = c.createInitialCrushMap()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create initial crushmap: %+v", err)
|
||||
}
|
||||
|
||||
mgrs := mgr.New(c.Info, c.context, c.Namespace, rookImage,
|
||||
c.Spec.CephVersion, cephv1.GetMgrPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork,
|
||||
c.Spec.Dashboard, cephv1.GetMgrResources(c.Spec.Resources), c.ownerRef)
|
||||
err = mgrs.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the ceph mgr. %+v", err)
|
||||
}
|
||||
|
||||
// Start the OSDs
|
||||
osds := osd.New(c.context, c.Namespace, rookImage, c.Spec.CephVersion, c.Spec.Storage, c.Spec.DataDirHostPath,
|
||||
cephv1.GetOSDPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork, cephv1.GetOSDResources(c.Spec.Resources), c.ownerRef)
|
||||
err = osds.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the osds. %+v", err)
|
||||
}
|
||||
|
||||
// Start the rbd mirroring daemon(s)
|
||||
rbdmirror := rbd.New(c.Info, c.context, c.Namespace, rookImage, c.Spec.CephVersion, cephv1.GetRBDMirrorPlacement(c.Spec.Placement),
|
||||
c.Spec.Network.HostNetwork, c.Spec.RBDMirroring, cephv1.GetRBDMirrorResources(c.Spec.Resources), c.ownerRef)
|
||||
err = rbdmirror.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the rbd mirrors. %+v", err)
|
||||
}
|
||||
|
||||
logger.Infof("Done creating rook instance in namespace %s", c.Namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -352,3 +373,49 @@ func versionSupported(version string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *cluster) unsetOrchestrationRunning() {
|
||||
c.orchRunMux.Lock()
|
||||
c.orchestrationRunning = false
|
||||
c.orchRunMux.Unlock()
|
||||
}
|
||||
|
||||
func (c *cluster) setOrchestrationPending() {
|
||||
c.orchPenMux.Lock()
|
||||
c.orchestrationPending = true
|
||||
c.orchPenMux.Unlock()
|
||||
}
|
||||
|
||||
func (c *cluster) getOrchestrationRunning() bool {
|
||||
c.orchRunMux.Lock()
|
||||
defer c.orchRunMux.Unlock()
|
||||
return c.orchestrationRunning
|
||||
}
|
||||
|
||||
func (c *cluster) getOrchestrationPending() bool {
|
||||
c.orchPenMux.Lock()
|
||||
defer c.orchPenMux.Unlock()
|
||||
return c.orchestrationPending
|
||||
}
|
||||
|
||||
// make checking and setting orchestrationRunning "atomic"
|
||||
func (c *cluster) checkSetOrchestrationRunning() bool {
|
||||
defer c.orchRunMux.Unlock()
|
||||
c.orchRunMux.Lock()
|
||||
if c.orchestrationRunning == false {
|
||||
c.orchestrationRunning = true
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// make checking and unsetting orchestrationPending "atomic"
|
||||
func (c *cluster) checkUnsetOrchestrationPending() bool {
|
||||
defer c.orchPenMux.Unlock()
|
||||
c.orchPenMux.Lock()
|
||||
if c.orchestrationPending == true {
|
||||
c.orchestrationPending = false
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -29,20 +29,23 @@ import (
|
||||
cephbeta "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
|
||||
"github.com/rook/rook/pkg/operator/ceph/file"
|
||||
"github.com/rook/rook/pkg/operator/ceph/nfs"
|
||||
"github.com/rook/rook/pkg/operator/ceph/object"
|
||||
"github.com/rook/rook/pkg/operator/ceph/object/user"
|
||||
objectuser "github.com/rook/rook/pkg/operator/ceph/object/user"
|
||||
"github.com/rook/rook/pkg/operator/ceph/pool"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
"k8s.io/kubernetes/pkg/kubelet/apis"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -116,6 +119,29 @@ func (c *ClusterController) StartWatch(namespace string, stopCh chan struct{}) e
|
||||
watcher := opkit.NewWatcher(ClusterResource, namespace, resourceHandlerFuncs, c.context.RookClientset.CephV1().RESTClient())
|
||||
go watcher.Watch(&cephv1.CephCluster{}, stopCh)
|
||||
|
||||
// watch for events on new/updated K8s nodes, too
|
||||
|
||||
lwNodes := &cache.ListWatch{
|
||||
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
|
||||
return c.context.Clientset.CoreV1().Nodes().List(options)
|
||||
},
|
||||
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
|
||||
return c.context.Clientset.CoreV1().Nodes().Watch(options)
|
||||
},
|
||||
}
|
||||
|
||||
_, nodeController := cache.NewInformer(
|
||||
lwNodes,
|
||||
&v1.Node{},
|
||||
0,
|
||||
cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: c.onK8sNodeAdd,
|
||||
UpdateFunc: c.onK8sNodeUpdate,
|
||||
DeleteFunc: nil,
|
||||
},
|
||||
)
|
||||
go nodeController.Run(stopCh)
|
||||
|
||||
// watch for events on all legacy types too
|
||||
c.watchLegacyClusters(namespace, stopCh, resourceHandlerFuncs)
|
||||
|
||||
@@ -132,6 +158,37 @@ func (c *ClusterController) StopWatch() {
|
||||
// ************************************************************************************************
|
||||
// Add event functions
|
||||
// ************************************************************************************************
|
||||
func (c *ClusterController) onK8sNodeAdd(obj interface{}) {
|
||||
newNode, ok := obj.(*v1.Node)
|
||||
if !ok {
|
||||
logger.Warningf("Expected NodeList but handler received %#v", obj)
|
||||
}
|
||||
|
||||
if k8sutil.GetNodeSchedulable(*newNode) == false {
|
||||
logger.Debugf("Skipping cluster update. Added node %s is unschedulable", newNode.Labels[apis.LabelHostname])
|
||||
return
|
||||
}
|
||||
|
||||
for _, cluster := range c.clusterMap {
|
||||
if cluster.Spec.Storage.UseAllNodes == false {
|
||||
logger.Debugf("Skipping -> Do not use all Nodes")
|
||||
continue
|
||||
}
|
||||
|
||||
if valid, _ := k8sutil.ValidNode(*newNode, cluster.Spec.Placement.All()); valid == true {
|
||||
logger.Debugf("Adding %s to cluster %s", newNode.Labels[apis.LabelHostname], cluster.Namespace)
|
||||
err := cluster.createInstance(c.rookImage)
|
||||
if err != nil {
|
||||
logger.Errorf("Failed to update cluster in namespace %s. Was not able to add %s. %+v", cluster.Namespace, newNode.Labels[apis.LabelHostname], err)
|
||||
}
|
||||
} else {
|
||||
logger.Infof("Could not add host %s . It is not valid", newNode.Labels[apis.LabelHostname])
|
||||
continue
|
||||
}
|
||||
logger.Infof("Added %s to cluster %s", newNode.Labels[apis.LabelHostname], cluster.Namespace)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ClusterController) onAdd(obj interface{}) {
|
||||
clusterObj, migrationNeeded, err := getClusterObject(obj)
|
||||
if err != nil {
|
||||
@@ -262,6 +319,52 @@ func (c *ClusterController) onAdd(obj interface{}) {
|
||||
// ************************************************************************************************
|
||||
// Update event functions
|
||||
// ************************************************************************************************
|
||||
func (c *ClusterController) onK8sNodeUpdate(oldObj, newObj interface{}) {
|
||||
// skip forced resyncs
|
||||
if reflect.DeepEqual(oldObj, newObj) {
|
||||
return
|
||||
}
|
||||
|
||||
// Checking for nodes where NoSchedule-Taint got removed
|
||||
newNode, ok := newObj.(*v1.Node)
|
||||
if !ok {
|
||||
logger.Warningf("Expected Node but handler received %#v", newObj)
|
||||
return
|
||||
}
|
||||
|
||||
oldNode, ok := oldObj.(*v1.Node)
|
||||
if !ok {
|
||||
logger.Warningf("Expected Node but handler received %#v", oldObj)
|
||||
return
|
||||
}
|
||||
|
||||
if k8sutil.GetNodeSchedulable(*newNode) == false {
|
||||
logger.Debugf("Skipping cluster update. Updated node %s is unschedulable", newNode.Labels[apis.LabelHostname])
|
||||
return
|
||||
}
|
||||
|
||||
// Checking for nodes where NoSchedule-Taint got removed
|
||||
if k8sutil.GetNodeSchedulable(*oldNode) == true {
|
||||
logger.Debugf("Skipping cluster update. Updated node %s was and it is still schedulable", oldNode.Labels[apis.LabelHostname])
|
||||
return
|
||||
}
|
||||
|
||||
for _, cluster := range c.clusterMap {
|
||||
if valid, _ := k8sutil.ValidNode(*newNode, cephv1.GetOSDPlacement(cluster.Spec.Placement)); valid == true {
|
||||
logger.Debugf("Adding %s to cluster %s", newNode.Labels[apis.LabelHostname], cluster.Namespace)
|
||||
err := cluster.createInstance(c.rookImage)
|
||||
if err != nil {
|
||||
logger.Errorf("Failed adding the updated node %s to cluster in namespace %s. %+v", newNode.Labels[apis.LabelHostname], cluster.Namespace, err)
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
logger.Infof("Updated node %s is not valid and could not get added to cluster in namespace %s.", newNode.Labels[apis.LabelHostname], cluster.Namespace)
|
||||
continue
|
||||
}
|
||||
logger.Infof("Added updated node %s to cluster %s", newNode.Labels[apis.LabelHostname], cluster.Namespace)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ClusterController) onUpdate(oldObj, newObj interface{}) {
|
||||
oldClust, _, err := getClusterObject(oldObj)
|
||||
if err != nil {
|
||||
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"k8s.io/api/core/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
@@ -144,3 +144,16 @@ func GetNodeHostNames(clientset kubernetes.Interface) (map[string]string, error)
|
||||
}
|
||||
return nodeMap, nil
|
||||
}
|
||||
|
||||
// GetNodeSchedulable returns a boolean if the node is tainted as Schedulable or not
|
||||
// true -> Node is schedulable
|
||||
// false -> Node is unschedulable
|
||||
func GetNodeSchedulable(node v1.Node) bool {
|
||||
for i := range node.Spec.Taints {
|
||||
if node.Spec.Taints[i].Effect == "NoSchedule" {
|
||||
logger.Debugf("Node %s is unschedulable", node.Labels[apis.LabelHostname])
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
Reference in new issue
Block a user