From add3129c12f1c558fda587e75b57e4172c26875d Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Tue, 3 Dec 2019 19:16:48 -0700 Subject: [PATCH] ceph: continue orchestration on osd update errors During an orchestration if there were failures to update any OSD deployment, the orchestration would fail and continue trying again. If the OSD(s) continued failing, the operator would be stuck in a loop and never respond to other CR updates. Now the operator will log the OSD errors, if any, and complete the orchestration. We will rely on K8s to continue to attempt to start the OSD pods instead of the operator being stuck in the loop. Signed-off-by: Travis Nielsen --- pkg/operator/ceph/cluster/mgr/mgr.go | 2 +- pkg/operator/ceph/cluster/osd/osd.go | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go index c6c93f674..923ff8059 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr.go +++ b/pkg/operator/ceph/cluster/mgr/mgr.go @@ -192,7 +192,7 @@ func (c *Cluster) Start() error { } if err := updateDeploymentAndWait(c.context, d, c.Namespace, daemon, mgrConfig.DaemonID, cephVersionToUse, c.isUpgrade, c.skipUpgradeChecks); err != nil { - return fmt.Errorf("failed to update mgr deployment %s. %+v", resourceName, err) + logger.Errorf("failed to update mgr deployment %s. %+v", resourceName, err) } } if existingDeployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(d.GetName(), metav1.GetOptions{}); err != nil { diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 16a245485..3b5c3d80f 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -352,11 +352,10 @@ func (c *Cluster) startProvisioningOverNodes(config *provisionConfig) { // no valid node is ready to run an osd if len(validNodes) == 0 { - logger.Warningf("no valid nodes available to run an osd in namespace %s. "+ - "Rook will not create any new OSD nodes and will skip checking for removed nodes since "+ - "removing all OSD nodes without destroying the Rook cluster is unlikely to be intentional", c.Namespace) + logger.Warningf("no valid nodes available to run osds on nodes in namespace %s", c.Namespace) return } + // start with nodes currently in the storage spec for _, node := range c.ValidStorage.Nodes { // fully resolve the storage config and resources for this node @@ -481,8 +480,7 @@ func (c *Cluster) startOSDDaemonsOnPVC(pvcName string, config *provisionConfig, err = c.associateKeyring(keyring, createdDeployment) if err != nil { - errMsg := fmt.Sprintf("failed to associate keyring for pvc %s, osd %v: %+v", osdProps.pvc.ClaimName, osd, err) - config.addError(errMsg) + logger.Errorf("failed to associate keyring for pvc %s, osd %v: %+v", osdProps.pvc.ClaimName, osd, err) } if createErr != nil && errors.IsAlreadyExists(createErr) { @@ -504,7 +502,7 @@ func (c *Cluster) startOSDDaemonsOnPVC(pvcName string, config *provisionConfig, } if err = updateDeploymentAndWait(c.context, dp, c.Namespace, daemon, strconv.Itoa(osd.ID), cephVersionToUse, c.isUpgrade, c.skipUpgradeChecks); err != nil { - config.addError(fmt.Sprintf("failed to update osd deployment %d. %+v", osd.ID, err)) + logger.Errorf("failed to update osd deployment %d. %+v", osd.ID, err) } } logger.Infof("started deployment for osd %d (dir=%t, type=%s)", osd.ID, osd.IsDirectory, storeConfig.StoreType) @@ -571,8 +569,7 @@ func (c *Cluster) startOSDDaemonsOnNode(nodeName string, config *provisionConfig err = c.associateKeyring(keyring, createdDeployment) if err != nil { - errMsg := fmt.Sprintf("failed to associate keyring for node %s, osd %v: %+v", n.Name, osd, err) - config.addError(errMsg) + logger.Errorf("failed to associate keyring for node %s, osd %v: %+v", n.Name, osd, err) } if createErr != nil && errors.IsAlreadyExists(createErr) { @@ -594,7 +591,7 @@ func (c *Cluster) startOSDDaemonsOnNode(nodeName string, config *provisionConfig } if err = updateDeploymentAndWait(c.context, dp, c.Namespace, daemon, strconv.Itoa(osd.ID), cephVersionToUse, c.isUpgrade, c.skipUpgradeChecks); err != nil { - config.addError(fmt.Sprintf("failed to update osd deployment %d. %+v", osd.ID, err)) + logger.Errorf("failed to update osd deployment %d. %+v", osd.ID, err) } } logger.Infof("started deployment for osd %d (dir=%t, type=%s)", osd.ID, osd.IsDirectory, storeConfig.StoreType)