From a52948e7f70d7e613cb038b67a96ccbfd9e4d7ae Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Thu, 5 Mar 2020 14:08:58 -0700 Subject: [PATCH] ceph: continue orchestration even if osd status update fails The communication between the operator and OSD prepare job is done through updates to the configmap. If an update fails to the configmap, we are anyway going to need to check the logs for failure. So let's just log the error and make a best effort to continue the orchestration. If the failure was temporary, the next update to the status would succeed and likely continue working as expected. Signed-off-by: Travis Nielsen --- pkg/daemon/ceph/osd/daemon.go | 16 +++------- pkg/operator/ceph/cluster/osd/osd.go | 33 +++++--------------- pkg/operator/ceph/cluster/osd/status.go | 15 ++++----- pkg/operator/ceph/cluster/osd/status_test.go | 3 +- 4 files changed, 18 insertions(+), 49 deletions(-) diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index b6cdcb4bb..30e047525 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -143,9 +143,7 @@ func killCephOSDProcess(context *clusterd.Context, lvPath string) error { func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation string) error { // set the initial orchestration status status := oposd.OrchestrationStatus{Status: oposd.OrchestrationStatusComputingDiff} - if err := oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status); err != nil { - return err - } + oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status) // create the ceph.conf with the default settings cephConfig, err := cephconfig.CreateDefaultCephConfig(context, agent.cluster) @@ -218,9 +216,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation string) // orchestration is about to start, update the status status = oposd.OrchestrationStatus{Status: oposd.OrchestrationStatusOrchestrating, PvcBackedOSD: agent.pvcBacked} - if err := oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status); err != nil { - return err - } + oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status) // start the desired OSDs on devices logger.Infof("configuring osd devices: %+v", devices) @@ -272,13 +268,9 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation string) } } - osds := deviceOSDs - // orchestration is completed, update the status - status = oposd.OrchestrationStatus{OSDs: osds, Status: oposd.OrchestrationStatusCompleted, PvcBackedOSD: agent.pvcBacked} - if err := oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status); err != nil { - return err - } + status = oposd.OrchestrationStatus{OSDs: deviceOSDs, Status: oposd.OrchestrationStatusCompleted, PvcBackedOSD: agent.pvcBacked} + oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status) return nil } diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 74f99030c..abae87e44 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -284,10 +284,7 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) { // Update the orchestration status of this pvc to the starting state status := OrchestrationStatus{Status: OrchestrationStatusStarting, PvcBackedOSD: true} - if err := c.updateOSDStatus(osdProps.crushHostname, status); err != nil { - config.addError("failed to set orchestration starting status for pvc %q. %v", osdProps.crushHostname, err) - continue - } + c.updateOSDStatus(osdProps.crushHostname, status) // Skip OSD prepare if deployment already exists for the PVC listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s,%s=%s", @@ -310,10 +307,7 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) { } // Update the orchestration status of this pvc to the completed state status = OrchestrationStatus{OSDs: osds, Status: OrchestrationStatusCompleted, PvcBackedOSD: true} - if err := c.updateOSDStatus(osdProps.crushHostname, status); err != nil { - config.addError("failed to update pvc %q status. %v", osdProps.crushHostname, err) - continue - } + c.updateOSDStatus(osdProps.crushHostname, status) continue } @@ -322,10 +316,7 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) { message := fmt.Sprintf("failed to create prepare job for pvc %s: %v", osdProps.crushHostname, err) config.addError(message) status := OrchestrationStatus{Status: OrchestrationStatusCompleted, Message: message, PvcBackedOSD: true} - if err := c.updateOSDStatus(osdProps.crushHostname, status); err != nil { - config.addError("failed to update pvc %q status. %v", osdProps.crushHostname, err) - continue - } + c.updateOSDStatus(osdProps.crushHostname, status) } if !c.runJob(job, osdProps.crushHostname, config, "provision") { @@ -334,9 +325,7 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) { Message: fmt.Sprintf("failed to start osd provisioning on pvc %s", osdProps.crushHostname), PvcBackedOSD: true, } - if err := c.updateOSDStatus(osdProps.crushHostname, status); err != nil { - config.addError("failed to update osd %q status. %v", osdProps.crushHostname, err) - } + c.updateOSDStatus(osdProps.crushHostname, status) } } logger.Infof("start osds after provisioning is completed, if needed") @@ -395,10 +384,7 @@ func (c *Cluster) startProvisioningOverNodes(config *provisionConfig) { // update the orchestration status of this node to the starting state status := OrchestrationStatus{Status: OrchestrationStatusStarting} - if err := c.updateOSDStatus(n.Name, status); err != nil { - config.addError("failed to set orchestration starting status for node %q. %v", n.Name, err) - continue - } + c.updateOSDStatus(n.Name, status) // create the job that prepares osds on the node storeConfig := osdconfig.ToStoreConfig(n.Config) @@ -416,17 +402,12 @@ func (c *Cluster) startProvisioningOverNodes(config *provisionConfig) { message := fmt.Sprintf("failed to create prepare job node %q. %v", n.Name, err) config.addError(message) status := OrchestrationStatus{Status: OrchestrationStatusCompleted, Message: message} - if err := c.updateOSDStatus(n.Name, status); err != nil { - config.addError("failed to update node %q status. %v", n.Name, err) - continue - } + c.updateOSDStatus(n.Name, status) } if !c.runJob(job, n.Name, config, "provision") { status := OrchestrationStatus{Status: OrchestrationStatusCompleted, Message: fmt.Sprintf("failed to start osd provisioning on node %s", n.Name)} - if err := c.updateOSDStatus(n.Name, status); err != nil { - config.addError("failed to update node %q status. %v", n.Name, err) - } + c.updateOSDStatus(n.Name, status) } } logger.Infof("start osds after provisioning is completed, if needed") diff --git a/pkg/operator/ceph/cluster/osd/status.go b/pkg/operator/ceph/cluster/osd/status.go index b95d50442..646b02783 100644 --- a/pkg/operator/ceph/cluster/osd/status.go +++ b/pkg/operator/ceph/cluster/osd/status.go @@ -22,7 +22,6 @@ import ( "fmt" "time" - "github.com/pkg/errors" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util" @@ -62,11 +61,11 @@ func (c *provisionConfig) addError(message string, args ...interface{}) { c.errorMessages = append(c.errorMessages, fmt.Sprintf(message, args...)) } -func (c *Cluster) updateOSDStatus(node string, status OrchestrationStatus) error { - return UpdateNodeStatus(c.kv, node, status) +func (c *Cluster) updateOSDStatus(node string, status OrchestrationStatus) { + UpdateNodeStatus(c.kv, node, status) } -func UpdateNodeStatus(kv *k8sutil.ConfigMapKVStore, node string, status OrchestrationStatus) error { +func UpdateNodeStatus(kv *k8sutil.ConfigMapKVStore, node string, status OrchestrationStatus) { labels := map[string]string{ k8sutil.AppAttr: AppName, orchestrationStatusKey: provisioningLabelKey, @@ -81,17 +80,15 @@ func UpdateNodeStatus(kv *k8sutil.ConfigMapKVStore, node string, status Orchestr string(s), labels, ); err != nil { - return errors.Wrapf(err, "failed to set node %s status", node) + // log the error, but allow the orchestration to continue even if the status update failed + logger.Errorf("failed to set node %q status to %q for osd orchestration. %s", node, status.Status, status.Message) } - return nil } func (c *Cluster) handleOrchestrationFailure(config *provisionConfig, nodeName, message string) { config.addError(message) status := OrchestrationStatus{Status: OrchestrationStatusFailed, Message: message} - if err := c.updateOSDStatus(nodeName, status); err != nil { - config.addError("failed to update status for node %q. %v", nodeName, err) - } + UpdateNodeStatus(c.kv, nodeName, status) } func isStatusCompleted(status OrchestrationStatus) bool { diff --git a/pkg/operator/ceph/cluster/osd/status_test.go b/pkg/operator/ceph/cluster/osd/status_test.go index 9c814ada5..682e98f20 100644 --- a/pkg/operator/ceph/cluster/osd/status_test.go +++ b/pkg/operator/ceph/cluster/osd/status_test.go @@ -53,8 +53,7 @@ func TestOrchestrationStatus(t *testing.T) { // update the status map with some status status := OrchestrationStatus{Status: OrchestrationStatusOrchestrating, Message: "doing work"} - err = UpdateNodeStatus(kv, nodeName, status) - assert.Nil(t, err) + UpdateNodeStatus(kv, nodeName, status) // retrieve the status and verify it statusMap, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(cmName, metav1.GetOptions{})