diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index b6cdcb4bb..30e047525 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -143,9 +143,7 @@ func killCephOSDProcess(context *clusterd.Context, lvPath string) error { func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation string) error { // set the initial orchestration status status := oposd.OrchestrationStatus{Status: oposd.OrchestrationStatusComputingDiff} - if err := oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status); err != nil { - return err - } + oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status) // create the ceph.conf with the default settings cephConfig, err := cephconfig.CreateDefaultCephConfig(context, agent.cluster) @@ -218,9 +216,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation string) // orchestration is about to start, update the status status = oposd.OrchestrationStatus{Status: oposd.OrchestrationStatusOrchestrating, PvcBackedOSD: agent.pvcBacked} - if err := oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status); err != nil { - return err - } + oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status) // start the desired OSDs on devices logger.Infof("configuring osd devices: %+v", devices) @@ -272,13 +268,9 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation string) } } - osds := deviceOSDs - // orchestration is completed, update the status - status = oposd.OrchestrationStatus{OSDs: osds, Status: oposd.OrchestrationStatusCompleted, PvcBackedOSD: agent.pvcBacked} - if err := oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status); err != nil { - return err - } + status = oposd.OrchestrationStatus{OSDs: deviceOSDs, Status: oposd.OrchestrationStatusCompleted, PvcBackedOSD: agent.pvcBacked} + oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status) return nil } diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 74f99030c..abae87e44 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -284,10 +284,7 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) { // Update the orchestration status of this pvc to the starting state status := OrchestrationStatus{Status: OrchestrationStatusStarting, PvcBackedOSD: true} - if err := c.updateOSDStatus(osdProps.crushHostname, status); err != nil { - config.addError("failed to set orchestration starting status for pvc %q. %v", osdProps.crushHostname, err) - continue - } + c.updateOSDStatus(osdProps.crushHostname, status) // Skip OSD prepare if deployment already exists for the PVC listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s,%s=%s", @@ -310,10 +307,7 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) { } // Update the orchestration status of this pvc to the completed state status = OrchestrationStatus{OSDs: osds, Status: OrchestrationStatusCompleted, PvcBackedOSD: true} - if err := c.updateOSDStatus(osdProps.crushHostname, status); err != nil { - config.addError("failed to update pvc %q status. %v", osdProps.crushHostname, err) - continue - } + c.updateOSDStatus(osdProps.crushHostname, status) continue } @@ -322,10 +316,7 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) { message := fmt.Sprintf("failed to create prepare job for pvc %s: %v", osdProps.crushHostname, err) config.addError(message) status := OrchestrationStatus{Status: OrchestrationStatusCompleted, Message: message, PvcBackedOSD: true} - if err := c.updateOSDStatus(osdProps.crushHostname, status); err != nil { - config.addError("failed to update pvc %q status. %v", osdProps.crushHostname, err) - continue - } + c.updateOSDStatus(osdProps.crushHostname, status) } if !c.runJob(job, osdProps.crushHostname, config, "provision") { @@ -334,9 +325,7 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) { Message: fmt.Sprintf("failed to start osd provisioning on pvc %s", osdProps.crushHostname), PvcBackedOSD: true, } - if err := c.updateOSDStatus(osdProps.crushHostname, status); err != nil { - config.addError("failed to update osd %q status. %v", osdProps.crushHostname, err) - } + c.updateOSDStatus(osdProps.crushHostname, status) } } logger.Infof("start osds after provisioning is completed, if needed") @@ -395,10 +384,7 @@ func (c *Cluster) startProvisioningOverNodes(config *provisionConfig) { // update the orchestration status of this node to the starting state status := OrchestrationStatus{Status: OrchestrationStatusStarting} - if err := c.updateOSDStatus(n.Name, status); err != nil { - config.addError("failed to set orchestration starting status for node %q. %v", n.Name, err) - continue - } + c.updateOSDStatus(n.Name, status) // create the job that prepares osds on the node storeConfig := osdconfig.ToStoreConfig(n.Config) @@ -416,17 +402,12 @@ func (c *Cluster) startProvisioningOverNodes(config *provisionConfig) { message := fmt.Sprintf("failed to create prepare job node %q. %v", n.Name, err) config.addError(message) status := OrchestrationStatus{Status: OrchestrationStatusCompleted, Message: message} - if err := c.updateOSDStatus(n.Name, status); err != nil { - config.addError("failed to update node %q status. %v", n.Name, err) - continue - } + c.updateOSDStatus(n.Name, status) } if !c.runJob(job, n.Name, config, "provision") { status := OrchestrationStatus{Status: OrchestrationStatusCompleted, Message: fmt.Sprintf("failed to start osd provisioning on node %s", n.Name)} - if err := c.updateOSDStatus(n.Name, status); err != nil { - config.addError("failed to update node %q status. %v", n.Name, err) - } + c.updateOSDStatus(n.Name, status) } } logger.Infof("start osds after provisioning is completed, if needed") diff --git a/pkg/operator/ceph/cluster/osd/status.go b/pkg/operator/ceph/cluster/osd/status.go index b95d50442..646b02783 100644 --- a/pkg/operator/ceph/cluster/osd/status.go +++ b/pkg/operator/ceph/cluster/osd/status.go @@ -22,7 +22,6 @@ import ( "fmt" "time" - "github.com/pkg/errors" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util" @@ -62,11 +61,11 @@ func (c *provisionConfig) addError(message string, args ...interface{}) { c.errorMessages = append(c.errorMessages, fmt.Sprintf(message, args...)) } -func (c *Cluster) updateOSDStatus(node string, status OrchestrationStatus) error { - return UpdateNodeStatus(c.kv, node, status) +func (c *Cluster) updateOSDStatus(node string, status OrchestrationStatus) { + UpdateNodeStatus(c.kv, node, status) } -func UpdateNodeStatus(kv *k8sutil.ConfigMapKVStore, node string, status OrchestrationStatus) error { +func UpdateNodeStatus(kv *k8sutil.ConfigMapKVStore, node string, status OrchestrationStatus) { labels := map[string]string{ k8sutil.AppAttr: AppName, orchestrationStatusKey: provisioningLabelKey, @@ -81,17 +80,15 @@ func UpdateNodeStatus(kv *k8sutil.ConfigMapKVStore, node string, status Orchestr string(s), labels, ); err != nil { - return errors.Wrapf(err, "failed to set node %s status", node) + // log the error, but allow the orchestration to continue even if the status update failed + logger.Errorf("failed to set node %q status to %q for osd orchestration. %s", node, status.Status, status.Message) } - return nil } func (c *Cluster) handleOrchestrationFailure(config *provisionConfig, nodeName, message string) { config.addError(message) status := OrchestrationStatus{Status: OrchestrationStatusFailed, Message: message} - if err := c.updateOSDStatus(nodeName, status); err != nil { - config.addError("failed to update status for node %q. %v", nodeName, err) - } + UpdateNodeStatus(c.kv, nodeName, status) } func isStatusCompleted(status OrchestrationStatus) bool { diff --git a/pkg/operator/ceph/cluster/osd/status_test.go b/pkg/operator/ceph/cluster/osd/status_test.go index 9c814ada5..682e98f20 100644 --- a/pkg/operator/ceph/cluster/osd/status_test.go +++ b/pkg/operator/ceph/cluster/osd/status_test.go @@ -53,8 +53,7 @@ func TestOrchestrationStatus(t *testing.T) { // update the status map with some status status := OrchestrationStatus{Status: OrchestrationStatusOrchestrating, Message: "doing work"} - err = UpdateNodeStatus(kv, nodeName, status) - assert.Nil(t, err) + UpdateNodeStatus(kv, nodeName, status) // retrieve the status and verify it statusMap, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(cmName, metav1.GetOptions{})