ceph: continue orchestration even if osd status update fails
The communication between the operator and OSD prepare job is done through updates to the configmap. If an update fails to the configmap, we are anyway going to need to check the logs for failure. So let's just log the error and make a best effort to continue the orchestration. If the failure was temporary, the next update to the status would succeed and likely continue working as expected. Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
4 files changed
+18
-49
No files matched your search
@@ -143,9 +143,7 @@ func killCephOSDProcess(context *clusterd.Context, lvPath string) error {
|
||||
func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation string) error {
|
||||
// set the initial orchestration status
|
||||
status := oposd.OrchestrationStatus{Status: oposd.OrchestrationStatusComputingDiff}
|
||||
if err := oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status); err != nil {
|
||||
return err
|
||||
}
|
||||
oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status)
|
||||
|
||||
// create the ceph.conf with the default settings
|
||||
cephConfig, err := cephconfig.CreateDefaultCephConfig(context, agent.cluster)
|
||||
@@ -218,9 +216,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation string)
|
||||
|
||||
// orchestration is about to start, update the status
|
||||
status = oposd.OrchestrationStatus{Status: oposd.OrchestrationStatusOrchestrating, PvcBackedOSD: agent.pvcBacked}
|
||||
if err := oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status); err != nil {
|
||||
return err
|
||||
}
|
||||
oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status)
|
||||
|
||||
// start the desired OSDs on devices
|
||||
logger.Infof("configuring osd devices: %+v", devices)
|
||||
@@ -272,13 +268,9 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation string)
|
||||
}
|
||||
}
|
||||
|
||||
osds := deviceOSDs
|
||||
|
||||
// orchestration is completed, update the status
|
||||
status = oposd.OrchestrationStatus{OSDs: osds, Status: oposd.OrchestrationStatusCompleted, PvcBackedOSD: agent.pvcBacked}
|
||||
if err := oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status); err != nil {
|
||||
return err
|
||||
}
|
||||
status = oposd.OrchestrationStatus{OSDs: deviceOSDs, Status: oposd.OrchestrationStatusCompleted, PvcBackedOSD: agent.pvcBacked}
|
||||
oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -284,10 +284,7 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) {
|
||||
|
||||
// Update the orchestration status of this pvc to the starting state
|
||||
status := OrchestrationStatus{Status: OrchestrationStatusStarting, PvcBackedOSD: true}
|
||||
if err := c.updateOSDStatus(osdProps.crushHostname, status); err != nil {
|
||||
config.addError("failed to set orchestration starting status for pvc %q. %v", osdProps.crushHostname, err)
|
||||
continue
|
||||
}
|
||||
c.updateOSDStatus(osdProps.crushHostname, status)
|
||||
|
||||
// Skip OSD prepare if deployment already exists for the PVC
|
||||
listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s,%s=%s",
|
||||
@@ -310,10 +307,7 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) {
|
||||
}
|
||||
// Update the orchestration status of this pvc to the completed state
|
||||
status = OrchestrationStatus{OSDs: osds, Status: OrchestrationStatusCompleted, PvcBackedOSD: true}
|
||||
if err := c.updateOSDStatus(osdProps.crushHostname, status); err != nil {
|
||||
config.addError("failed to update pvc %q status. %v", osdProps.crushHostname, err)
|
||||
continue
|
||||
}
|
||||
c.updateOSDStatus(osdProps.crushHostname, status)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -322,10 +316,7 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) {
|
||||
message := fmt.Sprintf("failed to create prepare job for pvc %s: %v", osdProps.crushHostname, err)
|
||||
config.addError(message)
|
||||
status := OrchestrationStatus{Status: OrchestrationStatusCompleted, Message: message, PvcBackedOSD: true}
|
||||
if err := c.updateOSDStatus(osdProps.crushHostname, status); err != nil {
|
||||
config.addError("failed to update pvc %q status. %v", osdProps.crushHostname, err)
|
||||
continue
|
||||
}
|
||||
c.updateOSDStatus(osdProps.crushHostname, status)
|
||||
}
|
||||
|
||||
if !c.runJob(job, osdProps.crushHostname, config, "provision") {
|
||||
@@ -334,9 +325,7 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) {
|
||||
Message: fmt.Sprintf("failed to start osd provisioning on pvc %s", osdProps.crushHostname),
|
||||
PvcBackedOSD: true,
|
||||
}
|
||||
if err := c.updateOSDStatus(osdProps.crushHostname, status); err != nil {
|
||||
config.addError("failed to update osd %q status. %v", osdProps.crushHostname, err)
|
||||
}
|
||||
c.updateOSDStatus(osdProps.crushHostname, status)
|
||||
}
|
||||
}
|
||||
logger.Infof("start osds after provisioning is completed, if needed")
|
||||
@@ -395,10 +384,7 @@ func (c *Cluster) startProvisioningOverNodes(config *provisionConfig) {
|
||||
|
||||
// update the orchestration status of this node to the starting state
|
||||
status := OrchestrationStatus{Status: OrchestrationStatusStarting}
|
||||
if err := c.updateOSDStatus(n.Name, status); err != nil {
|
||||
config.addError("failed to set orchestration starting status for node %q. %v", n.Name, err)
|
||||
continue
|
||||
}
|
||||
c.updateOSDStatus(n.Name, status)
|
||||
|
||||
// create the job that prepares osds on the node
|
||||
storeConfig := osdconfig.ToStoreConfig(n.Config)
|
||||
@@ -416,17 +402,12 @@ func (c *Cluster) startProvisioningOverNodes(config *provisionConfig) {
|
||||
message := fmt.Sprintf("failed to create prepare job node %q. %v", n.Name, err)
|
||||
config.addError(message)
|
||||
status := OrchestrationStatus{Status: OrchestrationStatusCompleted, Message: message}
|
||||
if err := c.updateOSDStatus(n.Name, status); err != nil {
|
||||
config.addError("failed to update node %q status. %v", n.Name, err)
|
||||
continue
|
||||
}
|
||||
c.updateOSDStatus(n.Name, status)
|
||||
}
|
||||
|
||||
if !c.runJob(job, n.Name, config, "provision") {
|
||||
status := OrchestrationStatus{Status: OrchestrationStatusCompleted, Message: fmt.Sprintf("failed to start osd provisioning on node %s", n.Name)}
|
||||
if err := c.updateOSDStatus(n.Name, status); err != nil {
|
||||
config.addError("failed to update node %q status. %v", n.Name, err)
|
||||
}
|
||||
c.updateOSDStatus(n.Name, status)
|
||||
}
|
||||
}
|
||||
logger.Infof("start osds after provisioning is completed, if needed")
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util"
|
||||
@@ -62,11 +61,11 @@ func (c *provisionConfig) addError(message string, args ...interface{}) {
|
||||
c.errorMessages = append(c.errorMessages, fmt.Sprintf(message, args...))
|
||||
}
|
||||
|
||||
func (c *Cluster) updateOSDStatus(node string, status OrchestrationStatus) error {
|
||||
return UpdateNodeStatus(c.kv, node, status)
|
||||
func (c *Cluster) updateOSDStatus(node string, status OrchestrationStatus) {
|
||||
UpdateNodeStatus(c.kv, node, status)
|
||||
}
|
||||
|
||||
func UpdateNodeStatus(kv *k8sutil.ConfigMapKVStore, node string, status OrchestrationStatus) error {
|
||||
func UpdateNodeStatus(kv *k8sutil.ConfigMapKVStore, node string, status OrchestrationStatus) {
|
||||
labels := map[string]string{
|
||||
k8sutil.AppAttr: AppName,
|
||||
orchestrationStatusKey: provisioningLabelKey,
|
||||
@@ -81,17 +80,15 @@ func UpdateNodeStatus(kv *k8sutil.ConfigMapKVStore, node string, status Orchestr
|
||||
string(s),
|
||||
labels,
|
||||
); err != nil {
|
||||
return errors.Wrapf(err, "failed to set node %s status", node)
|
||||
// log the error, but allow the orchestration to continue even if the status update failed
|
||||
logger.Errorf("failed to set node %q status to %q for osd orchestration. %s", node, status.Status, status.Message)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) handleOrchestrationFailure(config *provisionConfig, nodeName, message string) {
|
||||
config.addError(message)
|
||||
status := OrchestrationStatus{Status: OrchestrationStatusFailed, Message: message}
|
||||
if err := c.updateOSDStatus(nodeName, status); err != nil {
|
||||
config.addError("failed to update status for node %q. %v", nodeName, err)
|
||||
}
|
||||
UpdateNodeStatus(c.kv, nodeName, status)
|
||||
}
|
||||
|
||||
func isStatusCompleted(status OrchestrationStatus) bool {
|
||||
|
||||
@@ -53,8 +53,7 @@ func TestOrchestrationStatus(t *testing.T) {
|
||||
|
||||
// update the status map with some status
|
||||
status := OrchestrationStatus{Status: OrchestrationStatusOrchestrating, Message: "doing work"}
|
||||
err = UpdateNodeStatus(kv, nodeName, status)
|
||||
assert.Nil(t, err)
|
||||
UpdateNodeStatus(kv, nodeName, status)
|
||||
|
||||
// retrieve the status and verify it
|
||||
statusMap, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(cmName, metav1.GetOptions{})
|
||||
|
||||
Reference in new issue
Block a user