ceph: fix race condition on crd child update
We recently discovered a race condition which happens not to update the child CRDs when CR image changes. The race happens under the following sequence: * orchestration onAdd() is called when the operator image restarts or its image updated * the orchestration then goes into processing mon/mgr/osd * during this the cluster CR is updated so onUpdate() is called, however, onAdd() is not done yet for child CRDs (mds, rgw) * onUpdate() runs against the cluster CR and updates the cluster images * by the time the child CRDs reached onUpdate() from their respective ParentClusterChanged() methods, the image is already up to date so we were exiting since our check for updating these resources are based on the cluster image version To fix this, we changed the comparison, but using isUpgrade instead and slightly changed the way isUpgrade operates. So now, isUpgrade is true even if the Ceph version is identical which allows us to verify whether or not the image changed. Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1775624 Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
1 parent
b38f160001
commit
db614473c2
6 files changed
+6
-6
No files matched your search
@@ -401,9 +401,10 @@ func diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion,
|
||||
clusterRunningVersion := *version
|
||||
|
||||
// If this is the same version
|
||||
// We return 'true' so that we restart using the Ceph checks
|
||||
if cephver.IsIdentical(clusterRunningVersion, imageSpecVersion) {
|
||||
logger.Debugf("both cluster and image spec versions are identical, doing nothing %s", imageSpecVersion.String())
|
||||
return false, nil
|
||||
return true, nil
|
||||
}
|
||||
|
||||
if cephver.IsSuperior(imageSpecVersion, clusterRunningVersion) {
|
||||
|
||||
@@ -109,7 +109,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
|
||||
|
||||
m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions5)
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, m)
|
||||
assert.True(t, m)
|
||||
}
|
||||
|
||||
func TestMinVersion(t *testing.T) {
|
||||
|
||||
@@ -330,7 +330,6 @@ func UpdateCephDeploymentAndWait(context *clusterd.Context, deployment *apps.Dep
|
||||
|
||||
callback := func(action string) error {
|
||||
if !isUpgrade {
|
||||
logger.Info("this is not a Ceph upgrade, not performing upgrade checks")
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -164,7 +164,7 @@ func (c *FilesystemController) onUpdate(oldObj, newObj interface{}) {
|
||||
// ParentClusterChanged determines wether or not a CR update has been sent
|
||||
func (c *FilesystemController) ParentClusterChanged(cluster cephv1.ClusterSpec, clusterInfo *cephconfig.ClusterInfo, isUpgrade bool) {
|
||||
c.clusterInfo = clusterInfo
|
||||
if cluster.CephVersion.Image == c.clusterSpec.CephVersion.Image {
|
||||
if !isUpgrade {
|
||||
logger.Debugf("No need to update the file system after the parent cluster changed")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -165,7 +165,7 @@ func (c *CephNFSController) onDelete(obj interface{}) {
|
||||
// cluster has changed.
|
||||
func (c *CephNFSController) ParentClusterChanged(cluster cephv1.ClusterSpec, clusterInfo *cephconfig.ClusterInfo, isUpgrade bool) {
|
||||
c.clusterInfo = clusterInfo
|
||||
if cluster.CephVersion.Image == c.clusterSpec.CephVersion.Image || !c.clusterInfo.CephVersion.IsAtLeastNautilus() {
|
||||
if !isUpgrade || !c.clusterInfo.CephVersion.IsAtLeastNautilus() {
|
||||
logger.Debugf("No need to update the nfs daemons after the parent cluster changed")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -192,7 +192,7 @@ func (c *ObjectStoreController) onDelete(obj interface{}) {
|
||||
// ParentClusterChanged determines wether or not a CR update has been sent
|
||||
func (c *ObjectStoreController) ParentClusterChanged(cluster cephv1.ClusterSpec, clusterInfo *daemonconfig.ClusterInfo, isUpgrade bool) {
|
||||
c.clusterInfo = clusterInfo
|
||||
if cluster.CephVersion.Image == c.clusterSpec.CephVersion.Image {
|
||||
if !isUpgrade {
|
||||
logger.Debugf("No need to update the object store after the parent cluster changed")
|
||||
return
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user