ceph: fix race condition on crd child update

We recently discovered a race condition which happens not to update the
child CRDs when CR image changes.

The race happens under the following sequence:

* orchestration onAdd() is called when the operator image restarts
or its image updated
* the orchestration then goes into processing mon/mgr/osd
* during this the cluster CR is updated so onUpdate() is called,
however, onAdd() is not done yet for child CRDs (mds, rgw)
* onUpdate() runs against the cluster CR and updates the cluster images
* by the time the child CRDs reached onUpdate() from their respective
ParentClusterChanged() methods, the image is already up to date
so we were exiting since our check for updating
these resources are based on the cluster image version

To fix this, we changed the comparison, but using isUpgrade instead and
slightly changed the way isUpgrade operates. So now, isUpgrade is true
even if the Ceph version is identical which allows us to verify whether
or not the image changed.

Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1775624
Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
Sébastien Han committed 2019-12-02 17:33:52 +01:00
1 parent b38f160001
commit db614473c2
6 files changed
+6 -6

No files matched your search

+2 -1
View File
@@ -401,9 +401,10 @@ func diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion,
clusterRunningVersion := *version
// If this is the same version
// We return 'true' so that we restart using the Ceph checks
if cephver.IsIdentical(clusterRunningVersion, imageSpecVersion) {
logger.Debugf("both cluster and image spec versions are identical, doing nothing %s", imageSpecVersion.String())
return false, nil
return true, nil
}
if cephver.IsSuperior(imageSpecVersion, clusterRunningVersion) {
+1 -1
View File
@@ -109,7 +109,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions5)
assert.NoError(t, err)
assert.False(t, m)
assert.True(t, m)
}
func TestMinVersion(t *testing.T) {
-1
View File
@@ -330,7 +330,6 @@ func UpdateCephDeploymentAndWait(context *clusterd.Context, deployment *apps.Dep
callback := func(action string) error {
if !isUpgrade {
logger.Info("this is not a Ceph upgrade, not performing upgrade checks")
return nil
}
+1 -1
View File
@@ -164,7 +164,7 @@ func (c *FilesystemController) onUpdate(oldObj, newObj interface{}) {
// ParentClusterChanged determines wether or not a CR update has been sent
func (c *FilesystemController) ParentClusterChanged(cluster cephv1.ClusterSpec, clusterInfo *cephconfig.ClusterInfo, isUpgrade bool) {
c.clusterInfo = clusterInfo
if cluster.CephVersion.Image == c.clusterSpec.CephVersion.Image {
if !isUpgrade {
logger.Debugf("No need to update the file system after the parent cluster changed")
return
}
+1 -1
View File
@@ -165,7 +165,7 @@ func (c *CephNFSController) onDelete(obj interface{}) {
// cluster has changed.
func (c *CephNFSController) ParentClusterChanged(cluster cephv1.ClusterSpec, clusterInfo *cephconfig.ClusterInfo, isUpgrade bool) {
c.clusterInfo = clusterInfo
if cluster.CephVersion.Image == c.clusterSpec.CephVersion.Image || !c.clusterInfo.CephVersion.IsAtLeastNautilus() {
if !isUpgrade || !c.clusterInfo.CephVersion.IsAtLeastNautilus() {
logger.Debugf("No need to update the nfs daemons after the parent cluster changed")
return
}
+1 -1
View File
@@ -192,7 +192,7 @@ func (c *ObjectStoreController) onDelete(obj interface{}) {
// ParentClusterChanged determines wether or not a CR update has been sent
func (c *ObjectStoreController) ParentClusterChanged(cluster cephv1.ClusterSpec, clusterInfo *daemonconfig.ClusterInfo, isUpgrade bool) {
c.clusterInfo = clusterInfo
if cluster.CephVersion.Image == c.clusterSpec.CephVersion.Image {
if !isUpgrade {
logger.Debugf("No need to update the object store after the parent cluster changed")
return
}