From 082858bf53d8202d469ca5bb802594739bd9743b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Fri, 30 Jul 2021 16:01:18 +0200 Subject: [PATCH 1/3] ceph: use v16 for mirroring test in ci MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CI test for mirroring now runs on stable v16 tag. Internally our code has a minimum version of 16.2.5 for cephfs mirroring since it's available. Signed-off-by: Sébastien Han --- .github/workflows/canary-integration-test.yml | 1 - pkg/operator/ceph/file/controller.go | 5 ++--- pkg/operator/ceph/file/mirror/config.go | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index d2d89fcad..8a5a092d0 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -727,7 +727,6 @@ jobs: yq w -i -d1 cluster-test.yaml spec.dashboard.enabled false yq w -i -d1 cluster-test.yaml spec.storage.useAllDevices false yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter ${BLOCK}1 - yq w -i -d1 cluster-test.yaml spec.cephVersion.image ceph/daemon-base:latest-pacific-devel kubectl create -f cluster-test.yaml -f rbdmirror.yaml -f filesystem-mirror.yaml -f toolbox.yaml # cephfs-mirroring is a push operation diff --git a/pkg/operator/ceph/file/controller.go b/pkg/operator/ceph/file/controller.go index 406de8527..fb74c3571 100644 --- a/pkg/operator/ceph/file/controller.go +++ b/pkg/operator/ceph/file/controller.go @@ -31,6 +31,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/mon" opconfig "github.com/rook/rook/pkg/operator/ceph/config" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/ceph/file/mirror" "github.com/rook/rook/pkg/operator/k8sutil" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -289,9 +290,7 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil } // Enable mirroring if needed - // TODO: change me to 16.2.5 once it's out, in the mean this allows us to run the CI and validate the code - // if r.clusterInfo.CephVersion.IsAtLeast(mirror.PeerAdditionMinVersion) { - if r.clusterInfo.CephVersion.IsAtLeastPacific() { + if r.clusterInfo.CephVersion.IsAtLeast(mirror.PeerAdditionMinVersion) { // Disable mirroring on that filesystem if needed if cephFilesystem.Spec.Mirroring != nil { if !cephFilesystem.Spec.Mirroring.Enabled { diff --git a/pkg/operator/ceph/file/mirror/config.go b/pkg/operator/ceph/file/mirror/config.go index 4314c51a8..2edc5bdf8 100644 --- a/pkg/operator/ceph/file/mirror/config.go +++ b/pkg/operator/ceph/file/mirror/config.go @@ -41,8 +41,7 @@ const ( var ( // PeerAdditionMinVersion This version includes a number of fixes for snapshots and mirror status - // TODO change me to 16.2.5 - PeerAdditionMinVersion = version.CephVersion{Major: 16, Minor: 2, Extra: 2} + PeerAdditionMinVersion = version.CephVersion{Major: 16, Minor: 2, Extra: 5} ) // daemonConfig for a single rbd-mirror From 73b0ce08ffabe0d018bf9590daf8f3d55b78ed5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Tue, 3 Aug 2021 11:47:13 +0200 Subject: [PATCH 2/3] ceph: do not stack trace on error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We should not use %+v when printing errors and wrapping it in the caller. Passing %+v to errors.Wrap() returns a strack trace. See: ``` 2021-08-03 09:42:40.460946 E | ceph-file-controller: failed to reconcile failed to create filesystem "myfs2": failed to start deployment for MDS "a" for filesystem "myfs2": failed to update mds deployment "rook-ceph-mds-myfs2-a": failed to check if deployment "rook-ceph-mds-myfs2-a" can continue: failed to check if we can continue the deployment rook-ceph-mds-myfs2-a: failed to check if rook-ceph-mds-myfs2-a was ok to continue: max retries exceeded, last err: mds myfs2-a is up:creating, bad state github.com/rook/rook/pkg/daemon/ceph/client.MdsActiveOrStandbyReplay /home/leseb/go/src/github.com/rook/rook/pkg/daemon/ceph/client/status.go:286 github.com/rook/rook/pkg/daemon/ceph/client.okToContinueMDSDaemon.func1 /home/leseb/go/src/github.com/rook/rook/pkg/daemon/ceph/client/upgrade.go:215 github.com/rook/rook/pkg/util.Retry /home/leseb/go/src/github.com/rook/rook/pkg/util/retry.go:31 github.com/rook/rook/pkg/daemon/ceph/client.okToContinueMDSDaemon /home/leseb/go/src/github.com/rook/rook/pkg/daemon/ceph/client/upgrade.go:214 github.com/rook/rook/pkg/daemon/ceph/client.OkToContinue /home/leseb/go/src/github.com/rook/rook/pkg/daemon/ceph/client/upgrade.go:181 github.com/rook/rook/pkg/operator/ceph/cluster/mon.UpdateCephDeploymentAndWait.func1 /home/leseb/go/src/github.com/rook/rook/pkg/operator/ceph/cluster/mon/spec.go:370 github.com/rook/rook/pkg/operator/k8sutil.UpdateDeploymentAndWait /home/leseb/go/src/github.com/rook/rook/pkg/operator/k8sutil/deployment.go:113 github.com/rook/rook/pkg/operator/ceph/cluster/mon.UpdateCephDeploymentAndWait /home/leseb/go/src/github.com/rook/rook/pkg/operator/ceph/cluster/mon/spec.go:383 github.com/rook/rook/pkg/operator/ceph/file/mds.(*Cluster).startDeployment /home/leseb/go/src/github.com/rook/rook/pkg/operator/ceph/file/mds/mds.go:216 github.com/rook/rook/pkg/operator/ceph/file/mds.(*Cluster).Start /home/leseb/go/src/github.com/rook/rook/pkg/operator/ceph/file/mds/mds.go:138 github.com/rook/rook/pkg/operator/ceph/file.createFilesystem /home/leseb/go/src/github.com/rook/rook/pkg/operator/ceph/file/filesystem.go:81 github.com/rook/rook/pkg/operator/ceph/file.(*ReconcileCephFilesystem).reconcileCreateFilesystem /home/leseb/go/src/github.com/rook/rook/pkg/operator/ceph/file/controller.go:373 github.com/rook/rook/pkg/operator/ceph/file.(*ReconcileCephFilesystem).reconcile /home/leseb/go/src/github.com/rook/rook/pkg/operator/ceph/file/controller.go:288 github.com/rook/rook/pkg/operator/ceph/file.(*ReconcileCephFilesystem).Reconcile /home/leseb/go/src/github.com/rook/rook/pkg/operator/ceph/file/controller.go:167 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /home/leseb/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.9.0/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /home/leseb/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.9.0/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /home/leseb/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.9.0/pkg/internal/controller/controller.go:214 runtime.goexit /usr/local/go/src/runtime/asm_amd64.s:1371 ``` Signed-off-by: Sébastien Han --- pkg/util/retry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/retry.go b/pkg/util/retry.go index 3c15141ea..27622e805 100644 --- a/pkg/util/retry.go +++ b/pkg/util/retry.go @@ -36,7 +36,7 @@ func Retry(maxRetries int, delay time.Duration, f func() error) error { tries++ if tries > maxRetries { - return fmt.Errorf("max retries exceeded, last err: %+v", err) + return fmt.Errorf("max retries exceeded, last err: %v", err) } logger.Infof("retrying after %v, last error: %v", delay, err) From 73184235062d66c84315b48af4ef5bcd0bbf955b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Fri, 30 Jul 2021 16:18:23 +0200 Subject: [PATCH 3/3] ceph: ignore errors when mirroring is not enabled on the filesystem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trying to disable mirroring on a cluster where mirroring is not enabled results in an error during upgrades. Let's catch this error and ignore it. Funny enough the exec error resembles to: ``` Error ENOTSUP: Module 'mirroring' is not enabled (required by command 'fs snapshot mirror disable'): use `ceph mgr module enable mirroring` to enable it: exit status 95 ``` So we get ENOTSUP which is 45 but exit status 95... Closes: https://github.com/rook/rook/issues/8438 Signed-off-by: Sébastien Han --- pkg/daemon/ceph/client/filesystem_mirror.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/daemon/ceph/client/filesystem_mirror.go b/pkg/daemon/ceph/client/filesystem_mirror.go index 1d6ba98d5..dfe6151cc 100644 --- a/pkg/daemon/ceph/client/filesystem_mirror.go +++ b/pkg/daemon/ceph/client/filesystem_mirror.go @@ -79,6 +79,10 @@ func DisableFilesystemSnapshotMirror(context *clusterd.Context, clusterInfo *Clu // Run command output, err := cmd.Run() if err != nil { + if code, err := exec.ExtractExitCode(err); err == nil && code == int(syscall.ENOTSUP) { + logger.Debug("filesystem mirroring is not enabled, nothing to disable") + return nil + } return errors.Wrapf(err, "failed to disable ceph filesystem snapshot mirror for filesystem %q. %s", filesystem, output) }