core: rotate mirror daemon keys

rotate RBD mirror daemon keys, add secret resource version to the mirror
deployment and update the mirror resource status after the key is
rotated

Signed-off-by: Santosh Pillai <sapillai@redhat.com>
(cherry picked from commit d0fbf6c424)

Documentation/CRDs/specification.md

@@ -2625,8 +2625,8 @@ string
 <td>
 <code>status</code><br/>
 <em>
-<a href="#ceph.rook.io/v1.Status">
-Status
+<a href="#ceph.rook.io/v1.RBDMirrorStatus">
+RBDMirrorStatus
 </a>
 </em>
 </td>
@@ -8714,7 +8714,7 @@ int
 <h3 id="ceph.rook.io/v1.LocalCephxStatus">LocalCephxStatus
 </h3>
 <p>
-(<em>Appears on:</em><a href="#ceph.rook.io/v1.CephFilesystemStatus">CephFilesystemStatus</a>, <a href="#ceph.rook.io/v1.ObjectStoreStatus">ObjectStoreStatus</a>)
+(<em>Appears on:</em><a href="#ceph.rook.io/v1.CephFilesystemStatus">CephFilesystemStatus</a>, <a href="#ceph.rook.io/v1.ObjectStoreStatus">ObjectStoreStatus</a>, <a href="#ceph.rook.io/v1.RBDMirrorStatus">RBDMirrorStatus</a>)
 </p>
 <div>
 </div>
@@ -12671,6 +12671,51 @@ uint64
 </tr>
 </tbody>
 </table>
+<h3 id="ceph.rook.io/v1.RBDMirrorStatus">RBDMirrorStatus
+</h3>
+<p>
+(<em>Appears on:</em><a href="#ceph.rook.io/v1.CephRBDMirror">CephRBDMirror</a>)
+</p>
+<div>
+<p>RBDMirrorStatus represents the status of the RBD mirror resource</p>
+</div>
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>Status</code><br/>
+<em>
+<a href="#ceph.rook.io/v1.Status">
+Status
+</a>
+</em>
+</td>
+<td>
+<p>
+(Members of <code>Status</code> are embedded into this type.)
+</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>cephx</code><br/>
+<em>
+<a href="#ceph.rook.io/v1.LocalCephxStatus">
+LocalCephxStatus
+</a>
+</em>
+</td>
+<td>
+</td>
+</tr>
+</tbody>
+</table>
 <h3 id="ceph.rook.io/v1.RBDMirroringSpec">RBDMirroringSpec
 </h3>
 <p>
@@ -13932,7 +13977,7 @@ int
 <h3 id="ceph.rook.io/v1.Status">Status
 </h3>
 <p>
-(<em>Appears on:</em><a href="#ceph.rook.io/v1.CephBucketNotification">CephBucketNotification</a>, <a href="#ceph.rook.io/v1.CephFilesystemMirror">CephFilesystemMirror</a>, <a href="#ceph.rook.io/v1.CephNFS">CephNFS</a>, <a href="#ceph.rook.io/v1.CephObjectRealm">CephObjectRealm</a>, <a href="#ceph.rook.io/v1.CephObjectZone">CephObjectZone</a>, <a href="#ceph.rook.io/v1.CephObjectZoneGroup">CephObjectZoneGroup</a>, <a href="#ceph.rook.io/v1.CephRBDMirror">CephRBDMirror</a>)
+(<em>Appears on:</em><a href="#ceph.rook.io/v1.CephBucketNotification">CephBucketNotification</a>, <a href="#ceph.rook.io/v1.CephFilesystemMirror">CephFilesystemMirror</a>, <a href="#ceph.rook.io/v1.CephNFS">CephNFS</a>, <a href="#ceph.rook.io/v1.CephObjectRealm">CephObjectRealm</a>, <a href="#ceph.rook.io/v1.CephObjectZone">CephObjectZone</a>, <a href="#ceph.rook.io/v1.CephObjectZoneGroup">CephObjectZoneGroup</a>, <a href="#ceph.rook.io/v1.RBDMirrorStatus">RBDMirrorStatus</a>)
 </p>
 <div>
 <p>Status represents the status of an object</p>

deploy/charts/rook-ceph/templates/resources.yaml

@@ -15181,8 +15181,33 @@ spec:
                 - count
               type: object
             status:
-              description: Status represents the status of an object
+              description: RBDMirrorStatus represents the status of the RBD mirror resource
               properties:
+                cephx:
+                  properties:
+                    daemon:
+                      description: Daemon shows the CephX key status for local Ceph daemons associated with this resources.
+                      properties:
+                        keyCephVersion:
+                          description: |-
+                            KeyCephVersion reports the Ceph version that created the current generation's keys. This is
+                            same string format as reported by `CephCluster.status.version.version` to allow them to be
+                            compared. E.g., `20.2.0-0`.
+                            For all newly-created resources, this field set to the version of Ceph that created the key.
+                            The special value "Uninitialized" indicates that keys are being created for the first time.
+                            An empty string indicates that the version is unknown, as expected in brownfield deployments.
+                          type: string
+                        keyGeneration:
+                          description: |-
+                            KeyGeneration represents the CephX key generation for the last successful reconcile.
+                            For all newly-created resources, this field is set to `1`.
+                            When keys are rotated due to any rotation policy, the generation is incremented or updated to
+                            the configured policy generation.
+                            Generation `0` indicates that keys existed prior to the implementation of key tracking.
+                          format: int32
+                          type: integer
+                      type: object
+                  type: object
                 conditions:
                   items:
                     description: Condition represents a status condition on any Rook-Ceph Custom Resource.

deploy/examples/crds.yaml

@@ -15168,8 +15168,33 @@ spec:
                 - count
               type: object
             status:
-              description: Status represents the status of an object
+              description: RBDMirrorStatus represents the status of the RBD mirror resource
               properties:
+                cephx:
+                  properties:
+                    daemon:
+                      description: Daemon shows the CephX key status for local Ceph daemons associated with this resources.
+                      properties:
+                        keyCephVersion:
+                          description: |-
+                            KeyCephVersion reports the Ceph version that created the current generation's keys. This is
+                            same string format as reported by `CephCluster.status.version.version` to allow them to be
+                            compared. E.g., `20.2.0-0`.
+                            For all newly-created resources, this field set to the version of Ceph that created the key.
+                            The special value "Uninitialized" indicates that keys are being created for the first time.
+                            An empty string indicates that the version is unknown, as expected in brownfield deployments.
+                          type: string
+                        keyGeneration:
+                          description: |-
+                            KeyGeneration represents the CephX key generation for the last successful reconcile.
+                            For all newly-created resources, this field is set to `1`.
+                            When keys are rotated due to any rotation policy, the generation is incremented or updated to
+                            the configured policy generation.
+                            Generation `0` indicates that keys existed prior to the implementation of key tracking.
+                          format: int32
+                          type: integer
+                      type: object
+                  type: object
                 conditions:
                   items:
                     description: Condition represents a status condition on any Rook-Ceph Custom Resource.

pkg/apis/ceph.rook.io/v1/types.go

@@ -3245,7 +3245,13 @@ type CephRBDMirror struct {
 	Spec              RBDMirroringSpec `json:"spec"`
 	// +kubebuilder:pruning:PreserveUnknownFields
 	// +optional
-	Status *Status `json:"status,omitempty"`
+	Status *RBDMirrorStatus `json:"status,omitempty"`
+}
+
+// RBDMirrorStatus represents the status of the RBD mirror resource
+type RBDMirrorStatus struct {
+	Status `json:",inline"`
+	Cephx  LocalCephxStatus `json:"cephx,omitempty"`
 }
 
 // CephRBDMirrorList represents a list Ceph RBD Mirrors

pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go

@@ -1791,7 +1791,7 @@ func (in *CephRBDMirror) DeepCopyInto(out *CephRBDMirror) {
 	in.Spec.DeepCopyInto(&out.Spec)
 	if in.Status != nil {
 		in, out := &in.Status, &out.Status
-		*out = new(Status)
+		*out = new(RBDMirrorStatus)
 		(*in).DeepCopyInto(*out)
 	}
 	return
@@ -4669,6 +4669,24 @@ func (in *QuotaSpec) DeepCopy() *QuotaSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RBDMirrorStatus) DeepCopyInto(out *RBDMirrorStatus) {
+	*out = *in
+	in.Status.DeepCopyInto(&out.Status)
+	out.Cephx = in.Cephx
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RBDMirrorStatus.
+func (in *RBDMirrorStatus) DeepCopy() *RBDMirrorStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(RBDMirrorStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *RBDMirroringSpec) DeepCopyInto(out *RBDMirroringSpec) {
 	*out = *in

pkg/operator/ceph/cluster/rbd/config.go

@@ -59,6 +59,15 @@ func (r *ReconcileCephRBDMirror) generateKeyring(clusterInfo *client.ClusterInfo
 		return "", err
 	}
 
+	if r.shouldRotateCephxKeys {
+		logger.Infof("rotating cephx key for CephRBDMirror %q in the namespace %q", daemonConfig.ResourceName, clusterInfo.Namespace)
+		newKey, err := s.RotateKey(user)
+		if err != nil {
+			return "", errors.Wrapf(err, "failed to rotate cephx key for CephRDBMirror %q in the namespace %q", daemonConfig.ResourceName, clusterInfo.Namespace)
+		}
+		key = newKey
+	}
+
 	// Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x
 	err = r.context.Clientset.CoreV1().Secrets(clusterInfo.Namespace).Delete(r.opManagerContext, daemonConfig.ResourceName, metav1.DeleteOptions{})
 	if err != nil {

pkg/operator/ceph/cluster/rbd/controller.go

@@ -27,6 +27,7 @@ import (
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/clusterd"
 	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
+	"github.com/rook/rook/pkg/operator/ceph/config/keyring"
 	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/ceph/reporting"
 	"github.com/rook/rook/pkg/operator/k8sutil"
@@ -37,6 +38,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -70,15 +72,16 @@ var currentAndDesiredCephVersion = opcontroller.CurrentAndDesiredCephVersion
 
 // ReconcileCephRBDMirror reconciles a cephRBDMirror object
 type ReconcileCephRBDMirror struct {
-	context          *clusterd.Context
-	clusterInfo      *cephclient.ClusterInfo
-	client           client.Client
-	scheme           *runtime.Scheme
-	cephClusterSpec  *cephv1.ClusterSpec
-	peers            map[string]*peerSpec
-	opManagerContext context.Context
-	opConfig         opcontroller.OperatorConfig
-	recorder         record.EventRecorder
+	context               *clusterd.Context
+	clusterInfo           *cephclient.ClusterInfo
+	client                client.Client
+	scheme                *runtime.Scheme
+	cephClusterSpec       *cephv1.ClusterSpec
+	peers                 map[string]*peerSpec
+	opManagerContext      context.Context
+	opConfig              opcontroller.OperatorConfig
+	recorder              record.EventRecorder
+	shouldRotateCephxKeys bool
 }
 
 // peerSpec represents peer details
@@ -163,7 +166,10 @@ func (r *ReconcileCephRBDMirror) Reconcile(context context.Context, request reco
 	// workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface
 	reconcileResponse, cephRBDMirror, err := r.reconcile(request)
 	if err != nil {
-		r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.FailedStatus)
+		err := r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.FailedStatus, nil)
+		if err != nil {
+			return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed set ready failure status to the cephRBDMirror %q", request.NamespacedName)
+		}
 		logger.Errorf("failed to reconcile %v", err)
 	}
 
@@ -185,7 +191,16 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
 
 	// The CR was just created, initializing status fields
 	if cephRBDMirror.Status == nil {
-		r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.EmptyStatus)
+		cephxUninitialized := keyring.UninitializedCephxStatus()
+		err := r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.EmptyStatus, &cephxUninitialized)
+		if err != nil {
+			return opcontroller.ImmediateRetryResult, *cephRBDMirror, errors.Wrapf(err, "failed set empty status to the cephRBDMirror %q", request.NamespacedName)
+		}
+		cephRBDMirror.Status = &cephv1.RBDMirrorStatus{
+			Cephx: cephv1.LocalCephxStatus{
+				Daemon: cephxUninitialized,
+			},
+		}
 	}
 	// update observedGeneration local variable with current generation value,
 	// because generation can be changed before reconcile got completed
@@ -249,6 +264,15 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
 		return opcontroller.ImmediateRetryResult, *cephRBDMirror, errors.Wrap(err, "failed to add ceph rbd mirror peer")
 	}
 
+	// check if cephRBDMirror daemon keys should be rotated or not
+	r.shouldRotateCephxKeys, err = keyring.ShouldRotateCephxKeys(cephCluster.Spec.Security.CephX.Daemon, *runningCephVersion, *runningCephVersion, cephRBDMirror.Status.Cephx.Daemon)
+	if err != nil {
+		return reconcile.Result{}, *cephRBDMirror, errors.Wrapf(err, "failed to determine if cephx keys should be rotated for the cephRBDMirror %q", request.NamespacedName)
+	}
+	if r.shouldRotateCephxKeys {
+		logger.Infof("cephx keys for CephRBDMirror %q will be rotated", request.NamespacedName)
+	}
+
 	// CREATE/UPDATE
 	logger.Debug("reconciling ceph rbd mirror deployments")
 	reconcileResponse, err = r.reconcileCreateCephRBDMirror(cephRBDMirror)
@@ -256,9 +280,14 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
 		return opcontroller.ImmediateRetryResult, *cephRBDMirror, errors.Wrap(err, "failed to create ceph rbd mirror deployments")
 	}
 
+	cephxStatus := keyring.UpdatedCephxStatus(r.shouldRotateCephxKeys, r.cephClusterSpec.Security.CephX.Daemon, r.clusterInfo.CephVersion, cephRBDMirror.Status.Cephx.Daemon)
+
 	// update ObservedGeneration in status at the end of reconcile
 	// Set Ready status, we are done reconciling
-	r.updateStatus(observedGeneration, request.NamespacedName, k8sutil.ReadyStatus)
+	err = r.updateStatus(observedGeneration, request.NamespacedName, k8sutil.ReadyStatus, &cephxStatus)
+	if err != nil {
+		return opcontroller.ImmediateRetryResult, *cephRBDMirror, errors.Wrapf(err, "failed set ready status to the cephRBDMirror %q", request.NamespacedName)
+	}
 
 	// Return and do not requeue
 	logger.Debug("done reconciling ceph rbd mirror")
@@ -284,29 +313,42 @@ func (r *ReconcileCephRBDMirror) reconcileCreateCephRBDMirror(cephRBDMirror *cep
 }
 
 // updateStatus updates an object with a given status
-func (r *ReconcileCephRBDMirror) updateStatus(observedGeneration int64, name types.NamespacedName, status string) {
+func (r *ReconcileCephRBDMirror) updateStatus(observedGeneration int64, namespacedName types.NamespacedName, status string, cephx *cephv1.CephxStatus) error {
 	rbdMirror := &cephv1.CephRBDMirror{}
-	err := r.client.Get(r.opManagerContext, name, rbdMirror)
-	if err != nil {
-		if kerrors.IsNotFound(err) {
-			logger.Debug("CephRBDMirror resource not found. Ignoring since object must be deleted.")
-			return
+
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		err := r.client.Get(r.opManagerContext, namespacedName, rbdMirror)
+		if err != nil {
+			if kerrors.IsNotFound(err) {
+				logger.Debug("CephRBDMirror resource not found. Ignoring since object must be deleted.")
+				return nil
+			}
+			return errors.Wrapf(err, "failed to retrieve CephRBDMirror %q to update status to %q", namespacedName, status)
 		}
-		logger.Warningf("failed to retrieve rbd mirror %q to update status to %q. %v", name, status, err)
-		return
+
+		if rbdMirror.Status == nil {
+			rbdMirror.Status = &cephv1.RBDMirrorStatus{}
+		}
+
+		rbdMirror.Status.Phase = status
+		if observedGeneration != k8sutil.ObservedGenerationNotAvailable {
+			rbdMirror.Status.ObservedGeneration = observedGeneration
+		}
+
+		if cephx != nil {
+			rbdMirror.Status.Cephx.Daemon = *cephx
+		}
+
+		if err := reporting.UpdateStatus(r.client, rbdMirror); err != nil {
+			return errors.Wrapf(err, "failed to set CephRBDMirror %q status to %+v", namespacedName, rbdMirror.Status)
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
 	}
 
-	if rbdMirror.Status == nil {
-		rbdMirror.Status = &cephv1.Status{}
-	}
-
-	rbdMirror.Status.Phase = status
-	if observedGeneration != k8sutil.ObservedGenerationNotAvailable {
-		rbdMirror.Status.ObservedGeneration = observedGeneration
-	}
-	if err := reporting.UpdateStatus(r.client, rbdMirror); err != nil {
-		logger.Errorf("failed to set rbd mirror %q status to %q. %v", rbdMirror.Name, status, err)
-		return
-	}
-	logger.Debugf("rbd mirror %q status updated to %q", name, status)
+	logger.Debugf("successfully updated cephRBDMirror %q status updated to %+v", namespacedName, rbdMirror.Status)
+	return nil
 }

pkg/operator/ceph/cluster/rbd/controller_test.go

@@ -30,9 +30,11 @@ import (
 	"github.com/rook/rook/pkg/daemon/ceph/client"
 	"github.com/rook/rook/pkg/operator/ceph/version"
 	"github.com/rook/rook/pkg/operator/k8sutil"
+	testopk8s "github.com/rook/rook/pkg/operator/k8sutil/test"
 	"github.com/rook/rook/pkg/operator/test"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
 	"github.com/stretchr/testify/assert"
+	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -98,6 +100,13 @@ func TestCephRBDMirrorController(t *testing.T) {
 			Name:      namespace,
 			Namespace: namespace,
 		},
+		Spec: cephv1.ClusterSpec{
+			Security: cephv1.ClusterSecuritySpec{
+				CephX: cephv1.ClusterCephxConfig{
+					Daemon: cephv1.CephxConfig{},
+				},
+			},
+		},
 		Status: cephv1.ClusterStatus{
 			Phase: "",
 			CephStatus: &cephv1.CephStatus{
@@ -234,3 +243,240 @@ func TestCephRBDMirrorController(t *testing.T) {
 		assert.Equal(t, "Ready", rbdMirror.Status.Phase, rbdMirror)
 	})
 }
+
+func TestMirrorKeyRotation(t *testing.T) {
+	ctx := context.TODO()
+	var (
+		name      = "my-mirror"
+		namespace = "rook-ceph"
+	)
+	// Set DEBUG logging
+	t.Setenv("ROOK_LOG_LEVEL", "DEBUG")
+
+	currentAndDesiredCephVersion = func(ctx context.Context, rookImage, namespace, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) {
+		rotationSupportedVer := version.CephVersion{Major: 20, Minor: 2, Extra: 0}
+		return &rotationSupportedVer, &rotationSupportedVer, nil
+	}
+
+	var deploymentsUpdated *[]*apps.Deployment
+	updateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub()
+
+	// An rbd-mirror resource with metadata and spec.
+	rbdMirror := &cephv1.CephRBDMirror{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: cephv1.RBDMirroringSpec{
+			Count: 1,
+		},
+		TypeMeta: controllerTypeMeta,
+	}
+
+	// Mock request to simulate Reconcile() being called on an event for a
+	// watched resource .
+	req := reconcile.Request{
+		NamespacedName: types.NamespacedName{
+			Name:      name,
+			Namespace: namespace,
+		},
+	}
+
+	cephCluster := &cephv1.CephCluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      namespace,
+			Namespace: namespace,
+		},
+		Spec: cephv1.ClusterSpec{
+			Security: cephv1.ClusterSecuritySpec{
+				CephX: cephv1.ClusterCephxConfig{
+					Daemon: cephv1.CephxConfig{},
+				},
+			},
+		},
+		Status: cephv1.ClusterStatus{
+			Phase: k8sutil.ReadyStatus,
+			CephStatus: &cephv1.CephStatus{
+				Health: "HEALTH_OK",
+			},
+		},
+	}
+
+	// Objects to track in the fake client.
+	object := []runtime.Object{
+		rbdMirror,
+		cephCluster,
+	}
+
+	s := scheme.Scheme
+
+	mirrorDaemonRotatedKey := `{"key":"AQCvzWBeIV9lFRAAninzm+8XFxbSfTiPwoX50g=="}`
+	executor := &exectest.MockExecutor{
+		MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
+			if args[0] == "status" {
+				return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_ERR"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil
+			}
+			if args[0] == "auth" && args[1] == "get-or-create-key" {
+				return cephAuthGetOrCreateKey, nil
+			}
+			if args[0] == "auth" && args[1] == "rotate" {
+				t.Logf("rotating key and returning: %s", mirrorDaemonRotatedKey)
+				return mirrorDaemonRotatedKey, nil
+			}
+			if args[0] == "versions" {
+				return dummyVersionsRaw, nil
+			}
+			if args[0] == "mirror" && args[1] == "pool" && args[2] == "info" {
+				return `{"mode":"image","site_name":"39074576-5884-4ef3-8a4d-8a0c5ed33031","peers":[{"uuid":"4a6983c0-3c9d-40f5-b2a9-2334a4659827","direction":"rx-tx","site_name":"ocs","mirror_uuid":"","client_name":"client.rbd-mirror-peer"}]}`, nil
+			}
+			if args[0] == "mirror" && args[1] == "pool" && args[2] == "status" {
+				return `{"summary":{"health":"WARNING","daemon_health":"OK","image_health":"WARNING","states":{"unknown":1}}}`, nil
+			}
+			return "", nil
+		},
+	}
+	clientset := test.New(t, 3)
+	c := &clusterd.Context{
+		Executor:      executor,
+		RookClientset: rookclient.NewSimpleClientset(),
+		Clientset:     clientset,
+	}
+
+	// Register operator types with the runtime scheme.
+	s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephRBDMirror{})
+	s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{})
+
+	// Create a fake client to mock API calls.
+	cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
+	r := &ReconcileCephRBDMirror{client: cl, scheme: s, context: c, opManagerContext: ctx, recorder: record.NewFakeRecorder(5)}
+
+	t.Run("first reconcile", func(t *testing.T) {
+		// Mock clusterInfo
+		secrets := map[string][]byte{
+			"fsid":         []byte(name),
+			"mon-secret":   []byte("monsecret"),
+			"admin-secret": []byte("adminsecret"),
+		}
+		secret := &v1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "rook-ceph-mon",
+				Namespace: namespace,
+			},
+			Data: secrets,
+			Type: k8sutil.RookType,
+		}
+		_, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
+		assert.NoError(t, err)
+
+		_, err = r.Reconcile(ctx, req)
+		assert.NoError(t, err)
+
+		mirror := cephv1.CephRBDMirror{}
+		err = cl.Get(ctx, req.NamespacedName, &mirror)
+		assert.NoError(t, err)
+		assert.Equal(t, uint32(1), mirror.Status.Cephx.Daemon.KeyGeneration)
+		assert.Equal(t, "20.2.0-0", mirror.Status.Cephx.Daemon.KeyCephVersion)
+	})
+
+	t.Run("subsequent reconcile - retain cephx status", func(t *testing.T) {
+		r := &ReconcileCephRBDMirror{client: cl, scheme: s, context: c, opManagerContext: ctx, recorder: record.NewFakeRecorder(5)}
+		_, err := r.Reconcile(ctx, req)
+		assert.NoError(t, err)
+		mirror := cephv1.CephRBDMirror{}
+		err = cl.Get(ctx, req.NamespacedName, &mirror)
+		assert.NoError(t, err)
+		assert.Equal(t, uint32(1), mirror.Status.Cephx.Daemon.KeyGeneration)
+		assert.Equal(t, "20.2.0-0", mirror.Status.Cephx.Daemon.KeyCephVersion)
+		testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
+	})
+
+	t.Run("brownfield reconcile - retain unknown cephx status", func(t *testing.T) {
+		mirror := cephv1.CephRBDMirror{}
+		err := cl.Get(ctx, req.NamespacedName, &mirror)
+		assert.NoError(t, err)
+		mirror.Status.Cephx.Daemon = cephv1.CephxStatus{}
+		err = cl.Update(ctx, &mirror)
+		assert.NoError(t, err)
+
+		_, err = r.Reconcile(ctx, req)
+		assert.NoError(t, err)
+
+		err = cl.Get(ctx, req.NamespacedName, &mirror)
+		assert.NoError(t, err)
+		assert.Equal(t, cephv1.CephxStatus{}, mirror.Status.Cephx.Daemon)
+	})
+
+	t.Run("rotate key - brownfield unknown status becomes known", func(t *testing.T) {
+		cluster := cephv1.CephCluster{}
+		err := cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: namespace}, &cluster)
+		assert.NoError(t, err)
+		cluster.Spec.Security.CephX.Daemon = cephv1.CephxConfig{
+			KeyRotationPolicy: "KeyGeneration",
+			KeyGeneration:     2,
+		}
+		err = cl.Update(ctx, &cluster)
+		assert.NoError(t, err)
+
+		mirrorDaemonRotatedKey = `[{"key":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=="}]`
+
+		_, err = r.Reconcile(ctx, req)
+		assert.NoError(t, err)
+
+		mirror := cephv1.CephRBDMirror{}
+		err = cl.Get(ctx, req.NamespacedName, &mirror)
+		assert.NoError(t, err)
+		assert.Equal(t, uint32(2), mirror.Status.Cephx.Daemon.KeyGeneration)
+		assert.Equal(t, "20.2.0-0", mirror.Status.Cephx.Daemon.KeyCephVersion)
+
+		secret, err := clientset.CoreV1().Secrets(namespace).Get(ctx, "rook-ceph-rbd-mirror-a-keyring", metav1.GetOptions{})
+		assert.NoError(t, err)
+		assert.Contains(t, secret.StringData["keyring"], "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB==")
+	})
+
+	t.Run("brownfield reconcile - no further rotation happens", func(t *testing.T) {
+		// not expecting any rotation. So `ceph auth rotate` should not run and secret should not be updated
+		mirrorDaemonRotatedKey = `[{"key":"CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC=="}]`
+
+		res, err := r.Reconcile(ctx, req)
+		assert.NoError(t, err)
+		assert.False(t, res.Requeue)
+
+		mirror := cephv1.CephRBDMirror{}
+		err = cl.Get(ctx, req.NamespacedName, &mirror)
+		assert.NoError(t, err)
+		assert.Equal(t, uint32(2), mirror.Status.Cephx.Daemon.KeyGeneration)
+		assert.Equal(t, "20.2.0-0", mirror.Status.Cephx.Daemon.KeyCephVersion)
+
+		secret, err := clientset.CoreV1().Secrets(namespace).Get(ctx, "rook-ceph-rbd-mirror-a-keyring", metav1.GetOptions{})
+		assert.NoError(t, err)
+		assert.NotContains(t, secret.StringData["keyring"], "CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC==")
+		testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
+	})
+
+	t.Run("rotate key - cephx status updated", func(t *testing.T) {
+		cluster := cephv1.CephCluster{}
+		err := cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: namespace}, &cluster)
+		assert.NoError(t, err)
+		cluster.Spec.Security.CephX.Daemon = cephv1.CephxConfig{
+			KeyRotationPolicy: "KeyGeneration",
+			KeyGeneration:     3,
+		}
+		err = cl.Update(ctx, &cluster)
+		assert.NoError(t, err)
+
+		mirrorDaemonRotatedKey = `[{"key":"CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC=="}]`
+
+		_, err = r.Reconcile(ctx, req)
+		assert.NoError(t, err)
+
+		mirror := cephv1.CephRBDMirror{}
+		err = cl.Get(ctx, req.NamespacedName, &mirror)
+		assert.NoError(t, err)
+		assert.Equal(t, uint32(3), mirror.Status.Cephx.Daemon.KeyGeneration)
+		assert.Equal(t, "20.2.0-0", mirror.Status.Cephx.Daemon.KeyCephVersion)
+
+		secret, err := clientset.CoreV1().Secrets(namespace).Get(ctx, "rook-ceph-rbd-mirror-a-keyring", metav1.GetOptions{})
+		assert.NoError(t, err)
+		assert.Contains(t, secret.StringData["keyring"], "CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC==")
+	})
+}

pkg/operator/ceph/cluster/rbd/mirror.go

@@ -25,6 +25,7 @@ import (
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
 	"github.com/rook/rook/pkg/operator/ceph/config"
+	"github.com/rook/rook/pkg/operator/ceph/config/keyring"
 	"github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
@@ -62,11 +63,6 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro
 		ownerInfo:    ownerInfo,
 	}
 
-	_, err = r.generateKeyring(r.clusterInfo, daemonConf)
-	if err != nil {
-		return errors.Wrapf(err, "failed to generate keyring for %q", resourceName)
-	}
-
 	rbdMirrorToSkipReconcile, err := controller.GetDaemonsToSkipReconcile(r.clusterInfo.Context, r.context, r.clusterInfo.Namespace, config.RbdMirrorType, AppName)
 	if err != nil {
 		return errors.Wrap(err, "failed to check for RBD Mirror to skip reconcile")
@@ -76,12 +72,20 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro
 		return nil
 	}
 
+	secretResourceVersion, err := r.generateKeyring(r.clusterInfo, daemonConf)
+	if err != nil {
+		return errors.Wrapf(err, "failed to generate keyring for %q", resourceName)
+	}
+
 	// Start the deployment
 	d, err := r.makeDeployment(daemonConf, cephRBDMirror)
 	if err != nil {
 		return errors.Wrap(err, "failed to create rbd-mirror deployment")
 	}
 
+	// apply cephx secret resource version to the deployment to ensure it restarts when keyring updates
+	d.Spec.Template.Annotations[keyring.CephxKeyIdentifierAnnotation] = secretResourceVersion
+
 	// Set owner ref to cephRBDMirror object
 	err = controllerutil.SetControllerReference(cephRBDMirror, d, r.scheme)
 	if err != nil {