status-(Appears on:CephFilesystemStatus, ObjectStoreStatus) +(Appears on:CephFilesystemStatus, ObjectStoreStatus, RBDMirrorStatus)
+(Appears on:CephRBDMirror) +
+RBDMirrorStatus represents the status of the RBD mirror resource
+| Field | +Description | +
|---|---|
+Status+ + +Status + + + |
+
+
+(Members of |
+
+cephx+ + +LocalCephxStatus + + + |
++ | +
@@ -13932,7 +13977,7 @@ int
-(Appears on:CephBucketNotification, CephFilesystemMirror, CephNFS, CephObjectRealm, CephObjectZone, CephObjectZoneGroup, CephRBDMirror) +(Appears on:CephBucketNotification, CephFilesystemMirror, CephNFS, CephObjectRealm, CephObjectZone, CephObjectZoneGroup, RBDMirrorStatus)
Status represents the status of an object
diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 34b73dfa2..3bebd7afb 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -15181,8 +15181,33 @@ spec: - count type: object status: - description: Status represents the status of an object + description: RBDMirrorStatus represents the status of the RBD mirror resource properties: + cephx: + properties: + daemon: + description: Daemon shows the CephX key status for local Ceph daemons associated with this resources. + properties: + keyCephVersion: + description: |- + KeyCephVersion reports the Ceph version that created the current generation's keys. This is + same string format as reported by `CephCluster.status.version.version` to allow them to be + compared. E.g., `20.2.0-0`. + For all newly-created resources, this field set to the version of Ceph that created the key. + The special value "Uninitialized" indicates that keys are being created for the first time. + An empty string indicates that the version is unknown, as expected in brownfield deployments. + type: string + keyGeneration: + description: |- + KeyGeneration represents the CephX key generation for the last successful reconcile. + For all newly-created resources, this field is set to `1`. + When keys are rotated due to any rotation policy, the generation is incremented or updated to + the configured policy generation. + Generation `0` indicates that keys existed prior to the implementation of key tracking. + format: int32 + type: integer + type: object + type: object conditions: items: description: Condition represents a status condition on any Rook-Ceph Custom Resource. diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index d0aa0ec0a..da344e03d 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -15168,8 +15168,33 @@ spec: - count type: object status: - description: Status represents the status of an object + description: RBDMirrorStatus represents the status of the RBD mirror resource properties: + cephx: + properties: + daemon: + description: Daemon shows the CephX key status for local Ceph daemons associated with this resources. + properties: + keyCephVersion: + description: |- + KeyCephVersion reports the Ceph version that created the current generation's keys. This is + same string format as reported by `CephCluster.status.version.version` to allow them to be + compared. E.g., `20.2.0-0`. + For all newly-created resources, this field set to the version of Ceph that created the key. + The special value "Uninitialized" indicates that keys are being created for the first time. + An empty string indicates that the version is unknown, as expected in brownfield deployments. + type: string + keyGeneration: + description: |- + KeyGeneration represents the CephX key generation for the last successful reconcile. + For all newly-created resources, this field is set to `1`. + When keys are rotated due to any rotation policy, the generation is incremented or updated to + the configured policy generation. + Generation `0` indicates that keys existed prior to the implementation of key tracking. + format: int32 + type: integer + type: object + type: object conditions: items: description: Condition represents a status condition on any Rook-Ceph Custom Resource. diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 11548677b..d83c1a0ec 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -3245,7 +3245,13 @@ type CephRBDMirror struct { Spec RBDMirroringSpec `json:"spec"` // +kubebuilder:pruning:PreserveUnknownFields // +optional - Status *Status `json:"status,omitempty"` + Status *RBDMirrorStatus `json:"status,omitempty"` +} + +// RBDMirrorStatus represents the status of the RBD mirror resource +type RBDMirrorStatus struct { + Status `json:",inline"` + Cephx LocalCephxStatus `json:"cephx,omitempty"` } // CephRBDMirrorList represents a list Ceph RBD Mirrors diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index 8e93f0c77..70be2d75f 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -1791,7 +1791,7 @@ func (in *CephRBDMirror) DeepCopyInto(out *CephRBDMirror) { in.Spec.DeepCopyInto(&out.Spec) if in.Status != nil { in, out := &in.Status, &out.Status - *out = new(Status) + *out = new(RBDMirrorStatus) (*in).DeepCopyInto(*out) } return @@ -4669,6 +4669,24 @@ func (in *QuotaSpec) DeepCopy() *QuotaSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RBDMirrorStatus) DeepCopyInto(out *RBDMirrorStatus) { + *out = *in + in.Status.DeepCopyInto(&out.Status) + out.Cephx = in.Cephx + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RBDMirrorStatus. +func (in *RBDMirrorStatus) DeepCopy() *RBDMirrorStatus { + if in == nil { + return nil + } + out := new(RBDMirrorStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RBDMirroringSpec) DeepCopyInto(out *RBDMirroringSpec) { *out = *in diff --git a/pkg/operator/ceph/cluster/rbd/config.go b/pkg/operator/ceph/cluster/rbd/config.go index b34eb1837..62df748cd 100644 --- a/pkg/operator/ceph/cluster/rbd/config.go +++ b/pkg/operator/ceph/cluster/rbd/config.go @@ -59,6 +59,15 @@ func (r *ReconcileCephRBDMirror) generateKeyring(clusterInfo *client.ClusterInfo return "", err } + if r.shouldRotateCephxKeys { + logger.Infof("rotating cephx key for CephRBDMirror %q in the namespace %q", daemonConfig.ResourceName, clusterInfo.Namespace) + newKey, err := s.RotateKey(user) + if err != nil { + return "", errors.Wrapf(err, "failed to rotate cephx key for CephRDBMirror %q in the namespace %q", daemonConfig.ResourceName, clusterInfo.Namespace) + } + key = newKey + } + // Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x err = r.context.Clientset.CoreV1().Secrets(clusterInfo.Namespace).Delete(r.opManagerContext, daemonConfig.ResourceName, metav1.DeleteOptions{}) if err != nil { diff --git a/pkg/operator/ceph/cluster/rbd/controller.go b/pkg/operator/ceph/cluster/rbd/controller.go index d8a1f7622..5f7910469 100644 --- a/pkg/operator/ceph/cluster/rbd/controller.go +++ b/pkg/operator/ceph/cluster/rbd/controller.go @@ -27,6 +27,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/config/keyring" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/reporting" "github.com/rook/rook/pkg/operator/k8sutil" @@ -37,6 +38,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -70,15 +72,16 @@ var currentAndDesiredCephVersion = opcontroller.CurrentAndDesiredCephVersion // ReconcileCephRBDMirror reconciles a cephRBDMirror object type ReconcileCephRBDMirror struct { - context *clusterd.Context - clusterInfo *cephclient.ClusterInfo - client client.Client - scheme *runtime.Scheme - cephClusterSpec *cephv1.ClusterSpec - peers map[string]*peerSpec - opManagerContext context.Context - opConfig opcontroller.OperatorConfig - recorder record.EventRecorder + context *clusterd.Context + clusterInfo *cephclient.ClusterInfo + client client.Client + scheme *runtime.Scheme + cephClusterSpec *cephv1.ClusterSpec + peers map[string]*peerSpec + opManagerContext context.Context + opConfig opcontroller.OperatorConfig + recorder record.EventRecorder + shouldRotateCephxKeys bool } // peerSpec represents peer details @@ -163,7 +166,10 @@ func (r *ReconcileCephRBDMirror) Reconcile(context context.Context, request reco // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephRBDMirror, err := r.reconcile(request) if err != nil { - r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.FailedStatus) + err := r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.FailedStatus, nil) + if err != nil { + return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed set ready failure status to the cephRBDMirror %q", request.NamespacedName) + } logger.Errorf("failed to reconcile %v", err) } @@ -185,7 +191,16 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile // The CR was just created, initializing status fields if cephRBDMirror.Status == nil { - r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.EmptyStatus) + cephxUninitialized := keyring.UninitializedCephxStatus() + err := r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.EmptyStatus, &cephxUninitialized) + if err != nil { + return opcontroller.ImmediateRetryResult, *cephRBDMirror, errors.Wrapf(err, "failed set empty status to the cephRBDMirror %q", request.NamespacedName) + } + cephRBDMirror.Status = &cephv1.RBDMirrorStatus{ + Cephx: cephv1.LocalCephxStatus{ + Daemon: cephxUninitialized, + }, + } } // update observedGeneration local variable with current generation value, // because generation can be changed before reconcile got completed @@ -249,6 +264,15 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile return opcontroller.ImmediateRetryResult, *cephRBDMirror, errors.Wrap(err, "failed to add ceph rbd mirror peer") } + // check if cephRBDMirror daemon keys should be rotated or not + r.shouldRotateCephxKeys, err = keyring.ShouldRotateCephxKeys(cephCluster.Spec.Security.CephX.Daemon, *runningCephVersion, *runningCephVersion, cephRBDMirror.Status.Cephx.Daemon) + if err != nil { + return reconcile.Result{}, *cephRBDMirror, errors.Wrapf(err, "failed to determine if cephx keys should be rotated for the cephRBDMirror %q", request.NamespacedName) + } + if r.shouldRotateCephxKeys { + logger.Infof("cephx keys for CephRBDMirror %q will be rotated", request.NamespacedName) + } + // CREATE/UPDATE logger.Debug("reconciling ceph rbd mirror deployments") reconcileResponse, err = r.reconcileCreateCephRBDMirror(cephRBDMirror) @@ -256,9 +280,14 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile return opcontroller.ImmediateRetryResult, *cephRBDMirror, errors.Wrap(err, "failed to create ceph rbd mirror deployments") } + cephxStatus := keyring.UpdatedCephxStatus(r.shouldRotateCephxKeys, r.cephClusterSpec.Security.CephX.Daemon, r.clusterInfo.CephVersion, cephRBDMirror.Status.Cephx.Daemon) + // update ObservedGeneration in status at the end of reconcile // Set Ready status, we are done reconciling - r.updateStatus(observedGeneration, request.NamespacedName, k8sutil.ReadyStatus) + err = r.updateStatus(observedGeneration, request.NamespacedName, k8sutil.ReadyStatus, &cephxStatus) + if err != nil { + return opcontroller.ImmediateRetryResult, *cephRBDMirror, errors.Wrapf(err, "failed set ready status to the cephRBDMirror %q", request.NamespacedName) + } // Return and do not requeue logger.Debug("done reconciling ceph rbd mirror") @@ -284,29 +313,42 @@ func (r *ReconcileCephRBDMirror) reconcileCreateCephRBDMirror(cephRBDMirror *cep } // updateStatus updates an object with a given status -func (r *ReconcileCephRBDMirror) updateStatus(observedGeneration int64, name types.NamespacedName, status string) { +func (r *ReconcileCephRBDMirror) updateStatus(observedGeneration int64, namespacedName types.NamespacedName, status string, cephx *cephv1.CephxStatus) error { rbdMirror := &cephv1.CephRBDMirror{} - err := r.client.Get(r.opManagerContext, name, rbdMirror) - if err != nil { - if kerrors.IsNotFound(err) { - logger.Debug("CephRBDMirror resource not found. Ignoring since object must be deleted.") - return + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + err := r.client.Get(r.opManagerContext, namespacedName, rbdMirror) + if err != nil { + if kerrors.IsNotFound(err) { + logger.Debug("CephRBDMirror resource not found. Ignoring since object must be deleted.") + return nil + } + return errors.Wrapf(err, "failed to retrieve CephRBDMirror %q to update status to %q", namespacedName, status) } - logger.Warningf("failed to retrieve rbd mirror %q to update status to %q. %v", name, status, err) - return + + if rbdMirror.Status == nil { + rbdMirror.Status = &cephv1.RBDMirrorStatus{} + } + + rbdMirror.Status.Phase = status + if observedGeneration != k8sutil.ObservedGenerationNotAvailable { + rbdMirror.Status.ObservedGeneration = observedGeneration + } + + if cephx != nil { + rbdMirror.Status.Cephx.Daemon = *cephx + } + + if err := reporting.UpdateStatus(r.client, rbdMirror); err != nil { + return errors.Wrapf(err, "failed to set CephRBDMirror %q status to %+v", namespacedName, rbdMirror.Status) + } + + return nil + }) + if err != nil { + return err } - if rbdMirror.Status == nil { - rbdMirror.Status = &cephv1.Status{} - } - - rbdMirror.Status.Phase = status - if observedGeneration != k8sutil.ObservedGenerationNotAvailable { - rbdMirror.Status.ObservedGeneration = observedGeneration - } - if err := reporting.UpdateStatus(r.client, rbdMirror); err != nil { - logger.Errorf("failed to set rbd mirror %q status to %q. %v", rbdMirror.Name, status, err) - return - } - logger.Debugf("rbd mirror %q status updated to %q", name, status) + logger.Debugf("successfully updated cephRBDMirror %q status updated to %+v", namespacedName, rbdMirror.Status) + return nil } diff --git a/pkg/operator/ceph/cluster/rbd/controller_test.go b/pkg/operator/ceph/cluster/rbd/controller_test.go index 922459f76..f84e982e5 100644 --- a/pkg/operator/ceph/cluster/rbd/controller_test.go +++ b/pkg/operator/ceph/cluster/rbd/controller_test.go @@ -30,9 +30,11 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" + testopk8s "github.com/rook/rook/pkg/operator/k8sutil/test" "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" + apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -98,6 +100,13 @@ func TestCephRBDMirrorController(t *testing.T) { Name: namespace, Namespace: namespace, }, + Spec: cephv1.ClusterSpec{ + Security: cephv1.ClusterSecuritySpec{ + CephX: cephv1.ClusterCephxConfig{ + Daemon: cephv1.CephxConfig{}, + }, + }, + }, Status: cephv1.ClusterStatus{ Phase: "", CephStatus: &cephv1.CephStatus{ @@ -234,3 +243,240 @@ func TestCephRBDMirrorController(t *testing.T) { assert.Equal(t, "Ready", rbdMirror.Status.Phase, rbdMirror) }) } + +func TestMirrorKeyRotation(t *testing.T) { + ctx := context.TODO() + var ( + name = "my-mirror" + namespace = "rook-ceph" + ) + // Set DEBUG logging + t.Setenv("ROOK_LOG_LEVEL", "DEBUG") + + currentAndDesiredCephVersion = func(ctx context.Context, rookImage, namespace, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) { + rotationSupportedVer := version.CephVersion{Major: 20, Minor: 2, Extra: 0} + return &rotationSupportedVer, &rotationSupportedVer, nil + } + + var deploymentsUpdated *[]*apps.Deployment + updateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub() + + // An rbd-mirror resource with metadata and spec. + rbdMirror := &cephv1.CephRBDMirror{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: cephv1.RBDMirroringSpec{ + Count: 1, + }, + TypeMeta: controllerTypeMeta, + } + + // Mock request to simulate Reconcile() being called on an event for a + // watched resource . + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: name, + Namespace: namespace, + }, + } + + cephCluster := &cephv1.CephCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Namespace: namespace, + }, + Spec: cephv1.ClusterSpec{ + Security: cephv1.ClusterSecuritySpec{ + CephX: cephv1.ClusterCephxConfig{ + Daemon: cephv1.CephxConfig{}, + }, + }, + }, + Status: cephv1.ClusterStatus{ + Phase: k8sutil.ReadyStatus, + CephStatus: &cephv1.CephStatus{ + Health: "HEALTH_OK", + }, + }, + } + + // Objects to track in the fake client. + object := []runtime.Object{ + rbdMirror, + cephCluster, + } + + s := scheme.Scheme + + mirrorDaemonRotatedKey := `{"key":"AQCvzWBeIV9lFRAAninzm+8XFxbSfTiPwoX50g=="}` + executor := &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { + if args[0] == "status" { + return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_ERR"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil + } + if args[0] == "auth" && args[1] == "get-or-create-key" { + return cephAuthGetOrCreateKey, nil + } + if args[0] == "auth" && args[1] == "rotate" { + t.Logf("rotating key and returning: %s", mirrorDaemonRotatedKey) + return mirrorDaemonRotatedKey, nil + } + if args[0] == "versions" { + return dummyVersionsRaw, nil + } + if args[0] == "mirror" && args[1] == "pool" && args[2] == "info" { + return `{"mode":"image","site_name":"39074576-5884-4ef3-8a4d-8a0c5ed33031","peers":[{"uuid":"4a6983c0-3c9d-40f5-b2a9-2334a4659827","direction":"rx-tx","site_name":"ocs","mirror_uuid":"","client_name":"client.rbd-mirror-peer"}]}`, nil + } + if args[0] == "mirror" && args[1] == "pool" && args[2] == "status" { + return `{"summary":{"health":"WARNING","daemon_health":"OK","image_health":"WARNING","states":{"unknown":1}}}`, nil + } + return "", nil + }, + } + clientset := test.New(t, 3) + c := &clusterd.Context{ + Executor: executor, + RookClientset: rookclient.NewSimpleClientset(), + Clientset: clientset, + } + + // Register operator types with the runtime scheme. + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephRBDMirror{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}) + + // Create a fake client to mock API calls. + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + r := &ReconcileCephRBDMirror{client: cl, scheme: s, context: c, opManagerContext: ctx, recorder: record.NewFakeRecorder(5)} + + t.Run("first reconcile", func(t *testing.T) { + // Mock clusterInfo + secrets := map[string][]byte{ + "fsid": []byte(name), + "mon-secret": []byte("monsecret"), + "admin-secret": []byte("adminsecret"), + } + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-ceph-mon", + Namespace: namespace, + }, + Data: secrets, + Type: k8sutil.RookType, + } + _, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + assert.NoError(t, err) + + _, err = r.Reconcile(ctx, req) + assert.NoError(t, err) + + mirror := cephv1.CephRBDMirror{} + err = cl.Get(ctx, req.NamespacedName, &mirror) + assert.NoError(t, err) + assert.Equal(t, uint32(1), mirror.Status.Cephx.Daemon.KeyGeneration) + assert.Equal(t, "20.2.0-0", mirror.Status.Cephx.Daemon.KeyCephVersion) + }) + + t.Run("subsequent reconcile - retain cephx status", func(t *testing.T) { + r := &ReconcileCephRBDMirror{client: cl, scheme: s, context: c, opManagerContext: ctx, recorder: record.NewFakeRecorder(5)} + _, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + mirror := cephv1.CephRBDMirror{} + err = cl.Get(ctx, req.NamespacedName, &mirror) + assert.NoError(t, err) + assert.Equal(t, uint32(1), mirror.Status.Cephx.Daemon.KeyGeneration) + assert.Equal(t, "20.2.0-0", mirror.Status.Cephx.Daemon.KeyCephVersion) + testopk8s.ClearDeploymentsUpdated(deploymentsUpdated) + }) + + t.Run("brownfield reconcile - retain unknown cephx status", func(t *testing.T) { + mirror := cephv1.CephRBDMirror{} + err := cl.Get(ctx, req.NamespacedName, &mirror) + assert.NoError(t, err) + mirror.Status.Cephx.Daemon = cephv1.CephxStatus{} + err = cl.Update(ctx, &mirror) + assert.NoError(t, err) + + _, err = r.Reconcile(ctx, req) + assert.NoError(t, err) + + err = cl.Get(ctx, req.NamespacedName, &mirror) + assert.NoError(t, err) + assert.Equal(t, cephv1.CephxStatus{}, mirror.Status.Cephx.Daemon) + }) + + t.Run("rotate key - brownfield unknown status becomes known", func(t *testing.T) { + cluster := cephv1.CephCluster{} + err := cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: namespace}, &cluster) + assert.NoError(t, err) + cluster.Spec.Security.CephX.Daemon = cephv1.CephxConfig{ + KeyRotationPolicy: "KeyGeneration", + KeyGeneration: 2, + } + err = cl.Update(ctx, &cluster) + assert.NoError(t, err) + + mirrorDaemonRotatedKey = `[{"key":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=="}]` + + _, err = r.Reconcile(ctx, req) + assert.NoError(t, err) + + mirror := cephv1.CephRBDMirror{} + err = cl.Get(ctx, req.NamespacedName, &mirror) + assert.NoError(t, err) + assert.Equal(t, uint32(2), mirror.Status.Cephx.Daemon.KeyGeneration) + assert.Equal(t, "20.2.0-0", mirror.Status.Cephx.Daemon.KeyCephVersion) + + secret, err := clientset.CoreV1().Secrets(namespace).Get(ctx, "rook-ceph-rbd-mirror-a-keyring", metav1.GetOptions{}) + assert.NoError(t, err) + assert.Contains(t, secret.StringData["keyring"], "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB==") + }) + + t.Run("brownfield reconcile - no further rotation happens", func(t *testing.T) { + // not expecting any rotation. So `ceph auth rotate` should not run and secret should not be updated + mirrorDaemonRotatedKey = `[{"key":"CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC=="}]` + + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + + mirror := cephv1.CephRBDMirror{} + err = cl.Get(ctx, req.NamespacedName, &mirror) + assert.NoError(t, err) + assert.Equal(t, uint32(2), mirror.Status.Cephx.Daemon.KeyGeneration) + assert.Equal(t, "20.2.0-0", mirror.Status.Cephx.Daemon.KeyCephVersion) + + secret, err := clientset.CoreV1().Secrets(namespace).Get(ctx, "rook-ceph-rbd-mirror-a-keyring", metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotContains(t, secret.StringData["keyring"], "CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC==") + testopk8s.ClearDeploymentsUpdated(deploymentsUpdated) + }) + + t.Run("rotate key - cephx status updated", func(t *testing.T) { + cluster := cephv1.CephCluster{} + err := cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: namespace}, &cluster) + assert.NoError(t, err) + cluster.Spec.Security.CephX.Daemon = cephv1.CephxConfig{ + KeyRotationPolicy: "KeyGeneration", + KeyGeneration: 3, + } + err = cl.Update(ctx, &cluster) + assert.NoError(t, err) + + mirrorDaemonRotatedKey = `[{"key":"CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC=="}]` + + _, err = r.Reconcile(ctx, req) + assert.NoError(t, err) + + mirror := cephv1.CephRBDMirror{} + err = cl.Get(ctx, req.NamespacedName, &mirror) + assert.NoError(t, err) + assert.Equal(t, uint32(3), mirror.Status.Cephx.Daemon.KeyGeneration) + assert.Equal(t, "20.2.0-0", mirror.Status.Cephx.Daemon.KeyCephVersion) + + secret, err := clientset.CoreV1().Secrets(namespace).Get(ctx, "rook-ceph-rbd-mirror-a-keyring", metav1.GetOptions{}) + assert.NoError(t, err) + assert.Contains(t, secret.StringData["keyring"], "CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC==") + }) +} diff --git a/pkg/operator/ceph/cluster/rbd/mirror.go b/pkg/operator/ceph/cluster/rbd/mirror.go index ee02959f7..6ec03bd36 100644 --- a/pkg/operator/ceph/cluster/rbd/mirror.go +++ b/pkg/operator/ceph/cluster/rbd/mirror.go @@ -25,6 +25,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/config/keyring" "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -62,11 +63,6 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro ownerInfo: ownerInfo, } - _, err = r.generateKeyring(r.clusterInfo, daemonConf) - if err != nil { - return errors.Wrapf(err, "failed to generate keyring for %q", resourceName) - } - rbdMirrorToSkipReconcile, err := controller.GetDaemonsToSkipReconcile(r.clusterInfo.Context, r.context, r.clusterInfo.Namespace, config.RbdMirrorType, AppName) if err != nil { return errors.Wrap(err, "failed to check for RBD Mirror to skip reconcile") @@ -76,12 +72,20 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro return nil } + secretResourceVersion, err := r.generateKeyring(r.clusterInfo, daemonConf) + if err != nil { + return errors.Wrapf(err, "failed to generate keyring for %q", resourceName) + } + // Start the deployment d, err := r.makeDeployment(daemonConf, cephRBDMirror) if err != nil { return errors.Wrap(err, "failed to create rbd-mirror deployment") } + // apply cephx secret resource version to the deployment to ensure it restarts when keyring updates + d.Spec.Template.Annotations[keyring.CephxKeyIdentifierAnnotation] = secretResourceVersion + // Set owner ref to cephRBDMirror object err = controllerutil.SetControllerReference(cephRBDMirror, d, r.scheme) if err != nil {