diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index 0ee815f1b..3f770c74f 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -5095,6 +5095,19 @@ CephxStatus

Crash Collector represents the cephx key rotation status of the crash collector daemon

+ + +cephExporter
+ + +CephxStatus + + + + +

Ceph Exporter represents the cephx key rotation status of the ceph exporter daemon

+ +

ClusterSecuritySpec diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 4767345bf..aba6bbb2f 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -5927,6 +5927,28 @@ spec: cephx: description: ClusterCephxStatus defines the cephx key rotation status of various daemons on the cephCluster resource properties: + cephExporter: + description: Ceph Exporter represents the cephx key rotation status of the ceph exporter daemon + properties: + keyCephVersion: + description: |- + KeyCephVersion reports the Ceph version that created the current generation's keys. This is + same string format as reported by `CephCluster.status.version.version` to allow them to be + compared. E.g., `20.2.0-0`. + For all newly-created resources, this field set to the version of Ceph that created the key. + The special value "Uninitialized" indicates that keys are being created for the first time. + An empty string indicates that the version is unknown, as expected in brownfield deployments. + type: string + keyGeneration: + description: |- + KeyGeneration represents the CephX key generation for the last successful reconcile. + For all newly-created resources, this field is set to `1`. + When keys are rotated due to any rotation policy, the generation is incremented or updated to + the configured policy generation. + Generation `0` indicates that keys existed prior to the implementation of key tracking. + format: int32 + type: integer + type: object crashCollector: description: Crash Collector represents the cephx key rotation status of the crash collector daemon properties: diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index bc38e993a..4c23220e6 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -5925,6 +5925,28 @@ spec: cephx: description: ClusterCephxStatus defines the cephx key rotation status of various daemons on the cephCluster resource properties: + cephExporter: + description: Ceph Exporter represents the cephx key rotation status of the ceph exporter daemon + properties: + keyCephVersion: + description: |- + KeyCephVersion reports the Ceph version that created the current generation's keys. This is + same string format as reported by `CephCluster.status.version.version` to allow them to be + compared. E.g., `20.2.0-0`. + For all newly-created resources, this field set to the version of Ceph that created the key. + The special value "Uninitialized" indicates that keys are being created for the first time. + An empty string indicates that the version is unknown, as expected in brownfield deployments. + type: string + keyGeneration: + description: |- + KeyGeneration represents the CephX key generation for the last successful reconcile. + For all newly-created resources, this field is set to `1`. + When keys are rotated due to any rotation policy, the generation is incremented or updated to + the configured policy generation. + Generation `0` indicates that keys existed prior to the implementation of key tracking. + format: int32 + type: integer + type: object crashCollector: description: Crash Collector represents the cephx key rotation status of the crash collector daemon properties: diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index d2e4ebf79..a2a62e681 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -750,6 +750,8 @@ type ClusterCephxStatus struct { CSI *CephxStatusWithKeyCount `json:"csi,omitempty"` // Crash Collector represents the cephx key rotation status of the crash collector daemon CrashCollector *CephxStatus `json:"crashCollector,omitempty"` + // Ceph Exporter represents the cephx key rotation status of the ceph exporter daemon + CephExporter *CephxStatus `json:"cephExporter,omitempty"` } // MonSpec represents the specification of the monitor diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index 7c1ac02d3..fd9e86fae 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -2095,6 +2095,11 @@ func (in *ClusterCephxStatus) DeepCopyInto(out *ClusterCephxStatus) { *out = new(CephxStatus) **out = **in } + if in.CephExporter != nil { + in, out := &in.CephExporter, &out.CephExporter + *out = new(CephxStatus) + **out = **in + } return } diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index ee9a238ab..ae884de16 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -804,6 +804,7 @@ func initClusterCephxStatus(c *clusterd.Context, cluster *cephv1.CephCluster) er CephxStatus: uninitializedStatus, }, CrashCollector: &uninitializedStatus, + CephExporter: &uninitializedStatus, } if err := reporting.UpdateStatus(c.Client, cluster); err != nil { diff --git a/pkg/operator/ceph/cluster/nodedaemon/keyring.go b/pkg/operator/ceph/cluster/nodedaemon/keyring.go index 69619ceab..1a61f67f2 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/keyring.go +++ b/pkg/operator/ceph/cluster/nodedaemon/keyring.go @@ -107,7 +107,7 @@ func createCrashCollectorKeyring(s *keyring.SecretStore, context *clusterd.Conte err = updateCrashCollectorCephxStatus(context, clusterInfo, shouldRotateCephxKeys) if err != nil { - return "", errors.Wrapf(err, "failed to update crash collector cephx status for cluster %q", clusterInfo.Namespace) + return "", errors.Wrapf(err, "failed to update crash collector cephx status for cluster %q", clusterInfo.NamespacedName()) } return key, nil @@ -169,7 +169,7 @@ func CreateExporterSecret(context *clusterd.Context, clusterInfo *client.Cluster k := keyring.GetSecretStore(context, clusterInfo, clusterInfo.OwnerInfo) // Create exporter Ceph key - exporterSecretKey, err := createExporterKeyring(k) + exporterSecretKey, err := createExporterKeyring(k, context, clusterInfo) if err != nil { return errors.Wrapf(err, "failed to create %q ceph keyring", exporterKeyringUsername) } @@ -191,15 +191,65 @@ func createExporterKeyringCaps() []string { } } -func createExporterKeyring(s *keyring.SecretStore) (string, error) { +func createExporterKeyring(s *keyring.SecretStore, context *clusterd.Context, clusterInfo *client.ClusterInfo) (string, error) { key, err := s.GenerateKey(exporterKeyringUsername, createExporterKeyringCaps()) if err != nil { return "", err } + clusterObj := &cephv1.CephCluster{} + if err := context.Client.Get(clusterInfo.Context, clusterInfo.NamespacedName(), clusterObj); err != nil { + return "", errors.Wrapf(err, "failed to get cluster %v", clusterInfo.NamespacedName()) + } + // TODO: for rotation WithCephVersionUpdate fix this to have the right runningCephVersion and desiredCephVersion + shouldRotateCephxKeys, err := keyring.ShouldRotateCephxKeys( + clusterObj.Spec.Security.CephX.Daemon, + clusterInfo.CephVersion, clusterInfo.CephVersion, + *clusterObj.Status.Cephx.CephExporter, + ) + if err != nil { + return "", errors.Wrapf(err, "failed to check if cephx keys should be rotated for ceph exporter %q", exporterKeyringUsername) + } + + if shouldRotateCephxKeys { + logger.Infof("rotating cephx key for ceph exporter %q", exporterKeyringUsername) + newKey, err := s.RotateKey(exporterKeyringUsername) + if err != nil { + return "", errors.Wrapf(err, "failed to rotate cephx key for ceph exporter %q", exporterKeyringUsername) + } + key = newKey + } + + err = updateCephExporterCephxStatus(context, clusterInfo, shouldRotateCephxKeys) + if err != nil { + return "", errors.Wrapf(err, "failed to update ceph exporter cephx status for cluster %q", clusterInfo.NamespacedName()) + } + return key, nil } +func updateCephExporterCephxStatus(context *clusterd.Context, clusterInfo *client.ClusterInfo, didRotate bool) error { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + cluster := &cephv1.CephCluster{} + if err := context.Client.Get(clusterInfo.Context, clusterInfo.NamespacedName(), cluster); err != nil { + return errors.Wrapf(err, "failed to get cluster %v to update the conditions.", clusterInfo.NamespacedName()) + } + updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, clusterInfo.CephVersion, *cluster.Status.Cephx.CephExporter) + cluster.Status.Cephx.CephExporter = &updatedStatus + if err := reporting.UpdateStatus(context.Client, cluster); err != nil { + return errors.Wrap(err, "failed to update cluster cephx status for ceph exporter daemon") + } + logger.Debugf("successfully updated the ceph exporter cephx status for cluster in namespace %q to %+v", cluster.Namespace, cluster.Status.Cephx.CephExporter) + + return nil + }) + if err != nil { + return errors.Wrap(err, "failed to update cluster cephx status for ceph exporter daemon") + } + + return nil +} + func createOrUpdateExporterSecret(clusterInfo *client.ClusterInfo, exporterSecretKey string, k *keyring.SecretStore) error { keyring := fmt.Sprintf(exporterKeyringTemplate, exporterSecretKey) diff --git a/pkg/operator/ceph/cluster/nodedaemon/keyring_test.go b/pkg/operator/ceph/cluster/nodedaemon/keyring_test.go index cb3abe90e..31c06d60a 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/keyring_test.go +++ b/pkg/operator/ceph/cluster/nodedaemon/keyring_test.go @@ -18,6 +18,7 @@ package nodedaemon import ( "context" + "fmt" "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -32,6 +33,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +const ( + key = "AQCvzWBeIV9lFRAAninzm+8XFxbSfTiPwoX50g==" + rotatedKey = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" +) + func TestCephCrashCollectorKeyringCaps(t *testing.T) { caps := cephCrashCollectorKeyringCaps() assert.Equal(t, caps, []string{"mon", "allow profile crash", "mgr", "allow rw"}) @@ -94,11 +100,11 @@ func TestCreateCrashCollectorKeyring(t *testing.T) { cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build() clusterContext.Client = cl - rotatedKeyJson := `[{"key":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="}]` + rotatedKeyJson := fmt.Sprintf(`[{"key":"%s"}]`, rotatedKey) executor := &exectest.MockExecutor{ MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { if args[0] == "auth" && args[1] == "get-or-create-key" { - return `{"key":"AQCvzWBeIV9lFRAAninzm+8XFxbSfTiPwoX50g=="}`, nil + return fmt.Sprintf(`{"key":"%s"}`, key), nil } if args[0] == "auth" && args[1] == "rotate" { t.Logf("rotating key and returning: %s", rotatedKeyJson) @@ -113,7 +119,7 @@ func TestCreateCrashCollectorKeyring(t *testing.T) { k := keyring.GetSecretStore(clusterContext, clusterInfo, clusterInfo.OwnerInfo) key1, err := createCrashCollectorKeyring(k, clusterContext, clusterInfo) assert.NoError(t, err) - assert.Equal(t, "AQCvzWBeIV9lFRAAninzm+8XFxbSfTiPwoX50g==", key1) + assert.Equal(t, key, key1) // do key rotation err = clusterContext.Client.Get(ctx, clusterInfo.NamespacedName(), cephCluster) @@ -124,7 +130,7 @@ func TestCreateCrashCollectorKeyring(t *testing.T) { clusterInfo.CephVersion = cephver.CephVersion{Major: 20, Minor: 2, Extra: 0} key2, err := createCrashCollectorKeyring(k, clusterContext, clusterInfo) assert.NoError(t, err) - assert.Equal(t, "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", key2) + assert.Equal(t, rotatedKey, key2) // verify that the cephx status is updated // get the cluster again to ensure we have the latest status @@ -133,3 +139,95 @@ func TestCreateCrashCollectorKeyring(t *testing.T) { assert.NoError(t, err) assert.Equal(t, uint32(2), updatedCluster.Status.Cephx.CrashCollector.KeyGeneration) } + +func TestCreateCephExporterKeyring(t *testing.T) { + clusterContext := &clusterd.Context{} + ctx := context.TODO() + clusterInfo := &cephclient.ClusterInfo{ + Context: ctx, + Namespace: "rook-ceph", + CephVersion: cephver.Reef, + } + + clusterInfo.SetName("mycluster") + clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) + + // create a sample ceph cluster at add to fake controller + status := keyring.UninitializedCephxStatus() + cephCluster := &cephv1.CephCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mycluster", + Namespace: "rook-ceph", + }, + Spec: cephv1.ClusterSpec{ + Security: cephv1.ClusterSecuritySpec{ + CephX: cephv1.ClusterCephxConfig{ + Daemon: cephv1.CephxConfig{ + KeyRotationPolicy: "KeyGeneration", + KeyGeneration: 1, + }, + }, + }, + }, + Status: cephv1.ClusterStatus{ + Phase: "", + CephVersion: &cephv1.ClusterVersion{ + Version: "14.2.9-0", + }, + CephStatus: &cephv1.CephStatus{ + Health: "", + }, + Cephx: &cephv1.ClusterCephxStatus{ + CephExporter: &status, + }, + }, + } + + objects := []runtime.Object{ + cephCluster, + } + s := runtime.NewScheme() + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}) + + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build() + clusterContext.Client = cl + + rotatedKeyJson := fmt.Sprintf(`[{"key":"%s"}]`, rotatedKey) + executor := &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { + if args[0] == "auth" && args[1] == "get-or-create-key" { + return fmt.Sprintf(`{"key":"%s"}`, key), nil + } + if args[0] == "auth" && args[1] == "rotate" { + t.Logf("rotating key and returning: %s", rotatedKeyJson) + return rotatedKeyJson, nil + } + + return "", nil + }, + } + clusterContext.Executor = executor + + k := keyring.GetSecretStore(clusterContext, clusterInfo, clusterInfo.OwnerInfo) + key1, err := createExporterKeyring(k, clusterContext, clusterInfo) + assert.NoError(t, err) + assert.Equal(t, key, key1) + + // do key rotation + err = clusterContext.Client.Get(ctx, clusterInfo.NamespacedName(), cephCluster) + assert.NoError(t, err) + cephCluster.Spec.Security.CephX.Daemon.KeyGeneration = 2 + err = cl.Update(ctx, cephCluster) + assert.NoError(t, err) + clusterInfo.CephVersion = cephver.CephVersion{Major: 20, Minor: 2, Extra: 0} + key2, err := createExporterKeyring(k, clusterContext, clusterInfo) + assert.NoError(t, err) + assert.Equal(t, rotatedKey, key2) + + // verify that the cephx status is updated + // get the cluster again to ensure we have the latest status + updatedCluster := &cephv1.CephCluster{} + err = clusterContext.Client.Get(ctx, clusterInfo.NamespacedName(), updatedCluster) + assert.NoError(t, err) + assert.Equal(t, uint32(2), updatedCluster.Status.Cephx.CephExporter.KeyGeneration) +}