exporter: automate rotate exporter cephx keys

rotate cephx keys based on keyRotationPolicy
and update cephxStatus for the ceph exporter daemon pods.

Signed-off-by: parth-gr <partharora1010@gmail.com>
(cherry picked from commit 9ec7a767c3)
This commit is contained in:
parth-gr authored and Mergify committed 2025-08-01 16:11:14 +00:00
1 parent 9a838d185b
commit 3df15cacef
8 files changed
+220 -7

No files matched your search

+13
View File
@@ -5095,6 +5095,19 @@ CephxStatus
<p>Crash Collector represents the cephx key rotation status of the crash collector daemon</p>
</td>
</tr>
<tr>
<td>
<code>cephExporter</code><br/>
<em>
<a href="#ceph.rook.io/v1.CephxStatus">
CephxStatus
</a>
</em>
</td>
<td>
<p>Ceph Exporter represents the cephx key rotation status of the ceph exporter daemon</p>
</td>
</tr>
</tbody>
</table>
<h3 id="ceph.rook.io/v1.ClusterSecuritySpec">ClusterSecuritySpec
@@ -5927,6 +5927,28 @@ spec:
cephx:
description: ClusterCephxStatus defines the cephx key rotation status of various daemons on the cephCluster resource
properties:
cephExporter:
description: Ceph Exporter represents the cephx key rotation status of the ceph exporter daemon
properties:
keyCephVersion:
description: |-
KeyCephVersion reports the Ceph version that created the current generation's keys. This is
same string format as reported by `CephCluster.status.version.version` to allow them to be
compared. E.g., `20.2.0-0`.
For all newly-created resources, this field set to the version of Ceph that created the key.
The special value "Uninitialized" indicates that keys are being created for the first time.
An empty string indicates that the version is unknown, as expected in brownfield deployments.
type: string
keyGeneration:
description: |-
KeyGeneration represents the CephX key generation for the last successful reconcile.
For all newly-created resources, this field is set to `1`.
When keys are rotated due to any rotation policy, the generation is incremented or updated to
the configured policy generation.
Generation `0` indicates that keys existed prior to the implementation of key tracking.
format: int32
type: integer
type: object
crashCollector:
description: Crash Collector represents the cephx key rotation status of the crash collector daemon
properties:
+22
View File
@@ -5925,6 +5925,28 @@ spec:
cephx:
description: ClusterCephxStatus defines the cephx key rotation status of various daemons on the cephCluster resource
properties:
cephExporter:
description: Ceph Exporter represents the cephx key rotation status of the ceph exporter daemon
properties:
keyCephVersion:
description: |-
KeyCephVersion reports the Ceph version that created the current generation's keys. This is
same string format as reported by `CephCluster.status.version.version` to allow them to be
compared. E.g., `20.2.0-0`.
For all newly-created resources, this field set to the version of Ceph that created the key.
The special value "Uninitialized" indicates that keys are being created for the first time.
An empty string indicates that the version is unknown, as expected in brownfield deployments.
type: string
keyGeneration:
description: |-
KeyGeneration represents the CephX key generation for the last successful reconcile.
For all newly-created resources, this field is set to `1`.
When keys are rotated due to any rotation policy, the generation is incremented or updated to
the configured policy generation.
Generation `0` indicates that keys existed prior to the implementation of key tracking.
format: int32
type: integer
type: object
crashCollector:
description: Crash Collector represents the cephx key rotation status of the crash collector daemon
properties:
+2
View File
@@ -750,6 +750,8 @@ type ClusterCephxStatus struct {
CSI *CephxStatusWithKeyCount `json:"csi,omitempty"`
// Crash Collector represents the cephx key rotation status of the crash collector daemon
CrashCollector *CephxStatus `json:"crashCollector,omitempty"`
// Ceph Exporter represents the cephx key rotation status of the ceph exporter daemon
CephExporter *CephxStatus `json:"cephExporter,omitempty"`
}
// MonSpec represents the specification of the monitor
@@ -2095,6 +2095,11 @@ func (in *ClusterCephxStatus) DeepCopyInto(out *ClusterCephxStatus) {
*out = new(CephxStatus)
**out = **in
}
if in.CephExporter != nil {
in, out := &in.CephExporter, &out.CephExporter
*out = new(CephxStatus)
**out = **in
}
return
}
+1
View File
@@ -804,6 +804,7 @@ func initClusterCephxStatus(c *clusterd.Context, cluster *cephv1.CephCluster) er
CephxStatus: uninitializedStatus,
},
CrashCollector: &uninitializedStatus,
CephExporter: &uninitializedStatus,
}
if err := reporting.UpdateStatus(c.Client, cluster); err != nil {
@@ -107,7 +107,7 @@ func createCrashCollectorKeyring(s *keyring.SecretStore, context *clusterd.Conte
err = updateCrashCollectorCephxStatus(context, clusterInfo, shouldRotateCephxKeys)
if err != nil {
return "", errors.Wrapf(err, "failed to update crash collector cephx status for cluster %q", clusterInfo.Namespace)
return "", errors.Wrapf(err, "failed to update crash collector cephx status for cluster %q", clusterInfo.NamespacedName())
}
return key, nil
@@ -169,7 +169,7 @@ func CreateExporterSecret(context *clusterd.Context, clusterInfo *client.Cluster
k := keyring.GetSecretStore(context, clusterInfo, clusterInfo.OwnerInfo)
// Create exporter Ceph key
exporterSecretKey, err := createExporterKeyring(k)
exporterSecretKey, err := createExporterKeyring(k, context, clusterInfo)
if err != nil {
return errors.Wrapf(err, "failed to create %q ceph keyring", exporterKeyringUsername)
}
@@ -191,15 +191,65 @@ func createExporterKeyringCaps() []string {
}
}
func createExporterKeyring(s *keyring.SecretStore) (string, error) {
func createExporterKeyring(s *keyring.SecretStore, context *clusterd.Context, clusterInfo *client.ClusterInfo) (string, error) {
key, err := s.GenerateKey(exporterKeyringUsername, createExporterKeyringCaps())
if err != nil {
return "", err
}
clusterObj := &cephv1.CephCluster{}
if err := context.Client.Get(clusterInfo.Context, clusterInfo.NamespacedName(), clusterObj); err != nil {
return "", errors.Wrapf(err, "failed to get cluster %v", clusterInfo.NamespacedName())
}
// TODO: for rotation WithCephVersionUpdate fix this to have the right runningCephVersion and desiredCephVersion
shouldRotateCephxKeys, err := keyring.ShouldRotateCephxKeys(
clusterObj.Spec.Security.CephX.Daemon,
clusterInfo.CephVersion, clusterInfo.CephVersion,
*clusterObj.Status.Cephx.CephExporter,
)
if err != nil {
return "", errors.Wrapf(err, "failed to check if cephx keys should be rotated for ceph exporter %q", exporterKeyringUsername)
}
if shouldRotateCephxKeys {
logger.Infof("rotating cephx key for ceph exporter %q", exporterKeyringUsername)
newKey, err := s.RotateKey(exporterKeyringUsername)
if err != nil {
return "", errors.Wrapf(err, "failed to rotate cephx key for ceph exporter %q", exporterKeyringUsername)
}
key = newKey
}
err = updateCephExporterCephxStatus(context, clusterInfo, shouldRotateCephxKeys)
if err != nil {
return "", errors.Wrapf(err, "failed to update ceph exporter cephx status for cluster %q", clusterInfo.NamespacedName())
}
return key, nil
}
func updateCephExporterCephxStatus(context *clusterd.Context, clusterInfo *client.ClusterInfo, didRotate bool) error {
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
cluster := &cephv1.CephCluster{}
if err := context.Client.Get(clusterInfo.Context, clusterInfo.NamespacedName(), cluster); err != nil {
return errors.Wrapf(err, "failed to get cluster %v to update the conditions.", clusterInfo.NamespacedName())
}
updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, clusterInfo.CephVersion, *cluster.Status.Cephx.CephExporter)
cluster.Status.Cephx.CephExporter = &updatedStatus
if err := reporting.UpdateStatus(context.Client, cluster); err != nil {
return errors.Wrap(err, "failed to update cluster cephx status for ceph exporter daemon")
}
logger.Debugf("successfully updated the ceph exporter cephx status for cluster in namespace %q to %+v", cluster.Namespace, cluster.Status.Cephx.CephExporter)
return nil
})
if err != nil {
return errors.Wrap(err, "failed to update cluster cephx status for ceph exporter daemon")
}
return nil
}
func createOrUpdateExporterSecret(clusterInfo *client.ClusterInfo, exporterSecretKey string, k *keyring.SecretStore) error {
keyring := fmt.Sprintf(exporterKeyringTemplate, exporterSecretKey)
@@ -18,6 +18,7 @@ package nodedaemon
import (
"context"
"fmt"
"testing"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
@@ -32,6 +33,11 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)
const (
key = "AQCvzWBeIV9lFRAAninzm+8XFxbSfTiPwoX50g=="
rotatedKey = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
)
func TestCephCrashCollectorKeyringCaps(t *testing.T) {
caps := cephCrashCollectorKeyringCaps()
assert.Equal(t, caps, []string{"mon", "allow profile crash", "mgr", "allow rw"})
@@ -94,11 +100,11 @@ func TestCreateCrashCollectorKeyring(t *testing.T) {
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build()
clusterContext.Client = cl
rotatedKeyJson := `[{"key":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="}]`
rotatedKeyJson := fmt.Sprintf(`[{"key":"%s"}]`, rotatedKey)
executor := &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
if args[0] == "auth" && args[1] == "get-or-create-key" {
return `{"key":"AQCvzWBeIV9lFRAAninzm+8XFxbSfTiPwoX50g=="}`, nil
return fmt.Sprintf(`{"key":"%s"}`, key), nil
}
if args[0] == "auth" && args[1] == "rotate" {
t.Logf("rotating key and returning: %s", rotatedKeyJson)
@@ -113,7 +119,7 @@ func TestCreateCrashCollectorKeyring(t *testing.T) {
k := keyring.GetSecretStore(clusterContext, clusterInfo, clusterInfo.OwnerInfo)
key1, err := createCrashCollectorKeyring(k, clusterContext, clusterInfo)
assert.NoError(t, err)
assert.Equal(t, "AQCvzWBeIV9lFRAAninzm+8XFxbSfTiPwoX50g==", key1)
assert.Equal(t, key, key1)
// do key rotation
err = clusterContext.Client.Get(ctx, clusterInfo.NamespacedName(), cephCluster)
@@ -124,7 +130,7 @@ func TestCreateCrashCollectorKeyring(t *testing.T) {
clusterInfo.CephVersion = cephver.CephVersion{Major: 20, Minor: 2, Extra: 0}
key2, err := createCrashCollectorKeyring(k, clusterContext, clusterInfo)
assert.NoError(t, err)
assert.Equal(t, "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", key2)
assert.Equal(t, rotatedKey, key2)
// verify that the cephx status is updated
// get the cluster again to ensure we have the latest status
@@ -133,3 +139,95 @@ func TestCreateCrashCollectorKeyring(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, uint32(2), updatedCluster.Status.Cephx.CrashCollector.KeyGeneration)
}
func TestCreateCephExporterKeyring(t *testing.T) {
clusterContext := &clusterd.Context{}
ctx := context.TODO()
clusterInfo := &cephclient.ClusterInfo{
Context: ctx,
Namespace: "rook-ceph",
CephVersion: cephver.Reef,
}
clusterInfo.SetName("mycluster")
clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t)
// create a sample ceph cluster at add to fake controller
status := keyring.UninitializedCephxStatus()
cephCluster := &cephv1.CephCluster{
ObjectMeta: metav1.ObjectMeta{
Name: "mycluster",
Namespace: "rook-ceph",
},
Spec: cephv1.ClusterSpec{
Security: cephv1.ClusterSecuritySpec{
CephX: cephv1.ClusterCephxConfig{
Daemon: cephv1.CephxConfig{
KeyRotationPolicy: "KeyGeneration",
KeyGeneration: 1,
},
},
},
},
Status: cephv1.ClusterStatus{
Phase: "",
CephVersion: &cephv1.ClusterVersion{
Version: "14.2.9-0",
},
CephStatus: &cephv1.CephStatus{
Health: "",
},
Cephx: &cephv1.ClusterCephxStatus{
CephExporter: &status,
},
},
}
objects := []runtime.Object{
cephCluster,
}
s := runtime.NewScheme()
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{})
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build()
clusterContext.Client = cl
rotatedKeyJson := fmt.Sprintf(`[{"key":"%s"}]`, rotatedKey)
executor := &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
if args[0] == "auth" && args[1] == "get-or-create-key" {
return fmt.Sprintf(`{"key":"%s"}`, key), nil
}
if args[0] == "auth" && args[1] == "rotate" {
t.Logf("rotating key and returning: %s", rotatedKeyJson)
return rotatedKeyJson, nil
}
return "", nil
},
}
clusterContext.Executor = executor
k := keyring.GetSecretStore(clusterContext, clusterInfo, clusterInfo.OwnerInfo)
key1, err := createExporterKeyring(k, clusterContext, clusterInfo)
assert.NoError(t, err)
assert.Equal(t, key, key1)
// do key rotation
err = clusterContext.Client.Get(ctx, clusterInfo.NamespacedName(), cephCluster)
assert.NoError(t, err)
cephCluster.Spec.Security.CephX.Daemon.KeyGeneration = 2
err = cl.Update(ctx, cephCluster)
assert.NoError(t, err)
clusterInfo.CephVersion = cephver.CephVersion{Major: 20, Minor: 2, Extra: 0}
key2, err := createExporterKeyring(k, clusterContext, clusterInfo)
assert.NoError(t, err)
assert.Equal(t, rotatedKey, key2)
// verify that the cephx status is updated
// get the cluster again to ensure we have the latest status
updatedCluster := &cephv1.CephCluster{}
err = clusterContext.Client.Get(ctx, clusterInfo.NamespacedName(), updatedCluster)
assert.NoError(t, err)
assert.Equal(t, uint32(2), updatedCluster.Status.Cephx.CephExporter.KeyGeneration)
}