mon: rotate keys

rotate mon daemon keys and update the cephx status for mons in the
cephcluster resource.

Signed-off-by: Santosh Pillai <sapillai@redhat.com>
This commit is contained in:
Santosh Pillai committed 2025-08-14 11:16:39 +05:30
1 parent ea0203b86c
commit fe42fe102f
9 files changed
+267 -2

No files matched your search

+13
View File
@@ -5045,6 +5045,19 @@ CSI key rotation can affect existing PV connections, so take care when exercisin
<tbody>
<tr>
<td>
<code>mon</code><br/>
<em>
<a href="#ceph.rook.io/v1.CephxStatus">
CephxStatus
</a>
</em>
</td>
<td>
<p>Mon represents the CephX key status of the Monitor daemons</p>
</td>
</tr>
<tr>
<td>
<code>mgr</code><br/>
<em>
<a href="#ceph.rook.io/v1.CephxStatus">
@@ -6018,6 +6018,28 @@ spec:
format: int32
type: integer
type: object
mon:
description: Mon represents the CephX key status of the Monitor daemons
properties:
keyCephVersion:
description: |-
KeyCephVersion reports the Ceph version that created the current generation's keys. This is
same string format as reported by `CephCluster.status.version.version` to allow them to be
compared. E.g., `20.2.0-0`.
For all newly-created resources, this field set to the version of Ceph that created the key.
The special value "Uninitialized" indicates that keys are being created for the first time.
An empty string indicates that the version is unknown, as expected in brownfield deployments.
type: string
keyGeneration:
description: |-
KeyGeneration represents the CephX key generation for the last successful reconcile.
For all newly-created resources, this field is set to `1`.
When keys are rotated due to any rotation policy, the generation is incremented or updated to
the configured policy generation.
Generation `0` indicates that keys existed prior to the implementation of key tracking.
format: int32
type: integer
type: object
osd:
description: OSD shows the CephX key status of of OSDs
properties:
+22
View File
@@ -6016,6 +6016,28 @@ spec:
format: int32
type: integer
type: object
mon:
description: Mon represents the CephX key status of the Monitor daemons
properties:
keyCephVersion:
description: |-
KeyCephVersion reports the Ceph version that created the current generation's keys. This is
same string format as reported by `CephCluster.status.version.version` to allow them to be
compared. E.g., `20.2.0-0`.
For all newly-created resources, this field set to the version of Ceph that created the key.
The special value "Uninitialized" indicates that keys are being created for the first time.
An empty string indicates that the version is unknown, as expected in brownfield deployments.
type: string
keyGeneration:
description: |-
KeyGeneration represents the CephX key generation for the last successful reconcile.
For all newly-created resources, this field is set to `1`.
When keys are rotated due to any rotation policy, the generation is incremented or updated to
the configured policy generation.
Generation `0` indicates that keys existed prior to the implementation of key tracking.
format: int32
type: integer
type: object
osd:
description: OSD shows the CephX key status of of OSDs
properties:
+2
View File
@@ -742,6 +742,8 @@ type LocalCephxStatus struct {
// ClusterCephxStatus defines the cephx key rotation status of various daemons on the cephCluster resource
type ClusterCephxStatus struct {
// Mon represents the CephX key status of the Monitor daemons
Mon *CephxStatus `json:"mon,omitempty"`
// Mgr represents the cephx key rotation status of the ceph manager daemon
Mgr *CephxStatus `json:"mgr,omitempty"`
// OSD shows the CephX key status of of OSDs
@@ -2075,6 +2075,11 @@ func (in *ClusterCephxConfig) DeepCopy() *ClusterCephxConfig {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ClusterCephxStatus) DeepCopyInto(out *ClusterCephxStatus) {
*out = *in
if in.Mon != nil {
in, out := &in.Mon, &out.Mon
*out = new(CephxStatus)
**out = **in
}
if in.Mgr != nil {
in, out := &in.Mgr, &out.Mgr
*out = new(CephxStatus)
+22
View File
@@ -460,6 +460,27 @@ func (c *cluster) preMonStartupActions(cephVersion cephver.CephVersion) error {
// It gets executed right after the main mon Start() method
// Basically, it is executed between the monitors and the manager sequence
func (c *cluster) postMonStartupActions() error {
clusterObj := &cephv1.CephCluster{}
if err := c.context.Client.Get(c.ClusterInfo.Context, c.ClusterInfo.NamespacedName(), clusterObj); err != nil {
return errors.Wrapf(err, "failed to get cluster %v.", c.ClusterInfo.NamespacedName())
}
// rotate mon cephx keys if required
didRotateMonCephxKeys, err := c.mons.RotateMonCephxKeys(clusterObj)
if err != nil {
return errors.Wrapf(err, "failed to rotate mon cephx keys in the namespace %q", c.ClusterInfo.Namespace)
}
err = c.mons.UpdateMonCephxStatus(didRotateMonCephxKeys)
if err != nil {
return errors.Wrapf(err, "failed to update cephx status for mon daemons in the namespace %q", c.ClusterInfo.Namespace)
}
// reconcile to restart the mons after cephx key rotation
if didRotateMonCephxKeys {
// reconcile the rook operator so that it will restart the mons after mon cephx key rotation
return errors.New("triggering a new reconcile to restart the mon daemons after mon cephx key rotation")
}
// Create CSI Kubernetes Secrets
if err := csi.CreateCSISecrets(c.context, c.ClusterInfo, c.namespacedName); err != nil {
return errors.Wrap(err, "failed to create csi kubernetes secrets")
@@ -801,6 +822,7 @@ func initClusterCephxStatus(c *clusterd.Context, cluster *cephv1.CephCluster) er
OSD: &uninitializedStatus,
RBDMirrorPeer: &uninitializedStatus,
Mgr: &uninitializedStatus,
Mon: &uninitializedStatus,
CSI: &cephv1.CephxStatusWithKeyCount{
CephxStatus: uninitializedStatus,
},
+75 -1
View File
@@ -41,6 +41,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/ceph/csi"
"github.com/rook/rook/pkg/operator/ceph/reporting"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
apps "k8s.io/api/apps/v1"
@@ -51,6 +52,7 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
"k8s.io/utils/ptr"
cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes"
@@ -133,6 +135,8 @@ type Cluster struct {
arbiterMon string
// list of mons to be failed over
monsToFailover map[string]*monConfig
// reference to the secret that stores mon key
monKeySecretResourceVersion string
}
// monConfig for a single monitor
@@ -550,9 +554,12 @@ func (c *Cluster) initClusterInfo(cephVersion cephver.CephVersion, clusterName s
k := keyring.GetSecretStore(c.context, c.ClusterInfo, c.ownerInfo)
// store the keyring which all mons share
if _, err := k.CreateOrUpdate(keyringStoreName, c.genMonSharedKeyring()); err != nil {
secretResourceVersion, err := k.CreateOrUpdate(keyringStoreName, c.genMonSharedKeyring())
if err != nil {
return errors.Wrap(err, "failed to save mon keyring secret")
}
c.monKeySecretResourceVersion = secretResourceVersion
// also store the admin keyring for other daemons that might need it during init
if err := k.Admin().CreateOrUpdate(c.ClusterInfo, c.context, c.spec.Annotations); err != nil {
return errors.Wrap(err, "failed to save admin keyring secret")
@@ -1516,6 +1523,9 @@ func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) e
return err
}
// apply cephx secret resource version to the deployment to ensure it restarts after key is rotated
d.Spec.Template.Annotations[keyring.CephxKeyIdentifierAnnotation] = c.monKeySecretResourceVersion
// Set the deployment hash as an annotation
err = patch.DefaultAnnotator.SetLastAppliedAnnotation(d)
if err != nil {
@@ -1791,3 +1801,67 @@ func (c *Cluster) releaseOrchestrationLock() {
c.orchestrationMutex.Unlock()
logger.Debugf("Released lock for mon orchestration")
}
func (c *Cluster) RotateMonCephxKeys(clusterObj *cephv1.CephCluster) (bool, error) {
desiredCephVersion := c.ClusterInfo.CephVersion
// TODO: for rotation WithCephVersionUpdate fix this to have the right runningCephVersion and desiredCephVersion
runningCephVersion := c.ClusterInfo.CephVersion
shouldRotateMonKeys, err := keyring.ShouldRotateCephxKeys(clusterObj.Spec.Security.CephX.Daemon, runningCephVersion, desiredCephVersion, *clusterObj.Status.Cephx.Mon)
if err != nil {
return false, errors.Wrapf(err, "failed to check if mon daemon keys should be rotated in the namespace %q", c.ClusterInfo.Namespace)
}
if !shouldRotateMonKeys {
logger.Debugf("cephx key rotation for mon daemon in the namespace %q is not required", c.ClusterInfo.Namespace)
return shouldRotateMonKeys, nil
}
logger.Infof("cephx keys for mon daemons in the namespace %q will be rotated", c.ClusterInfo.Namespace)
k := keyring.GetSecretStore(c.context, c.ClusterInfo, c.ClusterInfo.OwnerInfo)
newKey, err := k.RotateKey(controller.MonCephxUser)
if err != nil {
return shouldRotateMonKeys, errors.Wrapf(err, "failed to rotate cephx key for mon daemon in the namespace %q", c.ClusterInfo.Namespace)
}
c.ClusterInfo.MonitorSecret = newKey
// update the mon-secret key in the cluster access secret
err = controller.UpdateClusterAccessSecret(c.context.Clientset, c.ClusterInfo)
if err != nil {
return shouldRotateMonKeys, errors.Wrapf(err, "failed to update the rook-ceph-mon secret after rotating the mon cephx keys in the namespace %q", c.ClusterInfo.Namespace)
}
// update the keyring which all mons share
if _, err := k.CreateOrUpdate(keyringStoreName, c.genMonSharedKeyring()); err != nil {
return shouldRotateMonKeys, errors.Wrapf(err, "failed to save mon keyring secret after rotating the mon cephx keys in the namespace %q", c.ClusterInfo.Namespace)
}
logger.Infof("successfully rotated cephx keys for mon daemons in the namespace %q", c.ClusterInfo.Namespace)
return shouldRotateMonKeys, nil
}
func (c *Cluster) UpdateMonCephxStatus(didRotate bool) error {
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
cluster := &cephv1.CephCluster{}
if err := c.context.Client.Get(c.ClusterInfo.Context, c.ClusterInfo.NamespacedName(), cluster); err != nil {
return errors.Wrapf(err, "failed to get cluster %v to update the mon cephx status.", c.ClusterInfo.NamespacedName())
}
updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, c.ClusterInfo.CephVersion, *cluster.Status.Cephx.Mon)
cluster.Status.Cephx.Mon = &updatedStatus
logger.Debugf("updating mon daemon cephx status to %+v", cluster.Status.Cephx.Mgr)
if err := reporting.UpdateStatus(c.context.Client, cluster); err != nil {
return errors.Wrapf(err, "failed to update cluster cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace)
}
logger.Infof("successfully updated the cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace)
return nil
})
if err != nil {
return errors.Wrapf(err, "failed to update cluster cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace)
}
return nil
}
+85
View File
@@ -30,10 +30,12 @@ import (
"github.com/coreos/pkg/capnslog"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
"github.com/rook/rook/pkg/clusterd"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test"
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
@@ -45,7 +47,9 @@ import (
discoveryv1 "k8s.io/api/discovery/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)
// generate a standard mon config from a mon id w/ default port and IP 2.4.6.{1,2,3,...}
@@ -1165,3 +1169,84 @@ func TestIsMonIPUpdateRequiredForHostNetwork(t *testing.T) {
assert.True(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork))
})
}
func TestRotateMonCephxKeys(t *testing.T) {
ctx := context.TODO()
namespace := "default"
context, err := newTestStartCluster(t, namespace)
assert.NoError(t, err)
c := newCluster(context, namespace, true, v1.ResourceRequirements{})
uninitializedStatus := keyring.UninitializedCephxStatus()
cluster := &cephv1.CephCluster{
ObjectMeta: metav1.ObjectMeta{
Name: namespace,
Namespace: namespace,
},
Spec: cephv1.ClusterSpec{
Security: cephv1.ClusterSecuritySpec{
CephX: cephv1.ClusterCephxConfig{
Daemon: cephv1.CephxConfig{},
},
},
},
Status: cephv1.ClusterStatus{
Cephx: &cephv1.ClusterCephxStatus{
Mon: &uninitializedStatus,
},
},
}
s := scheme.Scheme
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{})
s.AddKnownTypes(v1.SchemeGroupVersion, &v1.Secret{})
object := []runtime.Object{
cluster,
}
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
executor := &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
logger.Infof("%s %v", command, args)
if args[0] == "auth" && args[1] == "rotate" {
return `[{"key":"myrotatedkey"}]`, nil
}
return "", errors.New("unknown command")
},
}
c.context = &clusterd.Context{Clientset: test.New(t, 5), Executor: executor, Client: cl}
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
c.ClusterInfo.CephVersion = keyring.CephAuthRotateSupportedVersion
// create rook-ceph-mon secret
secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: AppName,
Namespace: namespace,
},
Data: map[string][]byte{opcontroller.MonSecretNameKey: []byte("bar")},
}
_, err = c.context.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
assert.NoError(t, err)
// No key rotation required
shouldRotate, err := c.RotateMonCephxKeys(cluster)
assert.NoError(t, err)
assert.False(t, shouldRotate)
// verify key is rotated in the secret
cluster.Spec.Security.CephX.Daemon.KeyRotationPolicy = cephv1.KeyGenerationCephxKeyRotationPolicy
cluster.Spec.Security.CephX.Daemon.KeyGeneration = 2
cluster.Status.Cephx.Mon.KeyGeneration = 1
cluster.Status.Cephx.Mon.KeyCephVersion = keyring.CephAuthRotateSupportedVersion.String()
err = c.context.Client.Update(ctx, cluster)
assert.NoError(t, err)
shouldRotate, err = c.RotateMonCephxKeys(cluster)
assert.NoError(t, err)
assert.True(t, shouldRotate)
secret, err = c.context.Clientset.CoreV1().Secrets(namespace).Get(c.ClusterInfo.Context, AppName, metav1.GetOptions{})
assert.NoError(t, err)
assert.Equal(t, "myrotatedkey", string(secret.Data[opcontroller.MonSecretNameKey]))
}
+21 -1
View File
@@ -68,6 +68,8 @@ const (
// AppName is the name of the secret storing cluster mon.admin key, fsid and name
AppName = "rook-ceph-mon"
DisasterProtectionFinalizerName = cephv1.CustomResourceGroup + "/disaster-protection"
// MonCephxUser represents the cephx user name for all the mon daemons
MonCephxUser = "mon."
)
var (
@@ -202,7 +204,7 @@ func createNamedClusterInfo(context *clusterd.Context, namespace string) (*cephc
}
// generate the mon secret
monSecret, err := genSecret(context.Executor, dir, "mon.", []string{"--cap", "mon", "'allow *'"})
monSecret, err := genSecret(context.Executor, dir, MonCephxUser, []string{"--cap", "mon", "'allow *'"})
if err != nil {
return nil, err
}
@@ -397,6 +399,24 @@ func createClusterAccessSecret(clientset kubernetes.Interface, namespace string,
return nil
}
// UpdateClusterAccessSecret updates the rook-ceph-mon secret
func UpdateClusterAccessSecret(clientset kubernetes.Interface, clusterInfo *cephclient.ClusterInfo) error {
secret, err := clientset.CoreV1().Secrets(clusterInfo.Namespace).Get(clusterInfo.Context, AppName, metav1.GetOptions{})
if err != nil {
return errors.Wrapf(err, "failed to get secret %q in the namespace %q", AppName, clusterInfo.Namespace)
}
secret.Data[MonSecretNameKey] = []byte(clusterInfo.MonitorSecret)
logger.Infof("updating secret %q in the namespace %q", secret.Name, clusterInfo.Namespace)
_, err = clientset.CoreV1().Secrets(clusterInfo.Namespace).Update(clusterInfo.Context, secret, metav1.UpdateOptions{})
if err != nil {
return errors.Wrapf(err, "failed to update secret %q in the namespace %q", secret.Name, clusterInfo.Namespace)
}
return nil
}
// ParseMonEndpoints parses a flattened representation of mons and endpoints in the form
// <mon-name>=<mon-endpoint> and returns a list of Ceph mon configs.
func ParseMonEndpoints(input string) map[string]*cephclient.MonInfo {