feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

Merge pull request #16354 from rook/mergify/bp/release-1.18/pr-16346

Browse Source 
CephX key rotation 1.18 cleanup (backport #16346)
This commit is contained in:
Blaine Gardner authored and GitHub committed 2025-08-19 15:35:04 -06:00
2 parents d9039a009c 278741fa40
commit d237730ee8
22 files changed
+469 -139
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Documentation/CRDs/specification.md
+13 -13
View File
@@ -5140,19 +5140,6 @@ CephxStatus
</tr>
<tr>
<td>
<code>rbdMirrorPeer</code><br/>
<em>
<a href="#ceph.rook.io/v1.CephxStatus">
CephxStatus
</a>
</em>
</td>
<td>
<p>RBDMirrorPeer represents the cephx key rotation status of the <code>rbd-mirror-peer</code> user</p>
</td>
</tr>
<tr>
<td>
<code>csi</code><br/>
<em>
<a href="#ceph.rook.io/v1.CephxStatusWithKeyCount">
@@ -5166,6 +5153,19 @@ CephxStatusWithKeyCount
</tr>
<tr>
<td>
<code>rbdMirrorPeer</code><br/>
<em>
<a href="#ceph.rook.io/v1.CephxStatus">
CephxStatus
</a>
</em>
</td>
<td>
<p>RBDMirrorPeer represents the cephx key rotation status of the <code>rbd-mirror-peer</code> user</p>
</td>
</tr>
<tr>
<td>
<code>crashCollector</code><br/>
<em>
<a href="#ceph.rook.io/v1.CephxStatus">
Documentation/Storage-Configuration/Advanced/.pages
+1
View File
@@ -4,4 +4,5 @@ nav:
- key-management-system.md
- ceph-osd-mgmt.md
- ceph-mon-health.md
- cephx-key-rotation.md
- ...
Documentation/Storage-Configuration/Advanced/cephx-key-rotation.md
+140
View File
@@ -0,0 +1,140 @@
---
title: CephX Key Rotation
---
!!! attention
This feature is experimental.
Rook is able to rotate [CephX authentication keys](https://docs.ceph.com/en/latest/dev/cephx/) used
by Ceph daemons and clients.
For this experimental feature, some caveats should be noted:
- Only Ceph versions v19.2.3 and higher have the capabilities Rook requires for key rotation.
- Ceph Monitor (mon) keys cannot be rotated in Ceph v19 due to Ceph architecture limitations.
## Overview
CephX keys can be rotated when desired on a one-off basis. To provide this capability, Rook utilizes
an approximation of Kubernetes's resource generation. A one-time key rotation is initiated by
specifying `KeyGeneration` as the desired policy (the default policy is `Disabled`) and also
specify a key generation higher than the current generation.
CephX keys can be divided into two categories, below.
### Daemon keys
Daemon keys are used internally within a Ceph cluster, and their rotation does not affect CSI
volumes or connections to a Ceph cluster from outside.
Daemon key rotation is configured via the CephCluster `spec.security.cephx.daemon` config. This will
also rotate daemon keys for any CephFilesystem MDSes and CephObjectStore RGWs.
Rotation requires most Ceph daemons to restart, so this operation is best done at the same time the
CephCluster `spec.cephVersion.image` is updated -- when daemons will normally need to restart.
### "Non-daemon" keys
Non-daemon keys may reasonably require user action beyond Rook API controls.
Because these keys affect non-daemon connections, Rook allows users to initiate rotation
independently during their desired maintenance window.
Below is a list of non-daemon keys along with the controlling config.
- CephCluster CSI keys are controlled via CephCluster `spec.security.cephx.csi`
- Rotated CSI keys only take effect for new PVCs. For CSI alone, Rook is able to create new keys
while also keeping a number of prior key generations active. This is configured using the
`keepPriorKeyCountMax` option.
- The CephCluster RBD mirror peer key is controlled via CephCluster `spec.security.cephx.rbdMirrorPeer`
- Each CephBlockPool that has mirroring configured will have a `peerToken` status that
references the CephCluster RBD mirror peer key
- Each CephClient key is controlled via its own `spec.security.cephx`
## Initiating key rotation
To begin experimenting with key rotation, check out
[CephX config](https://rook.io/docs/rook/latest/CRDs/specification/?h=cephx#ceph.rook.io/v1.CephxConfig)
options on Rook CRs.
### Example
Most key rotations are initiated from the CephCluster. An example spec that will rotate all CephX
keys for most new or upgraded Rook clusters is shown below.
```yaml
apiVersion: ceph.rook.io/v1
kind: CephCluster
metadata:
name: my-cluster
namespace: rook-ceph # namespace:cluster
spec:
cephVersion:
image: quay.io/ceph/ceph:v19.2.3
security:
cephx:
daemon:
keyRotationPolicy: KeyGeneration
keyGeneration: 2
csi:
keyRotationPolicy: KeyGeneration
keyGeneration: 2
keepPriorKeyCountMax: 1 # keep one prior key also
rbdMirrorPeer:
keyRotationPolicy: KeyGeneration
keyGeneration: 2
# ...
```
Once rotation is complete, CephCluster status should look something like below. Each CephX key
type managed for the cluster is listed.
```yaml
status:
# ...
cephx:
cephExporter:
keyCephVersion: 19.2.3-0
keyGeneration: 2
crashCollector:
keyCephVersion: 19.2.3-0
keyGeneration: 2
csi:
keyCephVersion: 19.2.3-0
keyGeneration: 2
priorKeyCount: 1
mgr:
keyCephVersion: 19.2.3-0
keyGeneration: 2
mon: {} # reminder: mon key rotation is unsupported currently
osd:
keyCephVersion: 19.2.3-0
keyGeneration: 2
rbdMirrorPeer:
keyCephVersion: 19.2.3-0
keyGeneration: 2
```
Additionally, any CephFilesystem or CephObjectStore will show the status of rotation for their
daemons:
```yaml
status:
# ...
cephx:
daemon:
keyCephVersion: 19.2.3-0
keyGeneration: 2
```
If mirroring is enabled on a CephBlockPool, the following status will mirror the CephCluster's
`rbdMirrorPeer` status:
```yaml
status:
# ...
cephx:
peerToken:
keyCephVersion: 19.2.3-0
keyGeneration: 2
```
pkg/apis/ceph.rook.io/v1/types.go
+16 -16
View File
@@ -516,14 +516,14 @@ type CephExporterSpec struct {
// ClusterStatus represents the status of a Ceph cluster
type ClusterStatus struct {
State ClusterState `json:"state,omitempty"`
Phase ConditionType `json:"phase,omitempty"`
Message string `json:"message,omitempty"`
Conditions []Condition `json:"conditions,omitempty"`
CephStatus *CephStatus `json:"ceph,omitempty"`
Cephx *ClusterCephxStatus `json:"cephx,omitempty"`
CephStorage *CephStorage `json:"storage,omitempty"`
CephVersion *ClusterVersion `json:"version,omitempty"`
State ClusterState `json:"state,omitempty"`
Phase ConditionType `json:"phase,omitempty"`
Message string `json:"message,omitempty"`
Conditions []Condition `json:"conditions,omitempty"`
CephStatus *CephStatus `json:"ceph,omitempty"`
Cephx ClusterCephxStatus `json:"cephx,omitempty"`
CephStorage *CephStorage `json:"storage,omitempty"`
CephVersion *ClusterVersion `json:"version,omitempty"`
// ObservedGeneration is the latest generation observed by the controller.
// +optional
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
@@ -749,19 +749,19 @@ type LocalCephxStatus struct {
// ClusterCephxStatus defines the cephx key rotation status of various daemons on the cephCluster resource
type ClusterCephxStatus struct {
// Mon represents the CephX key status of the Monitor daemons
Mon *CephxStatus `json:"mon,omitempty"`
Mon CephxStatus `json:"mon,omitempty"`
// Mgr represents the cephx key rotation status of the ceph manager daemon
Mgr *CephxStatus `json:"mgr,omitempty"`
Mgr CephxStatus `json:"mgr,omitempty"`
// OSD shows the CephX key status of of OSDs
OSD *CephxStatus `json:"osd,omitempty"`
// RBDMirrorPeer represents the cephx key rotation status of the `rbd-mirror-peer` user
RBDMirrorPeer *CephxStatus `json:"rbdMirrorPeer,omitempty"`
OSD CephxStatus `json:"osd,omitempty"`
// CSI shows the CephX key status for Ceph-CSI components.
CSI *CephxStatusWithKeyCount `json:"csi,omitempty"`
CSI CephxStatusWithKeyCount `json:"csi,omitempty"`
// RBDMirrorPeer represents the cephx key rotation status of the `rbd-mirror-peer` user
RBDMirrorPeer CephxStatus `json:"rbdMirrorPeer,omitempty"`
// Crash Collector represents the cephx key rotation status of the crash collector daemon
CrashCollector *CephxStatus `json:"crashCollector,omitempty"`
CrashCollector CephxStatus `json:"crashCollector,omitempty"`
// Ceph Exporter represents the cephx key rotation status of the ceph exporter daemon
CephExporter *CephxStatus `json:"cephExporter,omitempty"`
CephExporter CephxStatus `json:"cephExporter,omitempty"`
}
// MonSpec represents the specification of the monitor
pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go
+8 -40
View File
@@ -2077,41 +2077,13 @@ func (in *ClusterCephxConfig) DeepCopy() *ClusterCephxConfig {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ClusterCephxStatus) DeepCopyInto(out *ClusterCephxStatus) {
*out = *in
if in.Mon != nil {
in, out := &in.Mon, &out.Mon
*out = new(CephxStatus)
**out = **in
}
if in.Mgr != nil {
in, out := &in.Mgr, &out.Mgr
*out = new(CephxStatus)
**out = **in
}
if in.OSD != nil {
in, out := &in.OSD, &out.OSD
*out = new(CephxStatus)
**out = **in
}
if in.RBDMirrorPeer != nil {
in, out := &in.RBDMirrorPeer, &out.RBDMirrorPeer
*out = new(CephxStatus)
**out = **in
}
if in.CSI != nil {
in, out := &in.CSI, &out.CSI
*out = new(CephxStatusWithKeyCount)
**out = **in
}
if in.CrashCollector != nil {
in, out := &in.CrashCollector, &out.CrashCollector
*out = new(CephxStatus)
**out = **in
}
if in.CephExporter != nil {
in, out := &in.CephExporter, &out.CephExporter
*out = new(CephxStatus)
**out = **in
}
out.Mon = in.Mon
out.Mgr = in.Mgr
out.OSD = in.OSD
out.CSI = in.CSI
out.RBDMirrorPeer = in.RBDMirrorPeer
out.CrashCollector = in.CrashCollector
out.CephExporter = in.CephExporter
return
}
@@ -2279,11 +2251,7 @@ func (in *ClusterStatus) DeepCopyInto(out *ClusterStatus) {
*out = new(CephStatus)
(*in).DeepCopyInto(*out)
}
if in.Cephx != nil {
in, out := &in.Cephx, &out.Cephx
*out = new(ClusterCephxStatus)
(*in).DeepCopyInto(*out)
}
out.Cephx = in.Cephx
if in.CephStorage != nil {
in, out := &in.CephStorage, &out.CephStorage
*out = new(CephStorage)
pkg/operator/ceph/cluster/cluster.go
+51 -17
View File
@@ -50,6 +50,7 @@ import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/retry"
)
const (
@@ -453,6 +454,11 @@ func (c *cluster) replaceDefaultCrushMap(newRoot string) (err error) {
// preMonStartupActions is a collection of actions to run before the monitors are reconciled.
func (c *cluster) preMonStartupActions(cephVersion cephver.CephVersion) error {
err := initClusterCephxStatus(c)
if err != nil {
return errors.Wrap(err, "failed to initialized cluster cephx status")
}
return nil
}
@@ -815,24 +821,52 @@ func (c *cluster) fetchSecretValue(selector v1.SecretKeySelector) (string, error
return string(val), nil
}
// initClusterCephxStatus set `Uninitialized` state for the cephXstatus for new clusters.
func initClusterCephxStatus(c *clusterd.Context, cluster *cephv1.CephCluster) error {
uninitializedStatus := keyring.UninitializedCephxStatus()
cluster.Status.Cephx = &cephv1.ClusterCephxStatus{
OSD: &uninitializedStatus,
RBDMirrorPeer: &uninitializedStatus,
Mgr: &uninitializedStatus,
Mon: &uninitializedStatus,
CSI: &cephv1.CephxStatusWithKeyCount{
CephxStatus: uninitializedStatus,
},
CrashCollector: &uninitializedStatus,
CephExporter: &uninitializedStatus,
// initClusterCephxStatus set `Uninitialized` cephx status for new clusters.
// this should not be run for external mode clusters
func initClusterCephxStatus(c *cluster) error {
initErr := c.ClusterInfo.IsInitialized()
if initErr == nil {
logger.Debugf("not setting uninitialized cephx status on already initialized CephCluster in namespace %q", c.Namespace)
return nil
}
if c.ClusterInfo.Context.Err() != nil {
// most IsInitialized() errors mean the cluster is new, but if clusterInfo.Context is
// nil, it is a 'real' error to return
return c.ClusterInfo.Context.Err()
}
if err := reporting.UpdateStatus(c.Client, cluster); err != nil {
return errors.Wrapf(err, "failed to update cluster cephx status")
}
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
clusterObj := &cephv1.CephCluster{}
err := c.context.Client.Get(c.ClusterInfo.Context, c.namespacedName, clusterObj)
if err != nil {
return errors.Wrapf(err, "failed to get cluster in order to initialize its cephx status")
}
return nil
emptyStatus := cephv1.CephxStatus{}
// mon cephx status is one of the first set after mons are successfully running, so we only need to check it
if clusterObj.Status.Cephx.Mon != emptyStatus {
return nil // do not initialize multiple times
}
uninitializedStatus := keyring.UninitializedCephxStatus()
logger.Infof("initializing cephx status for CephCluster in namespace %q", c.Namespace)
clusterObj.Status.Cephx = cephv1.ClusterCephxStatus{
Mon: uninitializedStatus,
Mgr: uninitializedStatus,
// OSD statuses are determined entirely within OSD reconcile - don't set uninitialized here
CSI: cephv1.CephxStatusWithKeyCount{
CephxStatus: uninitializedStatus,
},
RBDMirrorPeer: uninitializedStatus,
CrashCollector: uninitializedStatus,
CephExporter: uninitializedStatus,
}
if err := reporting.UpdateStatus(c.context.Client, clusterObj); err != nil {
return errors.Wrapf(err, "failed to initialize cluster cephx status")
}
return nil
})
return err
}
pkg/operator/ceph/cluster/cluster_test.go
+174
View File
@@ -27,11 +27,13 @@ import (
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
"github.com/rook/rook/pkg/clusterd"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
"github.com/rook/rook/pkg/operator/ceph/cluster/nodedaemon"
"github.com/rook/rook/pkg/operator/ceph/cluster/telemetry"
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
"github.com/rook/rook/pkg/operator/ceph/csi"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
testop "github.com/rook/rook/pkg/operator/test"
@@ -41,6 +43,8 @@ import (
"gopkg.in/ini.v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)
func TestPreClusterStartValidation(t *testing.T) {
@@ -594,3 +598,173 @@ func TestFetchCephConfigFromSecrets(t *testing.T) {
})
}
}
func Test_initClusterCephxStatus(t *testing.T) {
ns := "ns"
clusterName := "testcluster"
newCluster := func(cephxStatus cephv1.CephxStatus, clusterInfo *cephclient.ClusterInfo) *cluster {
cephCluster := &cephv1.CephCluster{
ObjectMeta: metav1.ObjectMeta{
Name: clusterName,
Namespace: ns,
},
Status: cephv1.ClusterStatus{
Cephx: cephv1.ClusterCephxStatus{
Mon: cephxStatus,
// don't set other statuses for new clusters so we can tell if obj was updated
},
},
}
s := scheme.Scheme
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephClusterList{})
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(cephCluster).Build()
return &cluster{
ClusterInfo: clusterInfo,
Namespace: ns,
namespacedName: types.NamespacedName{Namespace: ns, Name: clusterName},
context: &clusterd.Context{
Client: cl,
},
}
}
readyClusterInfo := func() *cephclient.ClusterInfo {
return &cephclient.ClusterInfo{
FSID: "1234567890",
MonitorSecret: "AAAAAAAAAAAAA==",
CephCred: cephclient.CephCred{
Username: "client.admin",
Secret: "BBBBBBBBBBBBB==",
},
Context: context.TODO(),
}
}
notreadyClusterInfo := func() *cephclient.ClusterInfo {
return &cephclient.ClusterInfo{
Context: context.TODO(),
}
}
t.Run("cluster ready, empty status", func(t *testing.T) {
c := newCluster(cephv1.CephxStatus{}, readyClusterInfo())
err := initClusterCephxStatus(c)
assert.NoError(t, err)
cluster := cephv1.CephCluster{}
err = c.context.Client.Get(context.TODO(), c.namespacedName, &cluster)
assert.NoError(t, err)
assert.Equal(t, cephv1.ClusterCephxStatus{}, cluster.Status.Cephx)
})
t.Run("cluster ready, status already uninit", func(t *testing.T) {
c := newCluster(keyring.UninitializedCephxStatus(), readyClusterInfo())
err := initClusterCephxStatus(c)
assert.NoError(t, err)
cluster := cephv1.CephCluster{}
err = c.context.Client.Get(context.TODO(), c.namespacedName, &cluster)
assert.NoError(t, err)
assert.Equal(t, keyring.UninitializedCephxStatus(), cluster.Status.Cephx.Mon)
assert.Equal(t, cephv1.CephxStatus{}, cluster.Status.Cephx.Mgr) // no status means no update
})
t.Run("cluster ready, status nonzero", func(t *testing.T) {
monStatus := cephv1.CephxStatus{KeyGeneration: 3, KeyCephVersion: "19.2.3-0"}
c := newCluster(monStatus, readyClusterInfo())
err := initClusterCephxStatus(c)
assert.NoError(t, err)
cluster := cephv1.CephCluster{}
err = c.context.Client.Get(context.TODO(), c.namespacedName, &cluster)
assert.NoError(t, err)
assert.Equal(t, monStatus, cluster.Status.Cephx.Mon)
assert.Equal(t, cephv1.CephxStatus{}, cluster.Status.Cephx.Mgr) // no status means no update
})
t.Run("cluster not ready, empty status", func(t *testing.T) {
c := newCluster(cephv1.CephxStatus{}, notreadyClusterInfo())
err := initClusterCephxStatus(c)
assert.NoError(t, err)
cluster := cephv1.CephCluster{}
err = c.context.Client.Get(context.TODO(), c.namespacedName, &cluster)
assert.NoError(t, err)
assert.Equal(t, keyring.UninitializedCephxStatus(), cluster.Status.Cephx.Mon)
assert.Equal(t, keyring.UninitializedCephxStatus(), cluster.Status.Cephx.Mgr)
assert.Equal(t, keyring.UninitializedCephxStatus(), cluster.Status.Cephx.CSI.CephxStatus)
assert.Equal(t, keyring.UninitializedCephxStatus(), cluster.Status.Cephx.RBDMirrorPeer)
assert.Equal(t, keyring.UninitializedCephxStatus(), cluster.Status.Cephx.CrashCollector)
assert.Equal(t, keyring.UninitializedCephxStatus(), cluster.Status.Cephx.CephExporter)
})
t.Run("cluster not ready, status already uninit", func(t *testing.T) {
c := newCluster(keyring.UninitializedCephxStatus(), notreadyClusterInfo())
err := initClusterCephxStatus(c)
assert.NoError(t, err)
cluster := cephv1.CephCluster{}
err = c.context.Client.Get(context.TODO(), c.namespacedName, &cluster)
assert.NoError(t, err)
assert.Equal(t, keyring.UninitializedCephxStatus(), cluster.Status.Cephx.Mon)
assert.Equal(t, cephv1.CephxStatus{}, cluster.Status.Cephx.Mgr) // no status means no update
})
t.Run("cluster not ready, status nonzero", func(t *testing.T) {
monStatus := cephv1.CephxStatus{KeyGeneration: 3, KeyCephVersion: "19.2.3-0"}
c := newCluster(monStatus, notreadyClusterInfo())
err := initClusterCephxStatus(c)
assert.NoError(t, err)
cluster := cephv1.CephCluster{}
err = c.context.Client.Get(context.TODO(), c.namespacedName, &cluster)
assert.NoError(t, err)
assert.Equal(t, monStatus, cluster.Status.Cephx.Mon)
assert.Equal(t, cephv1.CephxStatus{}, cluster.Status.Cephx.Mgr) // no status means no update
})
t.Run("cluster ctx canceled, status empty", func(t *testing.T) {
notReady := notreadyClusterInfo()
newCtx, cancelFunc := context.WithCancel(notReady.Context)
notReady.Context = newCtx
cancelFunc()
c := newCluster(cephv1.CephxStatus{}, notReady) // would normally need initialized
err := initClusterCephxStatus(c)
assert.Error(t, err)
cluster := cephv1.CephCluster{}
err = c.context.Client.Get(context.TODO(), c.namespacedName, &cluster)
assert.NoError(t, err)
assert.Equal(t, cephv1.CephxStatus{}, cluster.Status.Cephx.Mon)
assert.Equal(t, cephv1.CephxStatus{}, cluster.Status.Cephx.Mgr) // no status means no update
})
t.Run("cluster ctx canceled, status nonzero", func(t *testing.T) {
notReady := notreadyClusterInfo()
newCtx, cancelFunc := context.WithCancel(notReady.Context)
notReady.Context = newCtx
cancelFunc()
monStatus := cephv1.CephxStatus{KeyGeneration: 3, KeyCephVersion: "19.2.3-0"}
c := newCluster(monStatus, notReady)
err := initClusterCephxStatus(c)
assert.Error(t, err)
cluster := cephv1.CephCluster{}
err = c.context.Client.Get(context.TODO(), c.namespacedName, &cluster)
assert.NoError(t, err)
assert.Equal(t, monStatus, cluster.Status.Cephx.Mon)
assert.Equal(t, cephv1.CephxStatus{}, cluster.Status.Cephx.Mgr) // no status means no update
})
}
pkg/operator/ceph/cluster/controller.go
-7
View File
@@ -385,13 +385,6 @@ func (c *ClusterController) reconcileCephCluster(clusterObj *cephv1.CephCluster,
return nil
}
if clusterObj.Status.Cephx == nil {
err := initClusterCephxStatus(c.context, clusterObj)
if err != nil {
return errors.Wrap(err, "failed to initialized cluster cephx status")
}
}
cluster, ok := c.clusterMap[clusterObj.Namespace]
if !ok {
// It's a new cluster so let's populate the struct
pkg/operator/ceph/cluster/mgr/mgr.go
+3 -3
View File
@@ -622,7 +622,7 @@ func shouldRotateMgrKeys(c *clusterd.Context, clusterInfo *cephclient.ClusterInf
// TODO: for rotation WithCephVersionUpdate fix this to have the right runningCephVersion and desiredCephVersion
runningCephVersion := clusterInfo.CephVersion
shouldRotateKeys, err := keyring.ShouldRotateCephxKeys(clusterObj.Spec.Security.CephX.Daemon, runningCephVersion, desiredCephVersion, *clusterObj.Status.Cephx.Mgr)
shouldRotateKeys, err := keyring.ShouldRotateCephxKeys(clusterObj.Spec.Security.CephX.Daemon, runningCephVersion, desiredCephVersion, clusterObj.Status.Cephx.Mgr)
if err != nil {
return false, errors.Wrap(err, "failed to check if mgr daemon keys should be rotated or not")
}
@@ -636,8 +636,8 @@ func updateMgrCephxStatus(c *clusterd.Context, clusterInfo *cephclient.ClusterIn
if err := c.Client.Get(clusterInfo.Context, clusterInfo.NamespacedName(), cluster); err != nil {
return errors.Wrapf(err, "failed to get cluster %v to update the conditions.", clusterInfo.NamespacedName())
}
updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, clusterInfo.CephVersion, *cluster.Status.Cephx.Mgr)
cluster.Status.Cephx.Mgr = &updatedStatus
updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, clusterInfo.CephVersion, cluster.Status.Cephx.Mgr)
cluster.Status.Cephx.Mgr = updatedStatus
logger.Debugf("updating mgr daemon cephx status to %+v", cluster.Status.Cephx.Mgr)
if err := reporting.UpdateStatus(c.Client, cluster); err != nil {
return errors.Wrap(err, "failed to update cluster cephx status for mgr daemon")
pkg/operator/ceph/cluster/mgr/mgr_test.go
+2 -2
View File
@@ -92,8 +92,8 @@ func createNewCluster(t *testing.T) *Cluster {
},
},
Status: cephv1.ClusterStatus{
Cephx: &cephv1.ClusterCephxStatus{
Mgr: &uninitializedStatus,
Cephx: cephv1.ClusterCephxStatus{
Mgr: uninitializedStatus,
},
},
}
pkg/operator/ceph/cluster/mon/health_test.go
+6 -6
View File
@@ -313,8 +313,8 @@ func TestEvictMonOnSameNode(t *testing.T) {
},
},
Status: cephv1.ClusterStatus{
Cephx: &cephv1.ClusterCephxStatus{
RBDMirrorPeer: &cephv1.CephxStatus{},
Cephx: cephv1.ClusterCephxStatus{
RBDMirrorPeer: cephv1.CephxStatus{},
},
},
}
@@ -865,8 +865,8 @@ func TestExternalMons_notInSpec_InQuorum(t *testing.T) {
},
},
Status: cephv1.ClusterStatus{
Cephx: &cephv1.ClusterCephxStatus{
RBDMirrorPeer: &cephv1.CephxStatus{},
Cephx: cephv1.ClusterCephxStatus{
RBDMirrorPeer: cephv1.CephxStatus{},
},
},
}
@@ -1319,8 +1319,8 @@ func getCephCluster(name, namespace string) *cephv1.CephCluster {
},
},
Status: cephv1.ClusterStatus{
Cephx: &cephv1.ClusterCephxStatus{
RBDMirrorPeer: &cephv1.CephxStatus{},
Cephx: cephv1.ClusterCephxStatus{
RBDMirrorPeer: cephv1.CephxStatus{},
},
},
}
pkg/operator/ceph/cluster/mon/mon.go
+8 -3
View File
@@ -1807,7 +1807,7 @@ func (c *Cluster) RotateMonCephxKeys(clusterObj *cephv1.CephCluster) (bool, erro
// TODO: for rotation WithCephVersionUpdate fix this to have the right runningCephVersion and desiredCephVersion
runningCephVersion := c.ClusterInfo.CephVersion
shouldRotateMonKeys, err := keyring.ShouldRotateCephxKeys(clusterObj.Spec.Security.CephX.Daemon, runningCephVersion, desiredCephVersion, *clusterObj.Status.Cephx.Mon)
shouldRotateMonKeys, err := keyring.ShouldRotateCephxKeys(clusterObj.Spec.Security.CephX.Daemon, runningCephVersion, desiredCephVersion, clusterObj.Status.Cephx.Mon)
if err != nil {
return false, errors.Wrapf(err, "failed to check if mon daemon keys should be rotated in the namespace %q", c.ClusterInfo.Namespace)
}
@@ -1817,6 +1817,11 @@ func (c *Cluster) RotateMonCephxKeys(clusterObj *cephv1.CephCluster) (bool, erro
return shouldRotateMonKeys, nil
}
if !c.ClusterInfo.CephVersion.IsAtLeast(keyring.CephAuthMonRotateSupportedVersion) {
logger.Debugf("cephx key rotation for mons in namespace %q is indicated, but ceph version %#v does not support mon key rotation", c.Namespace, c.ClusterInfo.CephVersion)
return false, nil
}
logger.Infof("cephx keys for mon daemons in the namespace %q will be rotated", c.ClusterInfo.Namespace)
k := keyring.GetSecretStore(c.context, c.ClusterInfo, c.ClusterInfo.OwnerInfo)
@@ -1849,8 +1854,8 @@ func (c *Cluster) UpdateMonCephxStatus(didRotate bool) error {
if err := c.context.Client.Get(c.ClusterInfo.Context, c.ClusterInfo.NamespacedName(), cluster); err != nil {
return errors.Wrapf(err, "failed to get cluster %v to update the mon cephx status.", c.ClusterInfo.NamespacedName())
}
updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, c.ClusterInfo.CephVersion, *cluster.Status.Cephx.Mon)
cluster.Status.Cephx.Mon = &updatedStatus
updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, c.ClusterInfo.CephVersion, cluster.Status.Cephx.Mon)
cluster.Status.Cephx.Mon = updatedStatus
logger.Debugf("updating mon daemon cephx status to %+v", cluster.Status.Cephx.Mgr)
if err := reporting.UpdateStatus(c.context.Client, cluster); err != nil {
return errors.Wrapf(err, "failed to update cluster cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace)
pkg/operator/ceph/cluster/mon/mon_test.go
+16 -4
View File
@@ -1191,8 +1191,8 @@ func TestRotateMonCephxKeys(t *testing.T) {
},
},
Status: cephv1.ClusterStatus{
Cephx: &cephv1.ClusterCephxStatus{
Mon: &uninitializedStatus,
Cephx: cephv1.ClusterCephxStatus{
Mon: uninitializedStatus,
},
},
}
@@ -1217,7 +1217,7 @@ func TestRotateMonCephxKeys(t *testing.T) {
}
c.context = &clusterd.Context{Clientset: test.New(t, 5), Executor: executor, Client: cl}
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
c.ClusterInfo.CephVersion = keyring.CephAuthRotateSupportedVersion
c.ClusterInfo.CephVersion = keyring.CephAuthMonRotateSupportedVersion
// create rook-ceph-mon secret
secret := &v1.Secret{
@@ -1240,7 +1240,7 @@ func TestRotateMonCephxKeys(t *testing.T) {
cluster.Spec.Security.CephX.Daemon.KeyRotationPolicy = cephv1.KeyGenerationCephxKeyRotationPolicy
cluster.Spec.Security.CephX.Daemon.KeyGeneration = 2
cluster.Status.Cephx.Mon.KeyGeneration = 1
cluster.Status.Cephx.Mon.KeyCephVersion = keyring.CephAuthRotateSupportedVersion.String()
cluster.Status.Cephx.Mon.KeyCephVersion = "19.2.3-0"
err = c.context.Client.Update(ctx, cluster)
assert.NoError(t, err)
shouldRotate, err = c.RotateMonCephxKeys(cluster)
@@ -1249,4 +1249,16 @@ func TestRotateMonCephxKeys(t *testing.T) {
secret, err = c.context.Clientset.CoreV1().Secrets(namespace).Get(c.ClusterInfo.Context, AppName, metav1.GetOptions{})
assert.NoError(t, err)
assert.Equal(t, "myrotatedkey", string(secret.Data[opcontroller.MonSecretNameKey]))
// verify mon key does not rotate when other daemons rotate
cluster.Spec.Security.CephX.Daemon.KeyRotationPolicy = cephv1.KeyGenerationCephxKeyRotationPolicy
cluster.Spec.Security.CephX.Daemon.KeyGeneration = 2
cluster.Status.Cephx.Mon.KeyGeneration = 1
cluster.Status.Cephx.Mon.KeyCephVersion = "19.2.3-0"
err = c.context.Client.Update(ctx, cluster)
assert.NoError(t, err)
c.ClusterInfo.CephVersion = keyring.CephAuthRotateSupportedVersion // but not CephAuthMonRotateSupportedVersion
shouldRotate, err = c.RotateMonCephxKeys(cluster)
assert.NoError(t, err)
assert.False(t, shouldRotate)
}
pkg/operator/ceph/cluster/nodedaemon/keyring.go
+6 -6
View File
@@ -89,7 +89,7 @@ func createCrashCollectorKeyring(s *keyring.SecretStore, context *clusterd.Conte
shouldRotateCephxKeys, err := keyring.ShouldRotateCephxKeys(
clusterObj.Spec.Security.CephX.Daemon,
clusterInfo.CephVersion, clusterInfo.CephVersion,
*clusterObj.Status.Cephx.CrashCollector,
clusterObj.Status.Cephx.CrashCollector,
)
if err != nil {
return "", errors.Wrapf(err, "failed to check if cephx keys should be rotated for crash collector %q", crashCollectorKeyringUsername)
@@ -119,8 +119,8 @@ func updateCrashCollectorCephxStatus(context *clusterd.Context, clusterInfo *cli
if err := context.Client.Get(clusterInfo.Context, clusterInfo.NamespacedName(), cluster); err != nil {
return errors.Wrapf(err, "failed to get cluster %v to update the conditions.", clusterInfo.NamespacedName())
}
updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, clusterInfo.CephVersion, *cluster.Status.Cephx.CrashCollector)
cluster.Status.Cephx.CrashCollector = &updatedStatus
updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, clusterInfo.CephVersion, cluster.Status.Cephx.CrashCollector)
cluster.Status.Cephx.CrashCollector = updatedStatus
if err := reporting.UpdateStatus(context.Client, cluster); err != nil {
return errors.Wrap(err, "failed to update cluster cephx status for crash collector daemon")
}
@@ -205,7 +205,7 @@ func createExporterKeyring(s *keyring.SecretStore, context *clusterd.Context, cl
shouldRotateCephxKeys, err := keyring.ShouldRotateCephxKeys(
clusterObj.Spec.Security.CephX.Daemon,
clusterInfo.CephVersion, clusterInfo.CephVersion,
*clusterObj.Status.Cephx.CephExporter,
clusterObj.Status.Cephx.CephExporter,
)
if err != nil {
return "", errors.Wrapf(err, "failed to check if cephx keys should be rotated for ceph exporter %q", exporterKeyringUsername)
@@ -234,8 +234,8 @@ func updateCephExporterCephxStatus(context *clusterd.Context, clusterInfo *clien
if err := context.Client.Get(clusterInfo.Context, clusterInfo.NamespacedName(), cluster); err != nil {
return errors.Wrapf(err, "failed to get cluster %v to update the conditions.", clusterInfo.NamespacedName())
}
updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, clusterInfo.CephVersion, *cluster.Status.Cephx.CephExporter)
cluster.Status.Cephx.CephExporter = &updatedStatus
updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, clusterInfo.CephVersion, cluster.Status.Cephx.CephExporter)
cluster.Status.Cephx.CephExporter = updatedStatus
if err := reporting.UpdateStatus(context.Client, cluster); err != nil {
return errors.Wrap(err, "failed to update cluster cephx status for ceph exporter daemon")
}
pkg/operator/ceph/cluster/nodedaemon/keyring_test.go
+4 -4
View File
@@ -85,8 +85,8 @@ func TestCreateCrashCollectorKeyring(t *testing.T) {
CephStatus: &cephv1.CephStatus{
Health: "",
},
Cephx: &cephv1.ClusterCephxStatus{
CrashCollector: &status,
Cephx: cephv1.ClusterCephxStatus{
CrashCollector: status,
},
},
}
@@ -177,8 +177,8 @@ func TestCreateCephExporterKeyring(t *testing.T) {
CephStatus: &cephv1.CephStatus{
Health: "",
},
Cephx: &cephv1.ClusterCephxStatus{
CephExporter: &status,
Cephx: cephv1.ClusterCephxStatus{
CephExporter: status,
},
},
}
pkg/operator/ceph/cluster/osd/osd.go
+1 -4
View File
@@ -1073,10 +1073,7 @@ func (c *Cluster) updateCephOsdStorageStatus() error {
cephCluster.Status.CephStorage = &cephClusterStorage
if cephx != nil {
if cephCluster.Status.Cephx == nil {
cephCluster.Status.Cephx = &cephv1.ClusterCephxStatus{}
}
cephCluster.Status.Cephx.OSD = cephx
cephCluster.Status.Cephx.OSD = *cephx
}
if err := reporting.UpdateStatus(c.context.Client, &cephCluster); err != nil {
pkg/operator/ceph/cluster/osd/osd_test.go
+4 -4
View File
@@ -952,7 +952,7 @@ func Test_updateCephOsdStorageStatus_cephx(t *testing.T) {
assert.NoError(t, err)
err = context.Client.Get(clusterInfo.Context, clusterInfo.NamespacedName(), cephCluster)
assert.NoError(t, err)
assert.Nil(t, cephCluster.Status.Cephx)
assert.Equal(t, cephv1.ClusterCephxStatus{}, cephCluster.Status.Cephx)
})
t.Run("unset cephx status, no cephcluster status", func(t *testing.T) {
@@ -1046,7 +1046,7 @@ func Test_updateCephOsdStorageStatus_cephx(t *testing.T) {
err = context.Client.Get(clusterInfo.Context, clusterInfo.NamespacedName(), cephCluster)
assert.NoError(t, err)
assert.NotNil(t, cephCluster.Status.Cephx)
assert.Equal(t, cephv1.CephxStatus{KeyGeneration: 1, KeyCephVersion: "v20"}, *cephCluster.Status.Cephx.OSD)
assert.Equal(t, cephv1.CephxStatus{KeyGeneration: 1, KeyCephVersion: "v20"}, cephCluster.Status.Cephx.OSD)
})
t.Run("gen 1 and unset cephx status", func(t *testing.T) {
@@ -1076,7 +1076,7 @@ func Test_updateCephOsdStorageStatus_cephx(t *testing.T) {
err = context.Client.Get(clusterInfo.Context, clusterInfo.NamespacedName(), cephCluster)
assert.NoError(t, err)
assert.NotNil(t, cephCluster.Status.Cephx)
assert.Equal(t, cephv1.CephxStatus{KeyGeneration: 0, KeyCephVersion: ""}, *cephCluster.Status.Cephx.OSD)
assert.Equal(t, cephv1.CephxStatus{KeyGeneration: 0, KeyCephVersion: ""}, cephCluster.Status.Cephx.OSD)
})
t.Run("gen 1 and gen 2 cephx status", func(t *testing.T) {
@@ -1108,7 +1108,7 @@ func Test_updateCephOsdStorageStatus_cephx(t *testing.T) {
err = context.Client.Get(clusterInfo.Context, clusterInfo.NamespacedName(), cephCluster)
assert.NoError(t, err)
assert.NotNil(t, cephCluster.Status.Cephx)
assert.Equal(t, cephv1.CephxStatus{KeyGeneration: 1, KeyCephVersion: "v20"}, *cephCluster.Status.Cephx.OSD)
assert.Equal(t, cephv1.CephxStatus{KeyGeneration: 1, KeyCephVersion: "v20"}, cephCluster.Status.Cephx.OSD)
})
}
pkg/operator/ceph/config/keyring/cephx.go
+7 -1
View File
@@ -26,7 +26,13 @@ import (
"github.com/rook/rook/pkg/operator/ceph/version"
)
var CephAuthRotateSupportedVersion = version.CephVersion{Major: 20, Minor: 2, Extra: 0}
// CephAuthRotateSupportedVersion identifies the first ceph release in which the `ceph auth rotate`
// command is present, thus allowing CephX key rotation.
var CephAuthRotateSupportedVersion = version.CephVersion{Major: 19, Minor: 2, Extra: 3}
// CephAuthMonRotateSupportedVersion identifies the first ceph release in which `ceph auth rotate`
// can be used to rotate the monitor CephX key.
var CephAuthMonRotateSupportedVersion = version.CephVersion{Major: 20, Minor: 3, Extra: 0} // v20.3.0 is unreleased tentacle vers
// CephxKeyIdentifierAnnotation is the annotation that should be applied to pod specs to
// ensure that pods restart after keys are rotated (and not restarted when keys are not rotated).
pkg/operator/ceph/config/keyring/cephx_test.go
+1 -1
View File
@@ -83,7 +83,7 @@ func TestShouldRotateCephxKeys(t *testing.T) {
for _, tt := range tests {
// and run all tests for case where ceph version does not support rotation
t.Run(tt.name, func(t *testing.T) {
got, err := ShouldRotateCephxKeys(tt.cfg, version.CephVersion{Major: 19, Minor: 2, Extra: 4}, tt.imageCephVersion, tt.status)
got, err := ShouldRotateCephxKeys(tt.cfg, version.CephVersion{Major: 19, Minor: 2, Extra: 2}, tt.imageCephVersion, tt.status)
assert.NoError(t, err)
assert.False(t, got)
})
pkg/operator/ceph/controller/mirror_peer.go
+3 -3
View File
@@ -249,7 +249,7 @@ func shouldRotateMirrorPeerKeys(c *clusterd.Context, clusterInfo *cephclient.Clu
desiredCephVersion := clusterInfo.CephVersion
runningCephVersion := clusterInfo.CephVersion
shouldRotateKeys, err := keyring.ShouldRotateCephxKeys(cephObj.Spec.Security.CephX.RBDMirrorPeer, runningCephVersion, desiredCephVersion, *cephObj.Status.Cephx.RBDMirrorPeer)
shouldRotateKeys, err := keyring.ShouldRotateCephxKeys(cephObj.Spec.Security.CephX.RBDMirrorPeer, runningCephVersion, desiredCephVersion, cephObj.Status.Cephx.RBDMirrorPeer)
if err != nil {
return false, errors.Wrap(err, "failed to check if mirror peer keys should be rotated or not")
}
@@ -264,8 +264,8 @@ func updateCephClusterCephxRbdMirrorStatus(c *clusterd.Context, clusterInfo *cep
if err := c.Client.Get(clusterInfo.Context, clusterInfo.NamespacedName(), cluster); err != nil {
return errors.Wrapf(err, "failed to get cluster %v to update the conditions.", clusterInfo.NamespacedName())
}
updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.RBDMirrorPeer, clusterInfo.CephVersion, *cluster.Status.Cephx.RBDMirrorPeer)
cluster.Status.Cephx.RBDMirrorPeer = &updatedStatus
updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.RBDMirrorPeer, clusterInfo.CephVersion, cluster.Status.Cephx.RBDMirrorPeer)
cluster.Status.Cephx.RBDMirrorPeer = updatedStatus
logger.Debugf("updating rbd-mirror cephx status to %+v", cluster.Status.Cephx.RBDMirrorPeer)
if err := reporting.UpdateStatus(c.Client, cluster); err != nil {
return errors.Wrap(err, "failed to update cluster cephx status for rbd-mirror")
pkg/operator/ceph/pool/controller.go
+1 -1
View File
@@ -364,7 +364,7 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile
}
// update rbdMirror cephXStatus immediately after bootstrapping the peer token
statusErr = r.updateStatus(request.NamespacedName, cephv1.ConditionProgressing, observedGeneration, cephCluster.Status.Cephx.RBDMirrorPeer)
statusErr = r.updateStatus(request.NamespacedName, cephv1.ConditionProgressing, observedGeneration, &cephCluster.Status.Cephx.RBDMirrorPeer)
if statusErr != nil {
return opcontroller.ImmediateRetryResult, *cephBlockPool, errors.Wrapf(statusErr, "failed to update %q status to %q", request.NamespacedName, cephv1.ConditionProgressing)
}
pkg/operator/ceph/pool/controller_test.go
+4 -4
View File
@@ -317,7 +317,7 @@ func TestCephBlockPoolController(t *testing.T) {
CephStatus: &cephv1.CephStatus{
Health: "",
},
Cephx: &cephv1.ClusterCephxStatus{},
Cephx: cephv1.ClusterCephxStatus{},
},
}
@@ -1026,7 +1026,7 @@ func TestMirrorPeerKeyRotationStatus(t *testing.T) {
CephStatus: &cephv1.CephStatus{
Health: "HEALTH_OK",
},
Cephx: &cephv1.ClusterCephxStatus{},
Cephx: cephv1.ClusterCephxStatus{},
},
}
@@ -1188,7 +1188,7 @@ func TestMirrorPeerKeyRotationStatus(t *testing.T) {
})
t.Run("PeerToken cephx status should be updated based on the cephCluster cephx status", func(t *testing.T) {
RDBMirrorPeerStatus := &cephv1.CephxStatus{
RDBMirrorPeerStatus := cephv1.CephxStatus{
KeyGeneration: 2,
KeyCephVersion: "20.2.0-0",
}
@@ -1201,6 +1201,6 @@ func TestMirrorPeerKeyRotationStatus(t *testing.T) {
pool := &cephv1.CephBlockPool{}
err = r.client.Get(context.TODO(), req.NamespacedName, pool)
assert.NoError(t, err)
assert.Equal(t, *RDBMirrorPeerStatus, pool.Status.Cephx.PeerToken)
assert.Equal(t, RDBMirrorPeerStatus, pool.Status.Cephx.PeerToken)
})
}