mon: rotate keys
rotate mon daemon keys and update the cephx status for mons in the cephcluster resource. Signed-off-by: Santosh Pillai <sapillai@redhat.com>
This commit is contained in:
1 parent
ea0203b86c
commit
fe42fe102f
9 files changed
+267
-2
No files matched your search
@@ -5045,6 +5045,19 @@ CSI key rotation can affect existing PV connections, so take care when exercisin
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<code>mon</code><br/>
|
||||
<em>
|
||||
<a href="#ceph.rook.io/v1.CephxStatus">
|
||||
CephxStatus
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<p>Mon represents the CephX key status of the Monitor daemons</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>mgr</code><br/>
|
||||
<em>
|
||||
<a href="#ceph.rook.io/v1.CephxStatus">
|
||||
|
||||
@@ -6018,6 +6018,28 @@ spec:
|
||||
format: int32
|
||||
type: integer
|
||||
type: object
|
||||
mon:
|
||||
description: Mon represents the CephX key status of the Monitor daemons
|
||||
properties:
|
||||
keyCephVersion:
|
||||
description: |-
|
||||
KeyCephVersion reports the Ceph version that created the current generation's keys. This is
|
||||
same string format as reported by `CephCluster.status.version.version` to allow them to be
|
||||
compared. E.g., `20.2.0-0`.
|
||||
For all newly-created resources, this field set to the version of Ceph that created the key.
|
||||
The special value "Uninitialized" indicates that keys are being created for the first time.
|
||||
An empty string indicates that the version is unknown, as expected in brownfield deployments.
|
||||
type: string
|
||||
keyGeneration:
|
||||
description: |-
|
||||
KeyGeneration represents the CephX key generation for the last successful reconcile.
|
||||
For all newly-created resources, this field is set to `1`.
|
||||
When keys are rotated due to any rotation policy, the generation is incremented or updated to
|
||||
the configured policy generation.
|
||||
Generation `0` indicates that keys existed prior to the implementation of key tracking.
|
||||
format: int32
|
||||
type: integer
|
||||
type: object
|
||||
osd:
|
||||
description: OSD shows the CephX key status of of OSDs
|
||||
properties:
|
||||
|
||||
@@ -6016,6 +6016,28 @@ spec:
|
||||
format: int32
|
||||
type: integer
|
||||
type: object
|
||||
mon:
|
||||
description: Mon represents the CephX key status of the Monitor daemons
|
||||
properties:
|
||||
keyCephVersion:
|
||||
description: |-
|
||||
KeyCephVersion reports the Ceph version that created the current generation's keys. This is
|
||||
same string format as reported by `CephCluster.status.version.version` to allow them to be
|
||||
compared. E.g., `20.2.0-0`.
|
||||
For all newly-created resources, this field set to the version of Ceph that created the key.
|
||||
The special value "Uninitialized" indicates that keys are being created for the first time.
|
||||
An empty string indicates that the version is unknown, as expected in brownfield deployments.
|
||||
type: string
|
||||
keyGeneration:
|
||||
description: |-
|
||||
KeyGeneration represents the CephX key generation for the last successful reconcile.
|
||||
For all newly-created resources, this field is set to `1`.
|
||||
When keys are rotated due to any rotation policy, the generation is incremented or updated to
|
||||
the configured policy generation.
|
||||
Generation `0` indicates that keys existed prior to the implementation of key tracking.
|
||||
format: int32
|
||||
type: integer
|
||||
type: object
|
||||
osd:
|
||||
description: OSD shows the CephX key status of of OSDs
|
||||
properties:
|
||||
|
||||
@@ -742,6 +742,8 @@ type LocalCephxStatus struct {
|
||||
|
||||
// ClusterCephxStatus defines the cephx key rotation status of various daemons on the cephCluster resource
|
||||
type ClusterCephxStatus struct {
|
||||
// Mon represents the CephX key status of the Monitor daemons
|
||||
Mon *CephxStatus `json:"mon,omitempty"`
|
||||
// Mgr represents the cephx key rotation status of the ceph manager daemon
|
||||
Mgr *CephxStatus `json:"mgr,omitempty"`
|
||||
// OSD shows the CephX key status of of OSDs
|
||||
|
||||
@@ -2075,6 +2075,11 @@ func (in *ClusterCephxConfig) DeepCopy() *ClusterCephxConfig {
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *ClusterCephxStatus) DeepCopyInto(out *ClusterCephxStatus) {
|
||||
*out = *in
|
||||
if in.Mon != nil {
|
||||
in, out := &in.Mon, &out.Mon
|
||||
*out = new(CephxStatus)
|
||||
**out = **in
|
||||
}
|
||||
if in.Mgr != nil {
|
||||
in, out := &in.Mgr, &out.Mgr
|
||||
*out = new(CephxStatus)
|
||||
|
||||
@@ -460,6 +460,27 @@ func (c *cluster) preMonStartupActions(cephVersion cephver.CephVersion) error {
|
||||
// It gets executed right after the main mon Start() method
|
||||
// Basically, it is executed between the monitors and the manager sequence
|
||||
func (c *cluster) postMonStartupActions() error {
|
||||
clusterObj := &cephv1.CephCluster{}
|
||||
if err := c.context.Client.Get(c.ClusterInfo.Context, c.ClusterInfo.NamespacedName(), clusterObj); err != nil {
|
||||
return errors.Wrapf(err, "failed to get cluster %v.", c.ClusterInfo.NamespacedName())
|
||||
}
|
||||
|
||||
// rotate mon cephx keys if required
|
||||
didRotateMonCephxKeys, err := c.mons.RotateMonCephxKeys(clusterObj)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to rotate mon cephx keys in the namespace %q", c.ClusterInfo.Namespace)
|
||||
}
|
||||
err = c.mons.UpdateMonCephxStatus(didRotateMonCephxKeys)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to update cephx status for mon daemons in the namespace %q", c.ClusterInfo.Namespace)
|
||||
}
|
||||
|
||||
// reconcile to restart the mons after cephx key rotation
|
||||
if didRotateMonCephxKeys {
|
||||
// reconcile the rook operator so that it will restart the mons after mon cephx key rotation
|
||||
return errors.New("triggering a new reconcile to restart the mon daemons after mon cephx key rotation")
|
||||
}
|
||||
|
||||
// Create CSI Kubernetes Secrets
|
||||
if err := csi.CreateCSISecrets(c.context, c.ClusterInfo, c.namespacedName); err != nil {
|
||||
return errors.Wrap(err, "failed to create csi kubernetes secrets")
|
||||
@@ -801,6 +822,7 @@ func initClusterCephxStatus(c *clusterd.Context, cluster *cephv1.CephCluster) er
|
||||
OSD: &uninitializedStatus,
|
||||
RBDMirrorPeer: &uninitializedStatus,
|
||||
Mgr: &uninitializedStatus,
|
||||
Mon: &uninitializedStatus,
|
||||
CSI: &cephv1.CephxStatusWithKeyCount{
|
||||
CephxStatus: uninitializedStatus,
|
||||
},
|
||||
|
||||
@@ -41,6 +41,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/ceph/csi"
|
||||
"github.com/rook/rook/pkg/operator/ceph/reporting"
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
@@ -51,6 +52,7 @@ import (
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/client-go/util/retry"
|
||||
"k8s.io/utils/ptr"
|
||||
|
||||
cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes"
|
||||
@@ -133,6 +135,8 @@ type Cluster struct {
|
||||
arbiterMon string
|
||||
// list of mons to be failed over
|
||||
monsToFailover map[string]*monConfig
|
||||
// reference to the secret that stores mon key
|
||||
monKeySecretResourceVersion string
|
||||
}
|
||||
|
||||
// monConfig for a single monitor
|
||||
@@ -550,9 +554,12 @@ func (c *Cluster) initClusterInfo(cephVersion cephver.CephVersion, clusterName s
|
||||
|
||||
k := keyring.GetSecretStore(c.context, c.ClusterInfo, c.ownerInfo)
|
||||
// store the keyring which all mons share
|
||||
if _, err := k.CreateOrUpdate(keyringStoreName, c.genMonSharedKeyring()); err != nil {
|
||||
secretResourceVersion, err := k.CreateOrUpdate(keyringStoreName, c.genMonSharedKeyring())
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to save mon keyring secret")
|
||||
}
|
||||
c.monKeySecretResourceVersion = secretResourceVersion
|
||||
|
||||
// also store the admin keyring for other daemons that might need it during init
|
||||
if err := k.Admin().CreateOrUpdate(c.ClusterInfo, c.context, c.spec.Annotations); err != nil {
|
||||
return errors.Wrap(err, "failed to save admin keyring secret")
|
||||
@@ -1516,6 +1523,9 @@ func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) e
|
||||
return err
|
||||
}
|
||||
|
||||
// apply cephx secret resource version to the deployment to ensure it restarts after key is rotated
|
||||
d.Spec.Template.Annotations[keyring.CephxKeyIdentifierAnnotation] = c.monKeySecretResourceVersion
|
||||
|
||||
// Set the deployment hash as an annotation
|
||||
err = patch.DefaultAnnotator.SetLastAppliedAnnotation(d)
|
||||
if err != nil {
|
||||
@@ -1791,3 +1801,67 @@ func (c *Cluster) releaseOrchestrationLock() {
|
||||
c.orchestrationMutex.Unlock()
|
||||
logger.Debugf("Released lock for mon orchestration")
|
||||
}
|
||||
|
||||
func (c *Cluster) RotateMonCephxKeys(clusterObj *cephv1.CephCluster) (bool, error) {
|
||||
desiredCephVersion := c.ClusterInfo.CephVersion
|
||||
// TODO: for rotation WithCephVersionUpdate fix this to have the right runningCephVersion and desiredCephVersion
|
||||
runningCephVersion := c.ClusterInfo.CephVersion
|
||||
|
||||
shouldRotateMonKeys, err := keyring.ShouldRotateCephxKeys(clusterObj.Spec.Security.CephX.Daemon, runningCephVersion, desiredCephVersion, *clusterObj.Status.Cephx.Mon)
|
||||
if err != nil {
|
||||
return false, errors.Wrapf(err, "failed to check if mon daemon keys should be rotated in the namespace %q", c.ClusterInfo.Namespace)
|
||||
}
|
||||
|
||||
if !shouldRotateMonKeys {
|
||||
logger.Debugf("cephx key rotation for mon daemon in the namespace %q is not required", c.ClusterInfo.Namespace)
|
||||
return shouldRotateMonKeys, nil
|
||||
}
|
||||
|
||||
logger.Infof("cephx keys for mon daemons in the namespace %q will be rotated", c.ClusterInfo.Namespace)
|
||||
|
||||
k := keyring.GetSecretStore(c.context, c.ClusterInfo, c.ClusterInfo.OwnerInfo)
|
||||
newKey, err := k.RotateKey(controller.MonCephxUser)
|
||||
if err != nil {
|
||||
return shouldRotateMonKeys, errors.Wrapf(err, "failed to rotate cephx key for mon daemon in the namespace %q", c.ClusterInfo.Namespace)
|
||||
}
|
||||
|
||||
c.ClusterInfo.MonitorSecret = newKey
|
||||
|
||||
// update the mon-secret key in the cluster access secret
|
||||
err = controller.UpdateClusterAccessSecret(c.context.Clientset, c.ClusterInfo)
|
||||
if err != nil {
|
||||
return shouldRotateMonKeys, errors.Wrapf(err, "failed to update the rook-ceph-mon secret after rotating the mon cephx keys in the namespace %q", c.ClusterInfo.Namespace)
|
||||
}
|
||||
|
||||
// update the keyring which all mons share
|
||||
if _, err := k.CreateOrUpdate(keyringStoreName, c.genMonSharedKeyring()); err != nil {
|
||||
return shouldRotateMonKeys, errors.Wrapf(err, "failed to save mon keyring secret after rotating the mon cephx keys in the namespace %q", c.ClusterInfo.Namespace)
|
||||
}
|
||||
|
||||
logger.Infof("successfully rotated cephx keys for mon daemons in the namespace %q", c.ClusterInfo.Namespace)
|
||||
|
||||
return shouldRotateMonKeys, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) UpdateMonCephxStatus(didRotate bool) error {
|
||||
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||
cluster := &cephv1.CephCluster{}
|
||||
if err := c.context.Client.Get(c.ClusterInfo.Context, c.ClusterInfo.NamespacedName(), cluster); err != nil {
|
||||
return errors.Wrapf(err, "failed to get cluster %v to update the mon cephx status.", c.ClusterInfo.NamespacedName())
|
||||
}
|
||||
updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, c.ClusterInfo.CephVersion, *cluster.Status.Cephx.Mon)
|
||||
cluster.Status.Cephx.Mon = &updatedStatus
|
||||
logger.Debugf("updating mon daemon cephx status to %+v", cluster.Status.Cephx.Mgr)
|
||||
if err := reporting.UpdateStatus(c.context.Client, cluster); err != nil {
|
||||
return errors.Wrapf(err, "failed to update cluster cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace)
|
||||
}
|
||||
logger.Infof("successfully updated the cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace)
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to update cluster cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -30,10 +30,12 @@ import (
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/pkg/errors"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
@@ -45,7 +47,9 @@ import (
|
||||
discoveryv1 "k8s.io/api/discovery/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/utils/ptr"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
)
|
||||
|
||||
// generate a standard mon config from a mon id w/ default port and IP 2.4.6.{1,2,3,...}
|
||||
@@ -1165,3 +1169,84 @@ func TestIsMonIPUpdateRequiredForHostNetwork(t *testing.T) {
|
||||
assert.True(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork))
|
||||
})
|
||||
}
|
||||
|
||||
func TestRotateMonCephxKeys(t *testing.T) {
|
||||
ctx := context.TODO()
|
||||
namespace := "default"
|
||||
context, err := newTestStartCluster(t, namespace)
|
||||
assert.NoError(t, err)
|
||||
c := newCluster(context, namespace, true, v1.ResourceRequirements{})
|
||||
|
||||
uninitializedStatus := keyring.UninitializedCephxStatus()
|
||||
cluster := &cephv1.CephCluster{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: namespace,
|
||||
Namespace: namespace,
|
||||
},
|
||||
Spec: cephv1.ClusterSpec{
|
||||
Security: cephv1.ClusterSecuritySpec{
|
||||
CephX: cephv1.ClusterCephxConfig{
|
||||
Daemon: cephv1.CephxConfig{},
|
||||
},
|
||||
},
|
||||
},
|
||||
Status: cephv1.ClusterStatus{
|
||||
Cephx: &cephv1.ClusterCephxStatus{
|
||||
Mon: &uninitializedStatus,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
s := scheme.Scheme
|
||||
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{})
|
||||
s.AddKnownTypes(v1.SchemeGroupVersion, &v1.Secret{})
|
||||
|
||||
object := []runtime.Object{
|
||||
cluster,
|
||||
}
|
||||
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
|
||||
|
||||
executor := &exectest.MockExecutor{
|
||||
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
||||
logger.Infof("%s %v", command, args)
|
||||
if args[0] == "auth" && args[1] == "rotate" {
|
||||
return `[{"key":"myrotatedkey"}]`, nil
|
||||
}
|
||||
return "", errors.New("unknown command")
|
||||
},
|
||||
}
|
||||
c.context = &clusterd.Context{Clientset: test.New(t, 5), Executor: executor, Client: cl}
|
||||
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
|
||||
c.ClusterInfo.CephVersion = keyring.CephAuthRotateSupportedVersion
|
||||
|
||||
// create rook-ceph-mon secret
|
||||
secret := &v1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: AppName,
|
||||
Namespace: namespace,
|
||||
},
|
||||
Data: map[string][]byte{opcontroller.MonSecretNameKey: []byte("bar")},
|
||||
}
|
||||
|
||||
_, err = c.context.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// No key rotation required
|
||||
shouldRotate, err := c.RotateMonCephxKeys(cluster)
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, shouldRotate)
|
||||
|
||||
// verify key is rotated in the secret
|
||||
cluster.Spec.Security.CephX.Daemon.KeyRotationPolicy = cephv1.KeyGenerationCephxKeyRotationPolicy
|
||||
cluster.Spec.Security.CephX.Daemon.KeyGeneration = 2
|
||||
cluster.Status.Cephx.Mon.KeyGeneration = 1
|
||||
cluster.Status.Cephx.Mon.KeyCephVersion = keyring.CephAuthRotateSupportedVersion.String()
|
||||
err = c.context.Client.Update(ctx, cluster)
|
||||
assert.NoError(t, err)
|
||||
shouldRotate, err = c.RotateMonCephxKeys(cluster)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, shouldRotate)
|
||||
secret, err = c.context.Clientset.CoreV1().Secrets(namespace).Get(c.ClusterInfo.Context, AppName, metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "myrotatedkey", string(secret.Data[opcontroller.MonSecretNameKey]))
|
||||
}
|
||||
@@ -68,6 +68,8 @@ const (
|
||||
// AppName is the name of the secret storing cluster mon.admin key, fsid and name
|
||||
AppName = "rook-ceph-mon"
|
||||
DisasterProtectionFinalizerName = cephv1.CustomResourceGroup + "/disaster-protection"
|
||||
// MonCephxUser represents the cephx user name for all the mon daemons
|
||||
MonCephxUser = "mon."
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -202,7 +204,7 @@ func createNamedClusterInfo(context *clusterd.Context, namespace string) (*cephc
|
||||
}
|
||||
|
||||
// generate the mon secret
|
||||
monSecret, err := genSecret(context.Executor, dir, "mon.", []string{"--cap", "mon", "'allow *'"})
|
||||
monSecret, err := genSecret(context.Executor, dir, MonCephxUser, []string{"--cap", "mon", "'allow *'"})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -397,6 +399,24 @@ func createClusterAccessSecret(clientset kubernetes.Interface, namespace string,
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateClusterAccessSecret updates the rook-ceph-mon secret
|
||||
func UpdateClusterAccessSecret(clientset kubernetes.Interface, clusterInfo *cephclient.ClusterInfo) error {
|
||||
secret, err := clientset.CoreV1().Secrets(clusterInfo.Namespace).Get(clusterInfo.Context, AppName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to get secret %q in the namespace %q", AppName, clusterInfo.Namespace)
|
||||
}
|
||||
|
||||
secret.Data[MonSecretNameKey] = []byte(clusterInfo.MonitorSecret)
|
||||
|
||||
logger.Infof("updating secret %q in the namespace %q", secret.Name, clusterInfo.Namespace)
|
||||
_, err = clientset.CoreV1().Secrets(clusterInfo.Namespace).Update(clusterInfo.Context, secret, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to update secret %q in the namespace %q", secret.Name, clusterInfo.Namespace)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ParseMonEndpoints parses a flattened representation of mons and endpoints in the form
|
||||
// <mon-name>=<mon-endpoint> and returns a list of Ceph mon configs.
|
||||
func ParseMonEndpoints(input string) map[string]*cephclient.MonInfo {
|
||||
|
||||
Reference in new issue
Block a user