diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index f9068e312..e88d19a3a 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -5045,6 +5045,19 @@ CSI key rotation can affect existing PV connections, so take care when exercisin +mon
+ + +CephxStatus + + + + +

Mon represents the CephX key status of the Monitor daemons

+ + + + mgr
diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index ae42d500e..889df80b7 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -6018,6 +6018,28 @@ spec: format: int32 type: integer type: object + mon: + description: Mon represents the CephX key status of the Monitor daemons + properties: + keyCephVersion: + description: |- + KeyCephVersion reports the Ceph version that created the current generation's keys. This is + same string format as reported by `CephCluster.status.version.version` to allow them to be + compared. E.g., `20.2.0-0`. + For all newly-created resources, this field set to the version of Ceph that created the key. + The special value "Uninitialized" indicates that keys are being created for the first time. + An empty string indicates that the version is unknown, as expected in brownfield deployments. + type: string + keyGeneration: + description: |- + KeyGeneration represents the CephX key generation for the last successful reconcile. + For all newly-created resources, this field is set to `1`. + When keys are rotated due to any rotation policy, the generation is incremented or updated to + the configured policy generation. + Generation `0` indicates that keys existed prior to the implementation of key tracking. + format: int32 + type: integer + type: object osd: description: OSD shows the CephX key status of of OSDs properties: diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 79f6700aa..4302f1da7 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -6016,6 +6016,28 @@ spec: format: int32 type: integer type: object + mon: + description: Mon represents the CephX key status of the Monitor daemons + properties: + keyCephVersion: + description: |- + KeyCephVersion reports the Ceph version that created the current generation's keys. This is + same string format as reported by `CephCluster.status.version.version` to allow them to be + compared. E.g., `20.2.0-0`. + For all newly-created resources, this field set to the version of Ceph that created the key. + The special value "Uninitialized" indicates that keys are being created for the first time. + An empty string indicates that the version is unknown, as expected in brownfield deployments. + type: string + keyGeneration: + description: |- + KeyGeneration represents the CephX key generation for the last successful reconcile. + For all newly-created resources, this field is set to `1`. + When keys are rotated due to any rotation policy, the generation is incremented or updated to + the configured policy generation. + Generation `0` indicates that keys existed prior to the implementation of key tracking. + format: int32 + type: integer + type: object osd: description: OSD shows the CephX key status of of OSDs properties: diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index fae5e1e07..64d67a067 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -742,6 +742,8 @@ type LocalCephxStatus struct { // ClusterCephxStatus defines the cephx key rotation status of various daemons on the cephCluster resource type ClusterCephxStatus struct { + // Mon represents the CephX key status of the Monitor daemons + Mon *CephxStatus `json:"mon,omitempty"` // Mgr represents the cephx key rotation status of the ceph manager daemon Mgr *CephxStatus `json:"mgr,omitempty"` // OSD shows the CephX key status of of OSDs diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index 8e93f0c77..4e26496c5 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -2075,6 +2075,11 @@ func (in *ClusterCephxConfig) DeepCopy() *ClusterCephxConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterCephxStatus) DeepCopyInto(out *ClusterCephxStatus) { *out = *in + if in.Mon != nil { + in, out := &in.Mon, &out.Mon + *out = new(CephxStatus) + **out = **in + } if in.Mgr != nil { in, out := &in.Mgr, &out.Mgr *out = new(CephxStatus) diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 4ce709f57..f015e6298 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -460,6 +460,27 @@ func (c *cluster) preMonStartupActions(cephVersion cephver.CephVersion) error { // It gets executed right after the main mon Start() method // Basically, it is executed between the monitors and the manager sequence func (c *cluster) postMonStartupActions() error { + clusterObj := &cephv1.CephCluster{} + if err := c.context.Client.Get(c.ClusterInfo.Context, c.ClusterInfo.NamespacedName(), clusterObj); err != nil { + return errors.Wrapf(err, "failed to get cluster %v.", c.ClusterInfo.NamespacedName()) + } + + // rotate mon cephx keys if required + didRotateMonCephxKeys, err := c.mons.RotateMonCephxKeys(clusterObj) + if err != nil { + return errors.Wrapf(err, "failed to rotate mon cephx keys in the namespace %q", c.ClusterInfo.Namespace) + } + err = c.mons.UpdateMonCephxStatus(didRotateMonCephxKeys) + if err != nil { + return errors.Wrapf(err, "failed to update cephx status for mon daemons in the namespace %q", c.ClusterInfo.Namespace) + } + + // reconcile to restart the mons after cephx key rotation + if didRotateMonCephxKeys { + // reconcile the rook operator so that it will restart the mons after mon cephx key rotation + return errors.New("triggering a new reconcile to restart the mon daemons after mon cephx key rotation") + } + // Create CSI Kubernetes Secrets if err := csi.CreateCSISecrets(c.context, c.ClusterInfo, c.namespacedName); err != nil { return errors.Wrap(err, "failed to create csi kubernetes secrets") @@ -801,6 +822,7 @@ func initClusterCephxStatus(c *clusterd.Context, cluster *cephv1.CephCluster) er OSD: &uninitializedStatus, RBDMirrorPeer: &uninitializedStatus, Mgr: &uninitializedStatus, + Mon: &uninitializedStatus, CSI: &cephv1.CephxStatusWithKeyCount{ CephxStatus: uninitializedStatus, }, diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index bc43bb8d2..2b170186b 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -41,6 +41,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config/keyring" "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/csi" + "github.com/rook/rook/pkg/operator/ceph/reporting" cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" @@ -51,6 +52,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" "k8s.io/utils/ptr" cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes" @@ -133,6 +135,8 @@ type Cluster struct { arbiterMon string // list of mons to be failed over monsToFailover map[string]*monConfig + // reference to the secret that stores mon key + monKeySecretResourceVersion string } // monConfig for a single monitor @@ -550,9 +554,12 @@ func (c *Cluster) initClusterInfo(cephVersion cephver.CephVersion, clusterName s k := keyring.GetSecretStore(c.context, c.ClusterInfo, c.ownerInfo) // store the keyring which all mons share - if _, err := k.CreateOrUpdate(keyringStoreName, c.genMonSharedKeyring()); err != nil { + secretResourceVersion, err := k.CreateOrUpdate(keyringStoreName, c.genMonSharedKeyring()) + if err != nil { return errors.Wrap(err, "failed to save mon keyring secret") } + c.monKeySecretResourceVersion = secretResourceVersion + // also store the admin keyring for other daemons that might need it during init if err := k.Admin().CreateOrUpdate(c.ClusterInfo, c.context, c.spec.Annotations); err != nil { return errors.Wrap(err, "failed to save admin keyring secret") @@ -1516,6 +1523,9 @@ func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) e return err } + // apply cephx secret resource version to the deployment to ensure it restarts after key is rotated + d.Spec.Template.Annotations[keyring.CephxKeyIdentifierAnnotation] = c.monKeySecretResourceVersion + // Set the deployment hash as an annotation err = patch.DefaultAnnotator.SetLastAppliedAnnotation(d) if err != nil { @@ -1791,3 +1801,67 @@ func (c *Cluster) releaseOrchestrationLock() { c.orchestrationMutex.Unlock() logger.Debugf("Released lock for mon orchestration") } + +func (c *Cluster) RotateMonCephxKeys(clusterObj *cephv1.CephCluster) (bool, error) { + desiredCephVersion := c.ClusterInfo.CephVersion + // TODO: for rotation WithCephVersionUpdate fix this to have the right runningCephVersion and desiredCephVersion + runningCephVersion := c.ClusterInfo.CephVersion + + shouldRotateMonKeys, err := keyring.ShouldRotateCephxKeys(clusterObj.Spec.Security.CephX.Daemon, runningCephVersion, desiredCephVersion, *clusterObj.Status.Cephx.Mon) + if err != nil { + return false, errors.Wrapf(err, "failed to check if mon daemon keys should be rotated in the namespace %q", c.ClusterInfo.Namespace) + } + + if !shouldRotateMonKeys { + logger.Debugf("cephx key rotation for mon daemon in the namespace %q is not required", c.ClusterInfo.Namespace) + return shouldRotateMonKeys, nil + } + + logger.Infof("cephx keys for mon daemons in the namespace %q will be rotated", c.ClusterInfo.Namespace) + + k := keyring.GetSecretStore(c.context, c.ClusterInfo, c.ClusterInfo.OwnerInfo) + newKey, err := k.RotateKey(controller.MonCephxUser) + if err != nil { + return shouldRotateMonKeys, errors.Wrapf(err, "failed to rotate cephx key for mon daemon in the namespace %q", c.ClusterInfo.Namespace) + } + + c.ClusterInfo.MonitorSecret = newKey + + // update the mon-secret key in the cluster access secret + err = controller.UpdateClusterAccessSecret(c.context.Clientset, c.ClusterInfo) + if err != nil { + return shouldRotateMonKeys, errors.Wrapf(err, "failed to update the rook-ceph-mon secret after rotating the mon cephx keys in the namespace %q", c.ClusterInfo.Namespace) + } + + // update the keyring which all mons share + if _, err := k.CreateOrUpdate(keyringStoreName, c.genMonSharedKeyring()); err != nil { + return shouldRotateMonKeys, errors.Wrapf(err, "failed to save mon keyring secret after rotating the mon cephx keys in the namespace %q", c.ClusterInfo.Namespace) + } + + logger.Infof("successfully rotated cephx keys for mon daemons in the namespace %q", c.ClusterInfo.Namespace) + + return shouldRotateMonKeys, nil +} + +func (c *Cluster) UpdateMonCephxStatus(didRotate bool) error { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + cluster := &cephv1.CephCluster{} + if err := c.context.Client.Get(c.ClusterInfo.Context, c.ClusterInfo.NamespacedName(), cluster); err != nil { + return errors.Wrapf(err, "failed to get cluster %v to update the mon cephx status.", c.ClusterInfo.NamespacedName()) + } + updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, c.ClusterInfo.CephVersion, *cluster.Status.Cephx.Mon) + cluster.Status.Cephx.Mon = &updatedStatus + logger.Debugf("updating mon daemon cephx status to %+v", cluster.Status.Cephx.Mgr) + if err := reporting.UpdateStatus(c.context.Client, cluster); err != nil { + return errors.Wrapf(err, "failed to update cluster cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace) + } + logger.Infof("successfully updated the cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace) + + return nil + }) + if err != nil { + return errors.Wrapf(err, "failed to update cluster cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace) + } + + return nil +} diff --git a/pkg/operator/ceph/cluster/mon/mon_test.go b/pkg/operator/ceph/cluster/mon/mon_test.go index 6484d2211..97648bcea 100644 --- a/pkg/operator/ceph/cluster/mon/mon_test.go +++ b/pkg/operator/ceph/cluster/mon/mon_test.go @@ -30,10 +30,12 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/client/clientset/versioned/scheme" "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test" "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/config/keyring" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" @@ -45,7 +47,9 @@ import ( discoveryv1 "k8s.io/api/discovery/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) // generate a standard mon config from a mon id w/ default port and IP 2.4.6.{1,2,3,...} @@ -1165,3 +1169,84 @@ func TestIsMonIPUpdateRequiredForHostNetwork(t *testing.T) { assert.True(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork)) }) } + +func TestRotateMonCephxKeys(t *testing.T) { + ctx := context.TODO() + namespace := "default" + context, err := newTestStartCluster(t, namespace) + assert.NoError(t, err) + c := newCluster(context, namespace, true, v1.ResourceRequirements{}) + + uninitializedStatus := keyring.UninitializedCephxStatus() + cluster := &cephv1.CephCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Namespace: namespace, + }, + Spec: cephv1.ClusterSpec{ + Security: cephv1.ClusterSecuritySpec{ + CephX: cephv1.ClusterCephxConfig{ + Daemon: cephv1.CephxConfig{}, + }, + }, + }, + Status: cephv1.ClusterStatus{ + Cephx: &cephv1.ClusterCephxStatus{ + Mon: &uninitializedStatus, + }, + }, + } + + s := scheme.Scheme + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}) + s.AddKnownTypes(v1.SchemeGroupVersion, &v1.Secret{}) + + object := []runtime.Object{ + cluster, + } + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + + executor := &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { + logger.Infof("%s %v", command, args) + if args[0] == "auth" && args[1] == "rotate" { + return `[{"key":"myrotatedkey"}]`, nil + } + return "", errors.New("unknown command") + }, + } + c.context = &clusterd.Context{Clientset: test.New(t, 5), Executor: executor, Client: cl} + c.ClusterInfo = clienttest.CreateTestClusterInfo(1) + c.ClusterInfo.CephVersion = keyring.CephAuthRotateSupportedVersion + + // create rook-ceph-mon secret + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: AppName, + Namespace: namespace, + }, + Data: map[string][]byte{opcontroller.MonSecretNameKey: []byte("bar")}, + } + + _, err = c.context.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + assert.NoError(t, err) + + // No key rotation required + shouldRotate, err := c.RotateMonCephxKeys(cluster) + assert.NoError(t, err) + assert.False(t, shouldRotate) + + // verify key is rotated in the secret + cluster.Spec.Security.CephX.Daemon.KeyRotationPolicy = cephv1.KeyGenerationCephxKeyRotationPolicy + cluster.Spec.Security.CephX.Daemon.KeyGeneration = 2 + cluster.Status.Cephx.Mon.KeyGeneration = 1 + cluster.Status.Cephx.Mon.KeyCephVersion = keyring.CephAuthRotateSupportedVersion.String() + err = c.context.Client.Update(ctx, cluster) + assert.NoError(t, err) + shouldRotate, err = c.RotateMonCephxKeys(cluster) + assert.NoError(t, err) + assert.True(t, shouldRotate) + secret, err = c.context.Clientset.CoreV1().Secrets(namespace).Get(c.ClusterInfo.Context, AppName, metav1.GetOptions{}) + assert.NoError(t, err) + assert.Equal(t, "myrotatedkey", string(secret.Data[opcontroller.MonSecretNameKey])) +} diff --git a/pkg/operator/ceph/controller/cluster_info.go b/pkg/operator/ceph/controller/cluster_info.go index 22e6d1d6c..9e3fb75a8 100644 --- a/pkg/operator/ceph/controller/cluster_info.go +++ b/pkg/operator/ceph/controller/cluster_info.go @@ -68,6 +68,8 @@ const ( // AppName is the name of the secret storing cluster mon.admin key, fsid and name AppName = "rook-ceph-mon" DisasterProtectionFinalizerName = cephv1.CustomResourceGroup + "/disaster-protection" + // MonCephxUser represents the cephx user name for all the mon daemons + MonCephxUser = "mon." ) var ( @@ -202,7 +204,7 @@ func createNamedClusterInfo(context *clusterd.Context, namespace string) (*cephc } // generate the mon secret - monSecret, err := genSecret(context.Executor, dir, "mon.", []string{"--cap", "mon", "'allow *'"}) + monSecret, err := genSecret(context.Executor, dir, MonCephxUser, []string{"--cap", "mon", "'allow *'"}) if err != nil { return nil, err } @@ -397,6 +399,24 @@ func createClusterAccessSecret(clientset kubernetes.Interface, namespace string, return nil } +// UpdateClusterAccessSecret updates the rook-ceph-mon secret +func UpdateClusterAccessSecret(clientset kubernetes.Interface, clusterInfo *cephclient.ClusterInfo) error { + secret, err := clientset.CoreV1().Secrets(clusterInfo.Namespace).Get(clusterInfo.Context, AppName, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "failed to get secret %q in the namespace %q", AppName, clusterInfo.Namespace) + } + + secret.Data[MonSecretNameKey] = []byte(clusterInfo.MonitorSecret) + + logger.Infof("updating secret %q in the namespace %q", secret.Name, clusterInfo.Namespace) + _, err = clientset.CoreV1().Secrets(clusterInfo.Namespace).Update(clusterInfo.Context, secret, metav1.UpdateOptions{}) + if err != nil { + return errors.Wrapf(err, "failed to update secret %q in the namespace %q", secret.Name, clusterInfo.Namespace) + } + + return nil +} + // ParseMonEndpoints parses a flattened representation of mons and endpoints in the form // = and returns a list of Ceph mon configs. func ParseMonEndpoints(input string) map[string]*cephclient.MonInfo {