From 14490a075e885495baa50c6ff3d22b2880c605f8 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Wed, 6 Aug 2025 13:24:33 -0600 Subject: [PATCH] core: admin cephx key rotation Implement CephX key rotation for Rook's client.admin user. Admin user rotation is risky, so this has been tested extensively both in unit tests as well as by manually injecting failures during runtime. In testing, all failures were able to be recovered by the recovery routine. A mutex is also added to help ensure that two simultaneous admin key rotation processes cannot be running simultaneously for any given namespace. The mutex is tested in unit tests, and it was verified during runtime via manual testing. Signed-off-by: Blaine Gardner (cherry picked from commit 7c33186e72b2e4fb7e7358d5ec378560f0e2aa63) --- Documentation/CRDs/specification.md | 13 + .../Advanced/cephx-key-rotation.md | 3 + .../charts/rook-ceph/templates/resources.yaml | 22 + deploy/examples/crds.yaml | 22 + pkg/apis/ceph.rook.io/v1/types.go | 2 + .../ceph.rook.io/v1/zz_generated.deepcopy.go | 1 + pkg/daemon/ceph/client/command.go | 6 +- pkg/daemon/ceph/client/command_test.go | 14 +- pkg/daemon/ceph/client/config.go | 2 +- pkg/daemon/ceph/client/info.go | 4 + pkg/daemon/ceph/client/keyring.go | 17 +- pkg/operator/ceph/cluster/cephx.go | 422 ++++++ pkg/operator/ceph/cluster/cephx_test.go | 1149 +++++++++++++++++ pkg/operator/ceph/cluster/cluster.go | 24 + pkg/operator/ceph/cluster/mon/health.go | 4 +- pkg/operator/ceph/config/keyring/store.go | 19 + .../ceph/config/keyring/store_test.go | 5 + pkg/operator/ceph/controller/cluster_info.go | 3 + 18 files changed, 1713 insertions(+), 19 deletions(-) create mode 100644 pkg/operator/ceph/cluster/cephx.go create mode 100644 pkg/operator/ceph/cluster/cephx_test.go diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index 15b434f8c..51d1f2c85 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -5101,6 +5101,19 @@ CSI key rotation can affect existing PV connections, so take care when exercisin +admin
+ + +CephxStatus + + + + +

Admin shows the CephX key status for the client.admin key

+ + + + mon
diff --git a/Documentation/Storage-Configuration/Advanced/cephx-key-rotation.md b/Documentation/Storage-Configuration/Advanced/cephx-key-rotation.md index 198cdcbf0..8ad490c97 100644 --- a/Documentation/Storage-Configuration/Advanced/cephx-key-rotation.md +++ b/Documentation/Storage-Configuration/Advanced/cephx-key-rotation.md @@ -93,6 +93,9 @@ type managed for the cluster is listed. status: # ... cephx: + admin: + keyCephVersion: 19.2.3-0 + keyGeneration: 2 cephExporter: keyCephVersion: 19.2.3-0 keyGeneration: 2 diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 2a10c1b4d..5f900806e 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -5990,6 +5990,28 @@ spec: cephx: description: ClusterCephxStatus defines the cephx key rotation status of various daemons on the cephCluster resource properties: + admin: + description: Admin shows the CephX key status for the client.admin key + properties: + keyCephVersion: + description: |- + KeyCephVersion reports the Ceph version that created the current generation's keys. This is + same string format as reported by `CephCluster.status.version.version` to allow them to be + compared. E.g., `20.2.0-0`. + For all newly-created resources, this field set to the version of Ceph that created the key. + The special value "Uninitialized" indicates that keys are being created for the first time. + An empty string indicates that the version is unknown, as expected in brownfield deployments. + type: string + keyGeneration: + description: |- + KeyGeneration represents the CephX key generation for the last successful reconcile. + For all newly-created resources, this field is set to `1`. + When keys are rotated due to any rotation policy, the generation is incremented or updated to + the configured policy generation. + Generation `0` indicates that keys existed prior to the implementation of key tracking. + format: int32 + type: integer + type: object cephExporter: description: Ceph Exporter represents the cephx key rotation status of the ceph exporter daemon properties: diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 78294a502..993aa8261 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -5988,6 +5988,28 @@ spec: cephx: description: ClusterCephxStatus defines the cephx key rotation status of various daemons on the cephCluster resource properties: + admin: + description: Admin shows the CephX key status for the client.admin key + properties: + keyCephVersion: + description: |- + KeyCephVersion reports the Ceph version that created the current generation's keys. This is + same string format as reported by `CephCluster.status.version.version` to allow them to be + compared. E.g., `20.2.0-0`. + For all newly-created resources, this field set to the version of Ceph that created the key. + The special value "Uninitialized" indicates that keys are being created for the first time. + An empty string indicates that the version is unknown, as expected in brownfield deployments. + type: string + keyGeneration: + description: |- + KeyGeneration represents the CephX key generation for the last successful reconcile. + For all newly-created resources, this field is set to `1`. + When keys are rotated due to any rotation policy, the generation is incremented or updated to + the configured policy generation. + Generation `0` indicates that keys existed prior to the implementation of key tracking. + format: int32 + type: integer + type: object cephExporter: description: Ceph Exporter represents the cephx key rotation status of the ceph exporter daemon properties: diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 11e52608c..0f56153ea 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -748,6 +748,8 @@ type LocalCephxStatus struct { // ClusterCephxStatus defines the cephx key rotation status of various daemons on the cephCluster resource type ClusterCephxStatus struct { + // Admin shows the CephX key status for the client.admin key + Admin CephxStatus `json:"admin,omitempty"` // Mon represents the CephX key status of the Monitor daemons Mon CephxStatus `json:"mon,omitempty"` // Mgr represents the cephx key rotation status of the ceph manager daemon diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index baa5564e0..82e543732 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -2077,6 +2077,7 @@ func (in *ClusterCephxConfig) DeepCopy() *ClusterCephxConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterCephxStatus) DeepCopyInto(out *ClusterCephxStatus) { *out = *in + out.Admin = in.Admin out.Mon = in.Mon out.Mgr = in.Mgr out.OSD = in.OSD diff --git a/pkg/daemon/ceph/client/command.go b/pkg/daemon/ceph/client/command.go index ad52d6292..a9f64cb41 100644 --- a/pkg/daemon/ceph/client/command.go +++ b/pkg/daemon/ceph/client/command.go @@ -95,11 +95,15 @@ func FinalizeCephCommandArgs(command string, clusterInfo *ClusterInfo, args []st default: // Append the standard flags for config and keyring keyringFile := fmt.Sprintf("%s.keyring", clusterInfo.CephCred.Username) + keyringPath := path.Join(configDir, clusterInfo.Namespace, keyringFile) + if clusterInfo.KeyringFileOverride != "" { + keyringPath = clusterInfo.KeyringFileOverride + } configArgs = []string{ fmt.Sprintf("--cluster=%s", clusterInfo.Namespace), fmt.Sprintf("--conf=%s", cephConfPath), fmt.Sprintf("--name=%s", clusterInfo.CephCred.Username), - fmt.Sprintf("--keyring=%s", path.Join(configDir, clusterInfo.Namespace, keyringFile)), + fmt.Sprintf("--keyring=%s", keyringPath), } } diff --git a/pkg/daemon/ceph/client/command_test.go b/pkg/daemon/ceph/client/command_test.go index 848825f17..86d36d45b 100644 --- a/pkg/daemon/ceph/client/command_test.go +++ b/pkg/daemon/ceph/client/command_test.go @@ -47,9 +47,19 @@ func TestFinalizeCephCommandArgs(t *testing.T) { } clusterInfo := AdminTestClusterInfo("rook") - cmd, args := FinalizeCephCommandArgs(expectedCommand, clusterInfo, args, configDir) + cmd, gotArgs := FinalizeCephCommandArgs(expectedCommand, clusterInfo, args, configDir) assert.Exactly(t, expectedCommand, cmd) - assert.Exactly(t, expectedArgs, args) + assert.Exactly(t, expectedArgs, gotArgs) + + t.Run("keyring override", func(t *testing.T) { + // run the same test as above, but set the keyring override + clusterInfo.KeyringFileOverride = "/some/random/file/path.extension" + // keyring arg should change but nothing else + expectedArgs[len(expectedArgs)-1] = "--keyring=/some/random/file/path.extension" + cmd, args := FinalizeCephCommandArgs(expectedCommand, clusterInfo, args, configDir) + assert.Exactly(t, expectedCommand, cmd) + assert.Exactly(t, expectedArgs, args) + }) } func TestFinalizeRadosGWAdminCommandArgs(t *testing.T) { diff --git a/pkg/daemon/ceph/client/config.go b/pkg/daemon/ceph/client/config.go index e5d950bd7..a5ab924f2 100644 --- a/pkg/daemon/ceph/client/config.go +++ b/pkg/daemon/ceph/client/config.go @@ -93,7 +93,7 @@ func GenerateConnectionConfig(context *clusterd.Context, cluster *ClusterInfo) ( func GenerateConnectionConfigWithSettings(context *clusterd.Context, clusterInfo *ClusterInfo, settings *CephConfig) (string, error) { root := path.Join(context.ConfigDir, clusterInfo.Namespace) keyringPath := path.Join(root, fmt.Sprintf("%s.keyring", clusterInfo.CephCred.Username)) - err := writeKeyring(CephKeyring(clusterInfo.CephCred), keyringPath) + err := WriteKeyring(keyringPath, CephKeyring(clusterInfo.CephCred)) if err != nil { return "", errors.Wrapf(err, "failed to write keyring %q to %s", clusterInfo.CephCred.Username, root) } diff --git a/pkg/daemon/ceph/client/info.go b/pkg/daemon/ceph/client/info.go index 79d273471..bbff3ddc7 100644 --- a/pkg/daemon/ceph/client/info.go +++ b/pkg/daemon/ceph/client/info.go @@ -60,6 +60,10 @@ type ClusterInfo struct { // Whereas if passed through clusterInfo, we don't have that problem since clusterInfo is // re-hydrated when a context is cancelled. Context context.Context + + // KeyringFileOverride, if set, configures a ceph client executor to use a specific keyring file + // instead of the default. + KeyringFileOverride string } func (c *ClusterInfo) AllMonitors() map[string]*MonInfo { diff --git a/pkg/daemon/ceph/client/keyring.go b/pkg/daemon/ceph/client/keyring.go index 476d816a0..2a7cada8f 100644 --- a/pkg/daemon/ceph/client/keyring.go +++ b/pkg/daemon/ceph/client/keyring.go @@ -53,15 +53,6 @@ func CephKeyring(cred CephCred) string { return fmt.Sprintf(UserKeyringTemplate, cred.Username, cred.Secret) } -// WriteKeyring calls the generate contents function with auth key as an argument then saves the -// output of the generateContents function to disk at the keyring path -// TODO: Kludgey; can keyring files be generated w/ go-ini package or using the '-o' option to -// 'ceph auth get-or-create ...'? -func WriteKeyring(keyringPath, authKey string, generateContents func(string) string) error { - contents := generateContents(authKey) - return writeKeyring(contents, keyringPath) -} - // CreateKeyring creates a keyring for access to the cluster with the desired set of privileges // and writes it to disk at the keyring path func CreateKeyring(context *clusterd.Context, clusterInfo *ClusterInfo, username, keyringPath string, access []string, generateContents func(string) string) error { @@ -81,12 +72,12 @@ func CreateKeyring(context *clusterd.Context, clusterInfo *ClusterInfo, username return errors.Wrapf(err, "failed to get or create auth key for %s", username) } - return WriteKeyring(keyringPath, key, generateContents) + contents := generateContents(key) + return WriteKeyring(keyringPath, contents) } -// writes the keyring to disk -// TODO: Write keyring only to the default ceph config location since we are in a container -func writeKeyring(keyring, keyringPath string) error { +// Writes the keyring to disk, creating parent dirs as necessary +func WriteKeyring(keyringPath, keyring string) error { // save the keyring to the given path if err := os.MkdirAll(filepath.Dir(keyringPath), 0o700); err != nil { return errors.Wrapf(err, "failed to create keyring directory for %s", keyringPath) diff --git a/pkg/operator/ceph/cluster/cephx.go b/pkg/operator/ceph/cluster/cephx.go new file mode 100644 index 000000000..6b6de386c --- /dev/null +++ b/pkg/operator/ceph/cluster/cephx.go @@ -0,0 +1,422 @@ +/* +Copyright 2025 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/clusterd" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/cluster/mon" + "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/config/keyring" + "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/ceph/reporting" + "github.com/rook/rook/pkg/operator/k8sutil" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/util/retry" +) + +const ( + //#nosec G101 -- This is only a secret name prefix + adminRotatorSecretPrefix = "rook-ceph-admin-rotator" + //#nosec G101 -- This is only the admin client name + adminRotatorUsername = "client.admin-rotator" +) + +// note: must match adminKeyringTemplate +var ( + adminKeyAccessCaps = []string{"mds", "allow *", "mon", "allow *", "osd", "allow *", "mgr", "allow *"} + + // ReloadManager sends SIGHUP to the main process. Allow this to be stubbed for unit tests. + reloadManagerFunc = controller.ReloadManager + + // admin key rotation ends with a rook restart, so return a reconcile error after to help + // prevent rook from continuing reconcile when restart is imminent + errSuccessfulAdminKeyRotation = fmt.Errorf("successful admin cephx key rotation requires the current cluster reconcile to restart") + + // Prevent multiple admin rotation operations from running simultaneously. This is important + // because the admin rotation routine does not stop as soon as the parent context is canceled. + // in the worst case, it could still be running while a CephCluster re-reconcile gets back + // around to admin key rotation again. + adminRotationMutex = sync.Mutex{} + adminRotationInProgress = map[string]struct{}{} +) + +// claimAdminRotationLock tries to claim a lock on admin key rotation for a cluster namespace +// returns error if the lock could not be granted. +// Because (*Mutex).Lock() blocks while trying to get a lock, we could end up stalling Rook +// reconciles if we rely solely on the lock itself. To prevent stalls, use a map 'set' to indicate +// whether rotation is in progress, and only lock access to the set. When a rotation is locked, +// return an error so that the reconcile can be requeued to try again later. +func claimAdminRotationLock(namespace string) error { + adminRotationMutex.Lock() + defer adminRotationMutex.Unlock() + if _, ok := adminRotationInProgress[namespace]; ok { + return fmt.Errorf("failed to claim admin rotation lock for cluster in namespace %q", namespace) + } + logger.Debugf("claimed admin rotation lock for cluster in namespace %q", namespace) + adminRotationInProgress[namespace] = struct{}{} + return nil +} + +// releaseAdminRotationLock releases a claimed lock for a cluster namespace +func releaseAdminRotationLock(namespace string) { + adminRotationMutex.Lock() + defer adminRotationMutex.Unlock() + delete(adminRotationInProgress, namespace) +} + +// turn a client name and its caps into keyring file contents +// example: client.my-user, []string{"mon", "allow *"} becomes: +// +// [client.my-user] +// caps mon = "allow *" +func genKeyring(clientName, authKey string, clientCaps []string) (string, error) { + // DO NOT LOG authKey + if authKey == "" { + return "", fmt.Errorf("cannot generate %q keyring with empty cephx key", clientName) + } + o := fmt.Sprintf("[%s]\n", clientName) + o += fmt.Sprintf(" key = %s\n", authKey) + if len(clientCaps)%2 != 0 { + return "", fmt.Errorf("cannot generate %q keyring for caps list %v with uneven number of items", clientName, clientCaps) + } + for i := 0; i < len(clientCaps); i += 2 { + o += fmt.Sprintf(" caps %s = %q\n", clientCaps[i], clientCaps[i+1]) + } + return o, nil +} + +// main routine for rotating the client.admin key +// should be called after mon daemons are upgraded to current ceph image in the cephcluster spec +func rotateAdminCephxKey( + clusterdCtx *clusterd.Context, + clusterInfo *cephclient.ClusterInfo, + ownerInfo *k8sutil.OwnerInfo, + cephCluster *cephv1.CephCluster, +) error { + if clusterInfo.CephCred.Username != cephclient.AdminUsername { + // this shouldn't happen during normal runtime - this could indicate external mode cluster + logger.Infof("cannot rotate admin cephx key with non-rotatable username %q for cluster in namespace %q", clusterInfo.CephCred.Username, clusterInfo.Namespace) + return nil + } + + desiredCephVersion := clusterInfo.CephVersion // TODO: update this when/if WithCephVersionUpdate is implemented + shouldRotate, err := keyring.ShouldRotateCephxKeys( + cephCluster.Spec.Security.CephX.Daemon, clusterInfo.CephVersion, desiredCephVersion, cephCluster.Status.Cephx.Admin) + if err != nil { + return errors.Wrap(err, "failed to determine if admin cephx key should be rotated") + } + if !shouldRotate { + logger.Debugf("not rotating admin cephx key for cluster in namespace %q", clusterInfo.Namespace) + // no rotation, but still update cephx status - critical for greenfield Uninitialized clusters + err := updateCephClusterAdminCephxStatus(clusterdCtx, clusterInfo, false) + if err != nil { + return errors.Wrap(err, "failed to update admin cephx key status after not rotating") + } + return nil + } + + if err := claimAdminRotationLock(clusterInfo.Namespace); err != nil { + return err + } + defer releaseAdminRotationLock(clusterInfo.Namespace) + + logger.Infof("beginning admin cephx key rotation for cluster in namespace %q", clusterInfo.Namespace) + + // As an optimization, use a (shallow) copy of clusterInfo with a context that won't be canceled + // during normal runtime. This disallows the rotation process from being interrupted partway + // through in the event of a CephCluster spec change, which can happen for any number of reasons + // during normal runtime. While this rotation is designed to be able to be recovered if + // interrupted, it is still safest to allow the rotation process to finish in full. + clusterInfoShallowCopy := *clusterInfo + clusterInfoCopy := &clusterInfoShallowCopy + clusterInfoCopy.Context = context.Background() + + s := keyring.GetSecretStore(clusterdCtx, clusterInfoCopy, ownerInfo) + + // generate client.admin-rotator admin user + // if client.admin rotation fails or is blocked, the client.admin-rotator user can be used to + // recover from bugs/blockages in rotation of primary client.admin key + rotatorKey, err := s.GenerateKey(adminRotatorUsername, adminKeyAccessCaps) + if err != nil { + return errors.Wrapf(err, "failed to generate cephx key for admin rotator %q", adminRotatorUsername) + } + + // store client.admin-rotator in secret + rotatorKeyring, err := genKeyring(adminRotatorUsername, rotatorKey, adminKeyAccessCaps) + if err != nil { + return errors.Wrapf(err, "failed to generate admin rotator cephx keyring file") + } + _, err = s.CreateOrUpdate(adminRotatorSecretPrefix, rotatorKeyring) + if err != nil { + return errors.Wrapf(err, "failed to store admin rotator cephx keyring to secret") + } + + // keep keyring files used for admin key rotation in a subdir intended to be temporary + // stored in rook-ceph-operator emptyDir + // do not clean up in defer() so that any issues can be debugged with shell into operator pod + tmpDir := adminRotationTmpDir(clusterdCtx, clusterInfoCopy) + + return rotateAdminCephxKeyUsingRotator(clusterdCtx, clusterInfoCopy, ownerInfo, tmpDir, rotatorKeyring) +} + +// routine for recovering from client.admin key rotation that was interrupted in a prior reconcile +// because the currently-stored client.admin key could be incorrect, this should be called very +// early in the cephcluster reconcile, before any ceph commands are run: failures running commands +// could block the reconcile from recovering admin key rotation +func recoverPriorAdminCephxKeyRotation( + clusterdCtx *clusterd.Context, + clusterInfo *cephclient.ClusterInfo, + ownerInfo *k8sutil.OwnerInfo, + clusterNamespace string, +) error { + if clusterInfo == nil { + // new cluster hasn't been deployed yet. there should be no way for recovery to be needed + logger.Debugf("no admin cephx key recovery possible for cluster with empty clusterInfo in namespace %q", clusterNamespace) + return nil + } + + s := keyring.GetSecretStore(clusterdCtx, clusterInfo, ownerInfo) + + rotatorKeyring, err := s.GetKeyringFromSecret(adminRotatorSecretPrefix) + if err != nil { + if kerrors.IsNotFound(err) { + logger.Debugf("no admin cephx key recovery needed for cluster in namespace %q", clusterInfo.Namespace) + return nil + } + return errors.Wrap(err, "failed to get admin rotator cephx key secret to check if admin cephx key rotation recovery is needed") + } + + if err := claimAdminRotationLock(clusterNamespace); err != nil { + return err + } + defer releaseAdminRotationLock(clusterNamespace) + + logger.Infof("recovering from interrupted admin cephx key rotation for cluster in namespace %q", clusterInfo.Namespace) + + // As an optimization, use a (shallow) copy of clusterInfo with a context that won't be canceled + // during normal runtime. This disallows the recovery process from being interrupted partway + // through in the event of a CephCluster spec change, which can happen for any number of reasons + // during normal runtime. While even the recovery is designed to be able to be recovered if + // interrupted, it is still safest to allow the rotation process to recover in full. + clusterInfoShallowCopy := *clusterInfo + clusterInfoCopy := &clusterInfoShallowCopy + clusterInfoCopy.Context = context.Background() + + // if the operator pod has restarted and recovery is needed, the ceph.conf file will not be + // present in the rook temp dir. Write/update the cluster config to handle this case + err = mon.WriteConnectionConfig(clusterdCtx, clusterInfoCopy) + if err != nil { + return errors.Wrap(err, "failed to write/update cluster config in ceph.conf before recovering admin cephx key rotation") + } + + // keep keyring files used for admin key rotation in a subdir intended to be temporary + // stored in rook-ceph-operator emptyDir + // do not clean up in defer() so that any issues can be debugged with shell into operator pod + tmpDir := adminRotationTmpDir(clusterdCtx, clusterInfoCopy) + + authList, err := cephclient.AuthList(clusterdCtx, clusterInfoCopy) + if err != nil { + logger.Debugf("primary admin user failed to list ceph auth - this is an expected condition when recovering from admin cephx key rotation") + } else if !entityExistsInAuthList(adminRotatorUsername, authList) { // err == nil + // corner case: if the primary client.admin works AND client.admin-rotator is not present in + // the `auth ls` output, it means rotation succeeded and was interrupted during cleanup. + // client.auth-rotator is no longer present/valid, so it can't be used to take any actions + return finalizeAdminKeyRotation(clusterdCtx, clusterInfoCopy, ownerInfo, tmpDir) + } + + // a prior rotation failed, but we don't know exactly where in the process it failed. we have to + // re-run the main rotation routine from the beginning to ensure we don't skip rotation + return rotateAdminCephxKeyUsingRotator(clusterdCtx, clusterInfoCopy, ownerInfo, tmpDir, rotatorKeyring) +} + +func entityExistsInAuthList(entity string, authList cephclient.AuthListOutput) bool { + for _, a := range authList.AuthDump { + if a.Entity == entity { + return true + } + } + return false +} + +// helper routine that rotate client.admin cephx key including cleanup and cephx status update +// uses the provided client.admin-rotator keyring to rotate client.admin +func rotateAdminCephxKeyUsingRotator( + clusterdCtx *clusterd.Context, + clusterInfo *cephclient.ClusterInfo, + ownerInfo *k8sutil.OwnerInfo, + tmpDir, rotatorKeyring string, +) error { + // write admin rotator keyring file to tempdir so it can be used for rotation + rotatorKeyringTmpfile := filepath.Join(tmpDir, adminRotatorUsername+".keyring") + logger.Infof("temporarily storing admin rotator keyring at %q for cluster in namespace %q", rotatorKeyringTmpfile, clusterInfo.Namespace) + err := cephclient.WriteKeyring(rotatorKeyringTmpfile, rotatorKeyring) + if err != nil { + return errors.Wrapf(err, "failed to write admin rotator keyring to %q", rotatorKeyringTmpfile) + } + + // make copy of ClusterInfo struct that uses admin-rotator user/key for ceph commands + rotatorInfo := minimalCopyClusterInfo(clusterInfo, adminRotatorUsername, rotatorKeyringTmpfile) + + // as `client.admin-rotator`: run `ceph auth ls` to ensure it has permissions + _, err = cephclient.AuthList(clusterdCtx, rotatorInfo) + if err != nil { + return errors.Wrapf(err, "failed to list ceph auth using admin rotator client") + } + + // as `client.admin-rotator`: run `ceph auth rotate client.admin` + logger.Infof("admin cephx key will be rotated for cluster in namespace %q. rook will restart afterwards. some reconciles and health checks may fail in between - this is normal", clusterInfo.Namespace) + newAdminKey, err := cephclient.AuthRotate(clusterdCtx, rotatorInfo, cephclient.AdminUsername) + if err != nil { + return errors.Wrapf(err, "failed to rotate admin key using admin rotator client") + } + newAdminKeyring := cephclient.CephKeyring(cephclient.CephCred{Username: cephclient.AdminUsername, Secret: newAdminKey}) + + // write new admin keyring to tempdir so it can be verified before it's stored permanently + newAdminKeyringTmpfile := filepath.Join(tmpDir, cephclient.AdminUsername+".keyring") + logger.Infof("temporarily storing rotated admin cephx keyring at %q for cluster in namespace %q", newAdminKeyringTmpfile, clusterInfo.Namespace) + err = cephclient.WriteKeyring(newAdminKeyringTmpfile, newAdminKeyring) + if err != nil { + return errors.Wrapf(err, "failed to write admin rotator keyring to %q", newAdminKeyringTmpfile) + } + + // `client.admin` run `ceph auth ls` to ensure it has permissions + // make copy of ClusterInfo struct that uses rotated admin's temporary keyring for ceph commands + newAdminInfo := minimalCopyClusterInfo(clusterInfo, cephclient.AdminUsername, newAdminKeyringTmpfile) + _, err = cephclient.AuthList(clusterdCtx, newAdminInfo) + if err != nil { + return errors.Wrapf(err, "admin rotator failed to list ceph auth") + } + + // now that the new key is verified working, update clusterInfo with new key + clusterInfo.CephCred.Secret = newAdminKey + + // update the on-disk ceph config file with new `client.admin` key (now in clusterInfo) + err = mon.WriteConnectionConfig(clusterdCtx, clusterInfo) + if err != nil { + return errors.Wrapf(err, "failed to write latest cluster config to disk") + } + + // update the `rook-ceph-mon` secret with new `client.admin` key (now in clusterInfo) + err = controller.UpdateClusterAccessSecret(clusterdCtx.Clientset, clusterInfo) + if err != nil { + return errors.Wrapf(err, "failed to update cluster access secret with latest %q admin cephx key", cephclient.AdminUsername) + } + + // as `client.admin`: run `ceph auth rm client.admin-rotator` + err = cephclient.AuthDelete(clusterdCtx, clusterInfo, adminRotatorUsername) + if err != nil { + return errors.Wrapf(err, "failed to delete cephx auth for admin rotator %q", adminRotatorUsername) + } + + return finalizeAdminKeyRotation(clusterdCtx, clusterInfo, ownerInfo, tmpDir) +} + +// helper routine that performs final cleanup steps for admin key rotation. should be used +// immediately after `ceph auth rm client.admin-rotator`. updates cephx status when finished +func finalizeAdminKeyRotation( + clusterdCtx *clusterd.Context, + clusterInfo *cephclient.ClusterInfo, + ownerInfo *k8sutil.OwnerInfo, + tmpDir string, +) error { + // clean up tmpDir now that it's no longer useful for recovery + if err := os.RemoveAll(tmpDir); err != nil { + // most likely, operator pod got rescheduled to a different node where dir isn't present + logger.Infof("non-critical failure removing admin cephx key rotation temp dir %q. %v", tmpDir, err) + } + + // delete the `rook-ceph-admin-rotator-keyring` secret (store.Delete) + s := keyring.GetSecretStore(clusterdCtx, clusterInfo, ownerInfo) + err := s.Delete(adminRotatorSecretPrefix) + if err != nil { + return errors.Wrapf(err, "failed to delete admin rotator cephx keyring secret") + } + + // during recovery, clusterInfo doesn't know the ceph version, so set it if needed + if clusterInfo.CephVersion.Major == 0 { // Major==0 should be good enough for detecting unknown ver + ver, err := cephclient.LeastUptodateDaemonVersion(clusterdCtx, clusterInfo, config.MonType) + if err != nil { + // this should be rare, and failure here doesn't need to block finalization of admin key + // rotation. status will be missing ceph version, but cluster can continue to reconcile + logger.Errorf("non-critical failure to determine missing ceph version of cluster in namespace %q for admin cephx key status", clusterInfo.Namespace) + } else { + clusterInfo.CephVersion = ver + } + } + + // update `CephCluster.status.cephx.admin` with updated CephX status + // rotation is a given if this code line was reached + if err := updateCephClusterAdminCephxStatus(clusterdCtx, clusterInfo, true); err != nil { + return err + } + + // restart Rook to force health checkers to restart with latest clusterInfo + // do this even during rotation recovery since the health checkers could still be running + logger.Infof("restarting rook operator after successful admin cephx key rotation for cluster in namespace %q", clusterInfo.Namespace) + reloadManagerFunc() + + return errSuccessfulAdminKeyRotation +} + +// helper routine that updates cephcluster 'admin' cephx status +func updateCephClusterAdminCephxStatus(clusterdCtx *clusterd.Context, clusterInfo *cephclient.ClusterInfo, didRotate bool) error { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + cluster := &cephv1.CephCluster{} + if err := clusterdCtx.Client.Get(clusterInfo.Context, clusterInfo.NamespacedName(), cluster); err != nil { + return errors.Wrap(err, "failed to get CephCluster to update the admin key cephx status") + } + cephx := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, clusterInfo.CephVersion, cluster.Status.Cephx.Admin) + cluster.Status.Cephx.Admin = cephx + logger.Debugf("updating admin key cephx status to %+v", cephx) + if err := reporting.UpdateStatus(clusterdCtx.Client, cluster); err != nil { + return errors.Wrap(err, "failed to update admin key cephx status") + } + + return nil + }) + return err +} + +func adminRotationTmpDir(clusterdCtx *clusterd.Context, clusterInfo *cephclient.ClusterInfo) string { + return filepath.Join(clusterdCtx.ConfigDir, clusterInfo.Namespace, "admin-rotate") +} + +// make a minimal copy of the clusterInfo that tells called cephclient functions to use the given +// username and keyring path instead the defaults +func minimalCopyClusterInfo(clusterInfo *cephclient.ClusterInfo, username, keyringFilePath string) *cephclient.ClusterInfo { + c := &cephclient.ClusterInfo{ + Context: clusterInfo.Context, + Namespace: clusterInfo.Namespace, + CephCred: cephclient.CephCred{ + Username: username, + // Secret not used by cephclient executor + }, + CephVersion: clusterInfo.CephVersion, + KeyringFileOverride: keyringFilePath, + } + c.SetName(clusterInfo.NamespacedName().Name) + return c +} diff --git a/pkg/operator/ceph/cluster/cephx_test.go b/pkg/operator/ceph/cluster/cephx_test.go new file mode 100644 index 000000000..dddb39b87 --- /dev/null +++ b/pkg/operator/ceph/cluster/cephx_test.go @@ -0,0 +1,1149 @@ +/* +Copyright 2025 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/coreos/pkg/capnslog" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/client/clientset/versioned/scheme" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/rook/rook/pkg/operator/k8sutil" + testop "github.com/rook/rook/pkg/operator/test" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func Test_genKeyring(t *testing.T) { + twoCapsKeyring := `[client.them] + key = themkey== + caps any = "thing" +` + adminRotatorKeyring := `[client.admin-rotator] + key = adminrotatorkey== + caps mds = "allow *" + caps mon = "allow *" + caps osd = "allow *" + caps mgr = "allow *" +` + + tests := []struct { + name string + clientName string + clientKey string + clientCaps []string + wantErr bool + want string + }{ + {"no caps", "client.me", "mekey==", []string{}, false, "[client.me]\n key = mekey==\n"}, + {"no caps, no key", "client.me", "", []string{}, true, ""}, + {"one cap", "client.you", "youkey==", []string{"bad"}, true, ""}, + {"two caps", "client.them", "themkey==", []string{"any", "thing"}, false, twoCapsKeyring}, + {"three caps", "client.batman", "batmankey==", []string{"some", "thing", "else"}, true, ""}, + {"admin rotator caps", "client.admin-rotator", "adminrotatorkey==", adminKeyAccessCaps, false, adminRotatorKeyring}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := genKeyring(tt.clientName, tt.clientKey, tt.clientCaps) + if (err != nil) != tt.wantErr { + t.Errorf("genKeyring() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("genKeyring() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_admin_key_rotation(t *testing.T) { + logger.SetLevel(capnslog.DEBUG) // TESTING TESTING TESTING + + reloadManagerCalled := false + reloadManagerFunc = func() { + reloadManagerCalled = true + } + + // this feature requires commands to execute in a specific order and often with specific arg(s) + // it also requires commands to not be skipped + // set up an executor that allows tests to expect commands in a certain order + type expect struct { + cargs []string + withArgs []string + returnFunc func(cmd string, args ...string) (string, error) + } + numCalls := 0 + expectList := []expect{} + + mockExecutor := &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(cmd string, args ...string) (string, error) { + cargs := append([]string{cmd}, args...) // cmd+args in one slice for easy comparison + t.Logf("mock command: %v", cargs) + + // get current command expectations + if len(expectList) <= numCalls { + e := fmt.Errorf("execution iteration=%d: no expectation set for called %v", numCalls, cargs) + t.Error(e) + return "", e + } + currentExpect := expectList[numCalls] + numCalls++ + + // assert current command matches expectations + if !isCommand(cargs, currentExpect.cargs...) { + e := fmt.Errorf("execution iteration=%d: expected command %v does not match called %v", numCalls, currentExpect.cargs, cargs) + t.Error(e) + return "", e + } + if len(currentExpect.withArgs) > 0 { + for _, arg := range currentExpect.withArgs { + if !hasArg(cargs, arg) { + e := fmt.Errorf("execution iteration=%d: expected arg %q missing in called %v", numCalls, arg, cargs) + t.Error(e) + return "", e + } + } + } + + // expectations met, call return func with args + return currentExpect.returnFunc(cmd, args...) + }, + } + + tmpCfgDir, err := os.MkdirTemp("", "") + t.Logf("temp dir: %q", tmpCfgDir) + assert.NoError(t, err) + defer func() { os.RemoveAll(tmpCfgDir) }() + + ns := "ns" + clusterName := "my-cluster" + + newTest := func() (*clusterd.Context, *client.ClusterInfo, *k8sutil.OwnerInfo, *cephv1.CephCluster) { + reloadManagerCalled = false + numCalls = 0 + expectList = []expect{} // clear expect list + + clusterInfo := &client.ClusterInfo{ + Context: context.TODO(), + FSID: "00000000-0000-0000-0000-000000000000", + Namespace: ns, + CephVersion: version.CephVersion{Major: 20, Minor: 3, Extra: 0}, + MonitorSecret: "MONSECRET/SHOULDNOTCHANGE=", + CephCred: client.CephCred{ + Username: "client.admin", + Secret: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + }, + OwnerInfo: &k8sutil.OwnerInfo{}, + } + clusterInfo.SetName(clusterName) + + cephCluster := &cephv1.CephCluster{ // create via controller-runtime client + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: clusterName, + }, + Spec: cephv1.ClusterSpec{ + Security: cephv1.ClusterSecuritySpec{ + CephX: cephv1.ClusterCephxConfig{ + Daemon: cephv1.CephxConfig{ + KeyRotationPolicy: "KeyGeneration", + KeyGeneration: 2, + }, + }, + }, + }, + Status: cephv1.ClusterStatus{ + Cephx: cephv1.ClusterCephxStatus{ + Admin: cephv1.CephxStatus{ + KeyGeneration: 1, + KeyCephVersion: "19.2.3-0", + }, + }, + }, + } + + monSecret := &corev1.Secret{ // create via clientset + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns", + Name: "rook-ceph-mon", + }, + Data: map[string][]byte{ + "ceph-secret": []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="), // before rotation key + "ceph-username": []byte("client.admin"), + "fsid": []byte("00000000-0000-0000-0000-000000000000"), + "mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="), + }, + } + + clientset := testop.New(t, 3) + _, err := clientset.CoreV1().Secrets(ns).Create(context.TODO(), monSecret, metav1.CreateOptions{}) + if err != nil { + panic(err) // test setup failed + } + + s := scheme.Scheme + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}) + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(cephCluster).Build() + + clusterdCtx := &clusterd.Context{ + Clientset: clientset, + Client: cl, + Executor: mockExecutor, + ConfigDir: tmpCfgDir, + } + + return clusterdCtx, clusterInfo, clusterInfo.OwnerInfo, cephCluster + } + + // args that the default clusterInfo should use + defaultClientArgs := []string{"--name=client.admin", "--keyring=" + tmpCfgDir + "/ns/client.admin.keyring"} + + // args that the temporary admin-rotator user should use + tmpRotatorClientArgs := []string{"--name=client.admin-rotator", "--keyring=" + tmpCfgDir + "/ns/admin-rotate/client.admin-rotator.keyring"} + + // args that the new admin client should use temporarily to verify it works before final storage + tmpAdminClientArgs := []string{"--name=client.admin", "--keyring=" + tmpCfgDir + "/ns/admin-rotate/client.admin.keyring"} + + t.Run("happy path rotation", func(t *testing.T) { + clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest() + + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authCreateAdminRotatorOutput, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authLsOutputBeforeRotation, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "rotate", "client.admin"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authRotateAdminOutput, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpAdminClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authLsOutputAfterRotatorDeletion, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "del", "client.admin-rotator"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return "", nil + }, + }) + + err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster) + assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation) + assert.Equal(t, len(expectList), numCalls) // any expected calls not called? + assert.True(t, reloadManagerCalled) + + cluster := cephv1.CephCluster{} + err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster) + assert.NoError(t, err) + assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated + assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated + + monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{}) + assert.NoError(t, err) + wantMonSecretData := map[string][]byte{ + "mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="), + "fsid": []byte("00000000-0000-0000-0000-000000000000"), + "ceph-username": []byte("client.admin"), + "ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // changed + } + assert.Equal(t, wantMonSecretData, monSec.Data) + + _, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{}) + assert.ErrorContains(t, err, "not found") // rotator keyring not persisted + + // after successful rotation, recovery should not occur + err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns) + assert.NoError(t, err) + assert.Equal(t, len(expectList), numCalls) // same number calls from before + + err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster) + assert.NoError(t, err) + assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged + assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged + + monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{}) + assert.NoError(t, err) + assert.Equal(t, wantMonSecretData, monSec.Data) // unchanged + + _, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{}) + assert.ErrorContains(t, err, "not found") // rotator keyring not persisted + }) + + t.Run("fail create admin-rotator", func(t *testing.T) { + clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest() + + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return "", fmt.Errorf("induced error") + }, + }) + expectList = append(expectList, expect{ + // auth generation tries to modify caps if auth get-or-create-key fails + cargs: []string{"ceph", "auth", "caps", "client.admin-rotator"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return "", fmt.Errorf("induced error") + }, + }) + + err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster) + assert.ErrorContains(t, err, "induced error") + assert.Equal(t, len(expectList), numCalls) // any expected calls not called? + assert.False(t, reloadManagerCalled) + + cluster := cephv1.CephCluster{} + err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster) + assert.NoError(t, err) + assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged + assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged + + monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{}) + assert.NoError(t, err) + wantMonSecretData := map[string][]byte{ + "mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="), + "fsid": []byte("00000000-0000-0000-0000-000000000000"), + "ceph-username": []byte("client.admin"), + "ceph-secret": []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="), // unchanged + } + assert.Equal(t, wantMonSecretData, monSec.Data) + + // nothing to recover before admin-rotator created and stored in secret + err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns) + assert.NoError(t, err) + assert.Equal(t, len(expectList), numCalls) // same number calls from before + + monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{}) + assert.NoError(t, err) + assert.Equal(t, wantMonSecretData, monSec.Data) // unchanged + + _, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{}) + assert.ErrorContains(t, err, "not found") // rotator keyring not persisted + }) + + t.Run("fail ls as admin rotator", func(t *testing.T) { + clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest() + + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authCreateAdminRotatorOutput, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return "", fmt.Errorf("induced error") + }, + }) + + err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster) + assert.ErrorContains(t, err, "induced error") + assert.Equal(t, len(expectList), numCalls) // any expected calls not called? + assert.False(t, reloadManagerCalled) + + cluster := cephv1.CephCluster{} + err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster) + assert.NoError(t, err) + assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged + assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged + + monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{}) + assert.NoError(t, err) + wantMonSecretData := map[string][]byte{ + "mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="), + "fsid": []byte("00000000-0000-0000-0000-000000000000"), + "ceph-username": []byte("client.admin"), + "ceph-secret": []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="), // unchanged + } + assert.Equal(t, wantMonSecretData, monSec.Data) + + _, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{}) + assert.NoError(t, err) // rotator secret now persisted + + // expect recovery to proceed + numCalls = 0 + expectList = []expect{} // clear expect list + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + // client.admin can still ls, and admin-rotator is present + return authLsOutputBeforeRotation, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authLsOutputBeforeRotation, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "rotate", "client.admin"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authRotateAdminOutput, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpAdminClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authLsOutputAfterRotatorDeletion, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "del", "client.admin-rotator"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return "", nil + }, + }) + + err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns) + assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation) + assert.Equal(t, len(expectList), numCalls) // any expected calls not called? + assert.True(t, reloadManagerCalled) + + cluster = cephv1.CephCluster{} + err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster) + assert.NoError(t, err) + assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated + assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated + + monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{}) + assert.NoError(t, err) + wantMonSecretData = map[string][]byte{ + "mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="), + "fsid": []byte("00000000-0000-0000-0000-000000000000"), + "ceph-username": []byte("client.admin"), + "ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // changed + } + assert.Equal(t, wantMonSecretData, monSec.Data) + + _, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{}) + assert.ErrorContains(t, err, "not found") // rotator keyring not persisted + + t.Run("recovery doesn't reoccur after successful recovery", func(t *testing.T) { + // only need to test this in one of the unit tests + err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns) + assert.NoError(t, err) + assert.Equal(t, len(expectList), numCalls) // same number calls from before + + err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster) + assert.NoError(t, err) + assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged + assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged + + monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{}) + assert.NoError(t, err) + assert.Equal(t, wantMonSecretData, monSec.Data) // unchanged + + _, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{}) + assert.ErrorContains(t, err, "not found") // rotator keyring not persisted + }) + }) + + t.Run("fail auth rotate", func(t *testing.T) { + clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest() + + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authCreateAdminRotatorOutput, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authLsOutputBeforeRotation, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "rotate", "client.admin"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return "", fmt.Errorf("induced error") + }, + }) + + err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster) + assert.ErrorContains(t, err, "induced error") + assert.Equal(t, len(expectList), numCalls) // any expected calls not called? + assert.False(t, reloadManagerCalled) + + cluster := cephv1.CephCluster{} + err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster) + assert.NoError(t, err) + assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged + assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged + + monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{}) + assert.NoError(t, err) + wantMonSecretData := map[string][]byte{ + "mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="), + "fsid": []byte("00000000-0000-0000-0000-000000000000"), + "ceph-username": []byte("client.admin"), + "ceph-secret": []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="), // unchanged + } + assert.Equal(t, wantMonSecretData, monSec.Data) + + _, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{}) + assert.NoError(t, err) // rotator secret now persisted + + // expect recovery to proceed + numCalls = 0 + expectList = []expect{} // clear expect list + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + // client.admin can still ls, and admin-rotator is present + return authLsOutputBeforeRotation, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authLsOutputBeforeRotation, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "rotate", "client.admin"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authRotateAdminOutput, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpAdminClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authLsOutputAfterRotatorDeletion, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "del", "client.admin-rotator"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return "", nil + }, + }) + + err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns) + assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation) + assert.Equal(t, len(expectList), numCalls) // any expected calls not called? + assert.True(t, reloadManagerCalled) + + cluster = cephv1.CephCluster{} + err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster) + assert.NoError(t, err) + assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated + assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated + + monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{}) + assert.NoError(t, err) + wantMonSecretData = map[string][]byte{ + "mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="), + "fsid": []byte("00000000-0000-0000-0000-000000000000"), + "ceph-username": []byte("client.admin"), + "ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // changed + } + assert.Equal(t, wantMonSecretData, monSec.Data) + + _, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{}) + assert.ErrorContains(t, err, "not found") // rotator keyring not persisted + }) + + t.Run("fail auth ls as temp rotated client.admin", func(t *testing.T) { + clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest() + + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authCreateAdminRotatorOutput, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authLsOutputBeforeRotation, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "rotate", "client.admin"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authRotateAdminOutput, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpAdminClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return "", fmt.Errorf("induced error") + }, + }) + + err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster) + assert.ErrorContains(t, err, "induced error") + assert.Equal(t, len(expectList), numCalls) // any expected calls not called? + assert.False(t, reloadManagerCalled) + + cluster := cephv1.CephCluster{} + err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster) + assert.NoError(t, err) + assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged + assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged + + monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{}) + assert.NoError(t, err) + wantMonSecretData := map[string][]byte{ + "mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="), + "fsid": []byte("00000000-0000-0000-0000-000000000000"), + "ceph-username": []byte("client.admin"), + "ceph-secret": []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="), // unchanged + } + assert.Equal(t, wantMonSecretData, monSec.Data) + + _, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{}) + assert.NoError(t, err) // rotator secret now persisted + + // expect recovery to proceed + numCalls = 0 + expectList = []expect{} // clear expect list + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return "", fmt.Errorf("client.admin key stored rook-ceph-mon secret now doesn't work") + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authLsOutputAfterRotation, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "rotate", "client.admin"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authRotateAdminOutput, nil // auth rotate happens again + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpAdminClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authLsOutputAfterRotatorDeletion, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "del", "client.admin-rotator"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return "", nil + }, + }) + + err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns) + assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation) + assert.Equal(t, len(expectList), numCalls) // any expected calls not called? + assert.True(t, reloadManagerCalled) + + cluster = cephv1.CephCluster{} + err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster) + assert.NoError(t, err) + assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated + assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated + + monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{}) + assert.NoError(t, err) + wantMonSecretData = map[string][]byte{ + "mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="), + "fsid": []byte("00000000-0000-0000-0000-000000000000"), + "ceph-username": []byte("client.admin"), + "ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // changed + } + assert.Equal(t, wantMonSecretData, monSec.Data) + + _, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{}) + assert.ErrorContains(t, err, "not found") // rotator keyring not persisted + }) + + t.Run("fail auth del admin-rotator", func(t *testing.T) { + clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest() + + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authCreateAdminRotatorOutput, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authLsOutputBeforeRotation, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "rotate", "client.admin"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authRotateAdminOutput, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpAdminClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authLsOutputAfterRotatorDeletion, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "del", "client.admin-rotator"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return "", fmt.Errorf("induced error") + }, + }) + + err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster) + assert.ErrorContains(t, err, "induced error") + assert.Equal(t, len(expectList), numCalls) // any expected calls not called? + assert.False(t, reloadManagerCalled) + + cluster := cephv1.CephCluster{} + err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster) + assert.NoError(t, err) + assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged + assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged + + monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{}) + assert.NoError(t, err) + wantMonSecretData := map[string][]byte{ + "mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="), + "fsid": []byte("00000000-0000-0000-0000-000000000000"), + "ceph-username": []byte("client.admin"), + "ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // IS UPDATED! + } + assert.Equal(t, wantMonSecretData, monSec.Data) + + _, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{}) + assert.NoError(t, err) // rotator secret still persisted + + // expect recovery to proceed + numCalls = 0 + expectList = []expect{} // clear expect list + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + // client.admin key stored rook-ceph-mon secret works, but Rook can't know if that + // is because it is un-rotated or because auth del failed + return authLsOutputAfterRotation, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authLsOutputAfterRotation, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "rotate", "client.admin"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authRotateAdminOutput, nil // auth rotate happens again + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpAdminClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authLsOutputAfterRotatorDeletion, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "del", "client.admin-rotator"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return "", nil + }, + }) + + err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns) + assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation) + assert.Equal(t, len(expectList), numCalls) // any expected calls not called? + assert.True(t, reloadManagerCalled) + + cluster = cephv1.CephCluster{} + err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster) + assert.NoError(t, err) + assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated + assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated + + monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{}) + assert.NoError(t, err) + wantMonSecretData = map[string][]byte{ + "mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="), + "fsid": []byte("00000000-0000-0000-0000-000000000000"), + "ceph-username": []byte("client.admin"), + "ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // rotated value + } + assert.Equal(t, wantMonSecretData, monSec.Data) + + _, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{}) + assert.ErrorContains(t, err, "not found") // rotator keyring not persisted + }) + + t.Run("recover cleanup", func(t *testing.T) { + clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest() + + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authCreateAdminRotatorOutput, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authLsOutputBeforeRotation, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "rotate", "client.admin"}, + withArgs: tmpRotatorClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authRotateAdminOutput, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: tmpAdminClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return authLsOutputAfterRotatorDeletion, nil + }, + }) + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "del", "client.admin-rotator"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + return "", fmt.Errorf("induced error") + }, + }) + + err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster) + assert.ErrorContains(t, err, "induced error") + assert.Equal(t, len(expectList), numCalls) // any expected calls not called? + assert.False(t, reloadManagerCalled) + + cluster := cephv1.CephCluster{} + err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster) + assert.NoError(t, err) + assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged + assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged + + monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{}) + assert.NoError(t, err) + wantMonSecretData := map[string][]byte{ + "mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="), + "fsid": []byte("00000000-0000-0000-0000-000000000000"), + "ceph-username": []byte("client.admin"), + "ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // IS UPDATED! + } + assert.Equal(t, wantMonSecretData, monSec.Data) + + _, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{}) + assert.NoError(t, err) // rotator secret still persisted + + // expect recovery to proceed + numCalls = 0 + expectList = []expect{} // clear expect list + expectList = append(expectList, expect{ + cargs: []string{"ceph", "auth", "ls"}, + withArgs: defaultClientArgs, + returnFunc: func(cmd string, args ...string) (string, error) { + // pretend that auth del successfully deleted auth-rotator + return authLsOutputAfterRotatorDeletion, nil + }, + }) + // don't expect any more commands, but just check that secrets are correct + + err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns) + assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation) + assert.Equal(t, len(expectList), numCalls) // any expected calls not called? + assert.True(t, reloadManagerCalled) + + cluster = cephv1.CephCluster{} + err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster) + assert.NoError(t, err) + assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated + assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated + + monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{}) + assert.NoError(t, err) + wantMonSecretData = map[string][]byte{ + "mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="), + "fsid": []byte("00000000-0000-0000-0000-000000000000"), + "ceph-username": []byte("client.admin"), + "ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // unchanged + } + assert.Equal(t, wantMonSecretData, monSec.Data) + + _, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{}) + assert.ErrorContains(t, err, "not found") // rotator keyring not persisted + }) +} + +// super simple helper to see if the command matches +func isCommand(cargs []string, lookFor ...string) bool { + for i, want := range lookFor { + if cargs[i] != want { + return false + } + } + return true +} + +func hasArg(args []string, lookFor string) bool { + for _, arg := range args { + if arg == lookFor { + return true + } + } + return false +} + +const ( + // `ceph auth get-or-create-key client.admin-rotator` output + authCreateAdminRotatorOutput = `{ + "entity": "client.admin-rotator", + "key": "ADMINROTATORADMINROTATORADMINROTATORADMINROTATORADMINROTATO=", + "caps": { + "mds": "allow *", + "mgr": "allow *", + "mon": "allow *", + "osd": "allow *" + } +}` + + // `ceph auth ls` output from before client.admin rotation (after admin-rotator user created) + // output pared down to a few entries to keep CI test more readable without being too trivial + authLsOutputBeforeRotation = `{ + "auth_dump": [ + { + "entity": "client.admin", + "key": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "caps": { + "mds": "allow *", + "mgr": "allow *", + "mon": "allow *", + "osd": "allow *" + } + }, + { + "entity": "client.admin-rotator", + "key": "ADMINROTATORADMINROTATORADMINROTATORADMINROTATORADMINROTATO=", + "caps": { + "mds": "allow *", + "mgr": "allow *", + "mon": "allow *", + "osd": "allow *" + } + }, + { + "entity": "client.csi-rbd-node.1", + "key": "AgASB6ZohDE6MCAANiC1Nt0tE6IjQi5nPeiFKvaNZLcZcIJ1bgYfRQ6hlPE=", + "caps": { + "mgr": "allow rw", + "mon": "profile rbd", + "osd": "profile rbd" + } + }, + { + "entity": "client.csi-rbd-provisioner.1", + "key": "AgASB6ZoEIZ1IyAANmYrvZmHsseRtTb2iftoD0Jp3Un3Ob+6wa+HDW1Qh5I=", + "caps": { + "mgr": "allow rw", + "mon": "profile rbd, allow command 'osd blocklist'", + "osd": "profile rbd" + } + } + ] +}` + + // `ceph auth rotate client.admin` output + authRotateAdminOutput = `[{ + "entity": "client.admin", + "key": "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=", + "caps": { + "mds": "allow *", + "mgr": "allow *", + "mon": "allow *", + "osd": "allow *" + } +}]` + + authLsOutputAfterRotation = `{ + "auth_dump": [ + { + "entity": "client.admin", + "key": "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=", + "caps": { + "mds": "allow *", + "mgr": "allow *", + "mon": "allow *", + "osd": "allow *" + } + }, + { + "entity": "client.admin-rotator", + "key": "ADMINROTATORADMINROTATORADMINROTATORADMINROTATORADMINROTATO=", + "caps": { + "mds": "allow *", + "mgr": "allow *", + "mon": "allow *", + "osd": "allow *" + } + }, + { + "entity": "client.csi-rbd-node.1", + "key": "AgASB6ZohDE6MCAANiC1Nt0tE6IjQi5nPeiFKvaNZLcZcIJ1bgYfRQ6hlPE=", + "caps": { + "mgr": "allow rw", + "mon": "profile rbd", + "osd": "profile rbd" + } + }, + { + "entity": "client.csi-rbd-provisioner.1", + "key": "AgASB6ZoEIZ1IyAANmYrvZmHsseRtTb2iftoD0Jp3Un3Ob+6wa+HDW1Qh5I=", + "caps": { + "mgr": "allow rw", + "mon": "profile rbd, allow command 'osd blocklist'", + "osd": "profile rbd" + } + } + ] +}` + + // `ceph auth ls` output from after client.admin-rotator deleted + // output pared down to a few entries to keep CI test more readable without being too trivial + authLsOutputAfterRotatorDeletion = `{ + "auth_dump": [ + { + "entity": "client.admin", + "key": "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=", + "caps": { + "mds": "allow *", + "mgr": "allow *", + "mon": "allow *", + "osd": "allow *" + } + }, + { + "entity": "client.csi-rbd-node.1", + "key": "AgASB6ZohDE6MCAANiC1Nt0tE6IjQi5nPeiFKvaNZLcZcIJ1bgYfRQ6hlPE=", + "caps": { + "mgr": "allow rw", + "mon": "profile rbd", + "osd": "profile rbd" + } + }, + { + "entity": "client.csi-rbd-provisioner.1", + "key": "AgASB6ZoEIZ1IyAANmYrvZmHsseRtTb2iftoD0Jp3Un3Ob+6wa+HDW1Qh5I=", + "caps": { + "mgr": "allow rw", + "mon": "profile rbd, allow command 'osd blocklist'", + "osd": "profile rbd" + } + } + ] +}` +) + +func Test_adminRotationLock(t *testing.T) { + var err error + + err = claimAdminRotationLock("ns") + assert.NoError(t, err) + + err = claimAdminRotationLock("ns") + assert.Error(t, err) + + err = claimAdminRotationLock("other") + assert.NoError(t, err) + + releaseAdminRotationLock("ns") + + err = claimAdminRotationLock("ns") + assert.NoError(t, err) + + assert.Equal(t, map[string]struct{}{ + "ns": {}, + "other": {}, + }, adminRotationInProgress) + + releaseAdminRotationLock("ns") + releaseAdminRotationLock("other") + + assert.Equal(t, map[string]struct{}{}, adminRotationInProgress) +} diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 47097ae9b..bccaf9595 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -204,6 +204,9 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error { if err != nil { if errors.Is(err, controller.ClusterInfoNoClusterNoSecret) { logger.Info("clusterInfo not yet found, must be a new cluster.") + // ClusterInfoNoClusterNoSecret should return nil clusterInfo, but this is a + // mandatory condition for recoverPriorAdminCephxKeyRotation() below, so ensure it + clusterInfo = nil } else { return errors.Wrap(err, "failed to load cluster info") } @@ -211,7 +214,21 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error { clusterInfo.OwnerInfo = cluster.ownerInfo clusterInfo.SetName(c.namespacedName.Name) cluster.ClusterInfo = clusterInfo + if cluster.mons.ClusterInfo == nil { + // ClusterInfo stored in cluster.mons can be lost from the object stored in the + // clusterMap during admin key rotation corner cases. rehydrate it if needed + logger.Debugf("applying missing ClusterInfo to tracked mons info for cluster in namespace %q", cluster.Namespace) + cluster.mons.ClusterInfo = clusterInfo + } } + + // if necessary, recover from failed/interrupted admin key rotation + // this must be done before any ceph cli commands are run + err = recoverPriorAdminCephxKeyRotation(c.context, clusterInfo, cluster.ownerInfo, cluster.Namespace) + if err != nil { + return errors.Wrap(err, "failed to recover from prior admin cephx key rotation") + } + // If the local cluster has already been configured, immediately start monitoring the cluster. // Test if the cluster has already been configured if the mgr deployment has been created. // If the mgr does not exist, the mons have never been verified to be in quorum. @@ -466,11 +483,18 @@ func (c *cluster) preMonStartupActions(cephVersion cephver.CephVersion) error { // It gets executed right after the main mon Start() method // Basically, it is executed between the monitors and the manager sequence func (c *cluster) postMonStartupActions() error { + // full cluster spec/status will be used to inform various cephx rotations clusterObj := &cephv1.CephCluster{} if err := c.context.Client.Get(c.ClusterInfo.Context, c.ClusterInfo.NamespacedName(), clusterObj); err != nil { return errors.Wrapf(err, "failed to get cluster %v.", c.ClusterInfo.NamespacedName()) } + // rotate admin key first thing after mons are updated + err := rotateAdminCephxKey(c.context, c.ClusterInfo, c.ownerInfo, clusterObj) // TODO: rename? + if err != nil { + return errors.Wrapf(err, "failed to rotate admin cephx key") + } + // rotate mon cephx keys if required didRotateMonCephxKeys, err := c.mons.RotateMonCephxKeys(clusterObj) if err != nil { diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index 056849d75..7b3216bb9 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -128,13 +128,13 @@ func (hc *HealthChecker) Check(monitoringRoutines map[string]*controller.Cluster select { case <-monitoringRoutines[daemon].InternalCtx.Done(): - logger.Infof("stopping monitoring of mons in namespace %q", hc.monCluster.Namespace) + logger.Infof("stopping monitoring of mons in namespace %q (internal context canceled)", hc.monCluster.Namespace) delete(monitoringRoutines, daemon) return // Since c.ClusterInfo.IsInitialized() below uses a different context, we need to check if the context is done case <-hc.monCluster.ClusterInfo.Context.Done(): - logger.Infof("stopping monitoring of mons in namespace %q", hc.monCluster.Namespace) + logger.Infof("stopping monitoring of mons in namespace %q (mon cluster info context canceled)", hc.monCluster.Namespace) delete(monitoringRoutines, daemon) return diff --git a/pkg/operator/ceph/config/keyring/store.go b/pkg/operator/ceph/config/keyring/store.go index 8691019bb..e75e5c7b3 100644 --- a/pkg/operator/ceph/config/keyring/store.go +++ b/pkg/operator/ceph/config/keyring/store.go @@ -20,6 +20,8 @@ limitations under the License. package keyring import ( + "fmt" + "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" @@ -149,3 +151,20 @@ func (k *SecretStore) CreateSecret(secret *v1.Secret) (string, error) { } return s.ResourceVersion, nil } + +// GetKeyringFromSecret returns the keyring present in a keyring secret +func (k *SecretStore) GetKeyringFromSecret(resourceName string) (string, error) { + secretName := keyringSecretName(resourceName) + s, err := k.context.Clientset.CoreV1().Secrets(k.clusterInfo.Namespace).Get(k.clusterInfo.Context, secretName, metav1.GetOptions{}) + if err != nil { + return "", err // do not wrap so that err can be compared to does not exist if needed + } + if keyring, ok := s.StringData[keyringFileName]; ok { // only available in unit tests + return keyring, nil + } + keyring, ok := s.Data[keyringFileName] + if !ok { + return "", fmt.Errorf("keyring secret %q does not contain keyring", secretName) + } + return string(keyring), nil +} diff --git a/pkg/operator/ceph/config/keyring/store_test.go b/pkg/operator/ceph/config/keyring/store_test.go index 6c3f604e1..9ae2496a9 100644 --- a/pkg/operator/ceph/config/keyring/store_test.go +++ b/pkg/operator/ceph/config/keyring/store_test.go @@ -111,6 +111,11 @@ func TestKeyringStore(t *testing.T) { assertKeyringData("test-resource-keyring", "qwertyuiop") assertKeyringData("second-resource-keyring", "lkjhgfdsa") + // get key from secret + keyring, err := k.GetKeyringFromSecret("test-resource") + assert.NoError(t, err) + assert.Equal(t, "qwertyuiop", keyring) + // delete a key err = k.Delete("test-resource") assert.NoError(t, err) diff --git a/pkg/operator/ceph/controller/cluster_info.go b/pkg/operator/ceph/controller/cluster_info.go index 9e3fb75a8..5f51f4b1c 100644 --- a/pkg/operator/ceph/controller/cluster_info.go +++ b/pkg/operator/ceph/controller/cluster_info.go @@ -406,6 +406,9 @@ func UpdateClusterAccessSecret(clientset kubernetes.Interface, clusterInfo *ceph return errors.Wrapf(err, "failed to get secret %q in the namespace %q", AppName, clusterInfo.Namespace) } + // update admin key secret + secret.Data[CephUserSecretKey] = []byte(clusterInfo.CephCred.Secret) + secret.Data[MonSecretNameKey] = []byte(clusterInfo.MonitorSecret) logger.Infof("updating secret %q in the namespace %q", secret.Name, clusterInfo.Namespace)