forked from rook/rook
Blaine Gardner
7c33186e72
Implement CephX key rotation for Rook's client.admin user. Admin user rotation is risky, so this has been tested extensively both in unit tests as well as by manually injecting failures during runtime. In testing, all failures were able to be recovered by the recovery routine. A mutex is also added to help ensure that two simultaneous admin key rotation processes cannot be running simultaneously for any given namespace. The mutex is tested in unit tests, and it was verified during runtime via manual testing. Signed-off-by: Blaine Gardner <blaine.gardner@ibm.com>
1150 lines
43 KiB
Go
1150 lines
43 KiB
Go
/*
|
|
Copyright 2025 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package cluster
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"testing"
|
|
|
|
"github.com/coreos/pkg/capnslog"
|
|
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
|
"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
|
|
"github.com/rook/rook/pkg/clusterd"
|
|
"github.com/rook/rook/pkg/daemon/ceph/client"
|
|
"github.com/rook/rook/pkg/operator/ceph/version"
|
|
"github.com/rook/rook/pkg/operator/k8sutil"
|
|
testop "github.com/rook/rook/pkg/operator/test"
|
|
exectest "github.com/rook/rook/pkg/util/exec/test"
|
|
"github.com/stretchr/testify/assert"
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
)
|
|
|
|
func Test_genKeyring(t *testing.T) {
|
|
twoCapsKeyring := `[client.them]
|
|
key = themkey==
|
|
caps any = "thing"
|
|
`
|
|
adminRotatorKeyring := `[client.admin-rotator]
|
|
key = adminrotatorkey==
|
|
caps mds = "allow *"
|
|
caps mon = "allow *"
|
|
caps osd = "allow *"
|
|
caps mgr = "allow *"
|
|
`
|
|
|
|
tests := []struct {
|
|
name string
|
|
clientName string
|
|
clientKey string
|
|
clientCaps []string
|
|
wantErr bool
|
|
want string
|
|
}{
|
|
{"no caps", "client.me", "mekey==", []string{}, false, "[client.me]\n key = mekey==\n"},
|
|
{"no caps, no key", "client.me", "", []string{}, true, ""},
|
|
{"one cap", "client.you", "youkey==", []string{"bad"}, true, ""},
|
|
{"two caps", "client.them", "themkey==", []string{"any", "thing"}, false, twoCapsKeyring},
|
|
{"three caps", "client.batman", "batmankey==", []string{"some", "thing", "else"}, true, ""},
|
|
{"admin rotator caps", "client.admin-rotator", "adminrotatorkey==", adminKeyAccessCaps, false, adminRotatorKeyring},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got, err := genKeyring(tt.clientName, tt.clientKey, tt.clientCaps)
|
|
if (err != nil) != tt.wantErr {
|
|
t.Errorf("genKeyring() error = %v, wantErr %v", err, tt.wantErr)
|
|
return
|
|
}
|
|
if got != tt.want {
|
|
t.Errorf("genKeyring() = %v, want %v", got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func Test_admin_key_rotation(t *testing.T) {
|
|
logger.SetLevel(capnslog.DEBUG) // TESTING TESTING TESTING
|
|
|
|
reloadManagerCalled := false
|
|
reloadManagerFunc = func() {
|
|
reloadManagerCalled = true
|
|
}
|
|
|
|
// this feature requires commands to execute in a specific order and often with specific arg(s)
|
|
// it also requires commands to not be skipped
|
|
// set up an executor that allows tests to expect commands in a certain order
|
|
type expect struct {
|
|
cargs []string
|
|
withArgs []string
|
|
returnFunc func(cmd string, args ...string) (string, error)
|
|
}
|
|
numCalls := 0
|
|
expectList := []expect{}
|
|
|
|
mockExecutor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(cmd string, args ...string) (string, error) {
|
|
cargs := append([]string{cmd}, args...) // cmd+args in one slice for easy comparison
|
|
t.Logf("mock command: %v", cargs)
|
|
|
|
// get current command expectations
|
|
if len(expectList) <= numCalls {
|
|
e := fmt.Errorf("execution iteration=%d: no expectation set for called %v", numCalls, cargs)
|
|
t.Error(e)
|
|
return "", e
|
|
}
|
|
currentExpect := expectList[numCalls]
|
|
numCalls++
|
|
|
|
// assert current command matches expectations
|
|
if !isCommand(cargs, currentExpect.cargs...) {
|
|
e := fmt.Errorf("execution iteration=%d: expected command %v does not match called %v", numCalls, currentExpect.cargs, cargs)
|
|
t.Error(e)
|
|
return "", e
|
|
}
|
|
if len(currentExpect.withArgs) > 0 {
|
|
for _, arg := range currentExpect.withArgs {
|
|
if !hasArg(cargs, arg) {
|
|
e := fmt.Errorf("execution iteration=%d: expected arg %q missing in called %v", numCalls, arg, cargs)
|
|
t.Error(e)
|
|
return "", e
|
|
}
|
|
}
|
|
}
|
|
|
|
// expectations met, call return func with args
|
|
return currentExpect.returnFunc(cmd, args...)
|
|
},
|
|
}
|
|
|
|
tmpCfgDir, err := os.MkdirTemp("", "")
|
|
t.Logf("temp dir: %q", tmpCfgDir)
|
|
assert.NoError(t, err)
|
|
defer func() { os.RemoveAll(tmpCfgDir) }()
|
|
|
|
ns := "ns"
|
|
clusterName := "my-cluster"
|
|
|
|
newTest := func() (*clusterd.Context, *client.ClusterInfo, *k8sutil.OwnerInfo, *cephv1.CephCluster) {
|
|
reloadManagerCalled = false
|
|
numCalls = 0
|
|
expectList = []expect{} // clear expect list
|
|
|
|
clusterInfo := &client.ClusterInfo{
|
|
Context: context.TODO(),
|
|
FSID: "00000000-0000-0000-0000-000000000000",
|
|
Namespace: ns,
|
|
CephVersion: version.CephVersion{Major: 20, Minor: 3, Extra: 0},
|
|
MonitorSecret: "MONSECRET/SHOULDNOTCHANGE=",
|
|
CephCred: client.CephCred{
|
|
Username: "client.admin",
|
|
Secret: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
|
|
},
|
|
OwnerInfo: &k8sutil.OwnerInfo{},
|
|
}
|
|
clusterInfo.SetName(clusterName)
|
|
|
|
cephCluster := &cephv1.CephCluster{ // create via controller-runtime client
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: ns,
|
|
Name: clusterName,
|
|
},
|
|
Spec: cephv1.ClusterSpec{
|
|
Security: cephv1.ClusterSecuritySpec{
|
|
CephX: cephv1.ClusterCephxConfig{
|
|
Daemon: cephv1.CephxConfig{
|
|
KeyRotationPolicy: "KeyGeneration",
|
|
KeyGeneration: 2,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Status: cephv1.ClusterStatus{
|
|
Cephx: cephv1.ClusterCephxStatus{
|
|
Admin: cephv1.CephxStatus{
|
|
KeyGeneration: 1,
|
|
KeyCephVersion: "19.2.3-0",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
monSecret := &corev1.Secret{ // create via clientset
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: "ns",
|
|
Name: "rook-ceph-mon",
|
|
},
|
|
Data: map[string][]byte{
|
|
"ceph-secret": []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="), // before rotation key
|
|
"ceph-username": []byte("client.admin"),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
},
|
|
}
|
|
|
|
clientset := testop.New(t, 3)
|
|
_, err := clientset.CoreV1().Secrets(ns).Create(context.TODO(), monSecret, metav1.CreateOptions{})
|
|
if err != nil {
|
|
panic(err) // test setup failed
|
|
}
|
|
|
|
s := scheme.Scheme
|
|
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{})
|
|
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(cephCluster).Build()
|
|
|
|
clusterdCtx := &clusterd.Context{
|
|
Clientset: clientset,
|
|
Client: cl,
|
|
Executor: mockExecutor,
|
|
ConfigDir: tmpCfgDir,
|
|
}
|
|
|
|
return clusterdCtx, clusterInfo, clusterInfo.OwnerInfo, cephCluster
|
|
}
|
|
|
|
// args that the default clusterInfo should use
|
|
defaultClientArgs := []string{"--name=client.admin", "--keyring=" + tmpCfgDir + "/ns/client.admin.keyring"}
|
|
|
|
// args that the temporary admin-rotator user should use
|
|
tmpRotatorClientArgs := []string{"--name=client.admin-rotator", "--keyring=" + tmpCfgDir + "/ns/admin-rotate/client.admin-rotator.keyring"}
|
|
|
|
// args that the new admin client should use temporarily to verify it works before final storage
|
|
tmpAdminClientArgs := []string{"--name=client.admin", "--keyring=" + tmpCfgDir + "/ns/admin-rotate/client.admin.keyring"}
|
|
|
|
t.Run("happy path rotation", func(t *testing.T) {
|
|
clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest()
|
|
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authCreateAdminRotatorOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authRotateAdminOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpAdminClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotatorDeletion, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "del", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", nil
|
|
},
|
|
})
|
|
|
|
err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster)
|
|
assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation)
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.True(t, reloadManagerCalled)
|
|
|
|
cluster := cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated
|
|
assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated
|
|
|
|
monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData := map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // changed
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
|
|
// after successful rotation, recovery should not occur
|
|
err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, len(expectList), numCalls) // same number calls from before
|
|
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged
|
|
assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged
|
|
|
|
monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, wantMonSecretData, monSec.Data) // unchanged
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
})
|
|
|
|
t.Run("fail create admin-rotator", func(t *testing.T) {
|
|
clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest()
|
|
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", fmt.Errorf("induced error")
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
// auth generation tries to modify caps if auth get-or-create-key fails
|
|
cargs: []string{"ceph", "auth", "caps", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", fmt.Errorf("induced error")
|
|
},
|
|
})
|
|
|
|
err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster)
|
|
assert.ErrorContains(t, err, "induced error")
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.False(t, reloadManagerCalled)
|
|
|
|
cluster := cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged
|
|
assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged
|
|
|
|
monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData := map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="), // unchanged
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
// nothing to recover before admin-rotator created and stored in secret
|
|
err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, len(expectList), numCalls) // same number calls from before
|
|
|
|
monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, wantMonSecretData, monSec.Data) // unchanged
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
})
|
|
|
|
t.Run("fail ls as admin rotator", func(t *testing.T) {
|
|
clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest()
|
|
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authCreateAdminRotatorOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", fmt.Errorf("induced error")
|
|
},
|
|
})
|
|
|
|
err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster)
|
|
assert.ErrorContains(t, err, "induced error")
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.False(t, reloadManagerCalled)
|
|
|
|
cluster := cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged
|
|
assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged
|
|
|
|
monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData := map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="), // unchanged
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.NoError(t, err) // rotator secret now persisted
|
|
|
|
// expect recovery to proceed
|
|
numCalls = 0
|
|
expectList = []expect{} // clear expect list
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
// client.admin can still ls, and admin-rotator is present
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authRotateAdminOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpAdminClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotatorDeletion, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "del", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", nil
|
|
},
|
|
})
|
|
|
|
err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns)
|
|
assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation)
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.True(t, reloadManagerCalled)
|
|
|
|
cluster = cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated
|
|
assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated
|
|
|
|
monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData = map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // changed
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
|
|
t.Run("recovery doesn't reoccur after successful recovery", func(t *testing.T) {
|
|
// only need to test this in one of the unit tests
|
|
err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, len(expectList), numCalls) // same number calls from before
|
|
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged
|
|
assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged
|
|
|
|
monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, wantMonSecretData, monSec.Data) // unchanged
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
})
|
|
})
|
|
|
|
t.Run("fail auth rotate", func(t *testing.T) {
|
|
clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest()
|
|
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authCreateAdminRotatorOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", fmt.Errorf("induced error")
|
|
},
|
|
})
|
|
|
|
err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster)
|
|
assert.ErrorContains(t, err, "induced error")
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.False(t, reloadManagerCalled)
|
|
|
|
cluster := cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged
|
|
assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged
|
|
|
|
monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData := map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="), // unchanged
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.NoError(t, err) // rotator secret now persisted
|
|
|
|
// expect recovery to proceed
|
|
numCalls = 0
|
|
expectList = []expect{} // clear expect list
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
// client.admin can still ls, and admin-rotator is present
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authRotateAdminOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpAdminClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotatorDeletion, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "del", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", nil
|
|
},
|
|
})
|
|
|
|
err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns)
|
|
assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation)
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.True(t, reloadManagerCalled)
|
|
|
|
cluster = cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated
|
|
assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated
|
|
|
|
monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData = map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // changed
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
})
|
|
|
|
t.Run("fail auth ls as temp rotated client.admin", func(t *testing.T) {
|
|
clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest()
|
|
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authCreateAdminRotatorOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authRotateAdminOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpAdminClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", fmt.Errorf("induced error")
|
|
},
|
|
})
|
|
|
|
err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster)
|
|
assert.ErrorContains(t, err, "induced error")
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.False(t, reloadManagerCalled)
|
|
|
|
cluster := cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged
|
|
assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged
|
|
|
|
monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData := map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="), // unchanged
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.NoError(t, err) // rotator secret now persisted
|
|
|
|
// expect recovery to proceed
|
|
numCalls = 0
|
|
expectList = []expect{} // clear expect list
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", fmt.Errorf("client.admin key stored rook-ceph-mon secret now doesn't work")
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authRotateAdminOutput, nil // auth rotate happens again
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpAdminClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotatorDeletion, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "del", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", nil
|
|
},
|
|
})
|
|
|
|
err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns)
|
|
assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation)
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.True(t, reloadManagerCalled)
|
|
|
|
cluster = cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated
|
|
assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated
|
|
|
|
monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData = map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // changed
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
})
|
|
|
|
t.Run("fail auth del admin-rotator", func(t *testing.T) {
|
|
clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest()
|
|
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authCreateAdminRotatorOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authRotateAdminOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpAdminClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotatorDeletion, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "del", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", fmt.Errorf("induced error")
|
|
},
|
|
})
|
|
|
|
err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster)
|
|
assert.ErrorContains(t, err, "induced error")
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.False(t, reloadManagerCalled)
|
|
|
|
cluster := cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged
|
|
assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged
|
|
|
|
monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData := map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // IS UPDATED!
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.NoError(t, err) // rotator secret still persisted
|
|
|
|
// expect recovery to proceed
|
|
numCalls = 0
|
|
expectList = []expect{} // clear expect list
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
// client.admin key stored rook-ceph-mon secret works, but Rook can't know if that
|
|
// is because it is un-rotated or because auth del failed
|
|
return authLsOutputAfterRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authRotateAdminOutput, nil // auth rotate happens again
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpAdminClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotatorDeletion, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "del", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", nil
|
|
},
|
|
})
|
|
|
|
err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns)
|
|
assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation)
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.True(t, reloadManagerCalled)
|
|
|
|
cluster = cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated
|
|
assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated
|
|
|
|
monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData = map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // rotated value
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
})
|
|
|
|
t.Run("recover cleanup", func(t *testing.T) {
|
|
clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest()
|
|
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authCreateAdminRotatorOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authRotateAdminOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpAdminClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotatorDeletion, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "del", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", fmt.Errorf("induced error")
|
|
},
|
|
})
|
|
|
|
err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster)
|
|
assert.ErrorContains(t, err, "induced error")
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.False(t, reloadManagerCalled)
|
|
|
|
cluster := cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged
|
|
assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged
|
|
|
|
monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData := map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // IS UPDATED!
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.NoError(t, err) // rotator secret still persisted
|
|
|
|
// expect recovery to proceed
|
|
numCalls = 0
|
|
expectList = []expect{} // clear expect list
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
// pretend that auth del successfully deleted auth-rotator
|
|
return authLsOutputAfterRotatorDeletion, nil
|
|
},
|
|
})
|
|
// don't expect any more commands, but just check that secrets are correct
|
|
|
|
err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns)
|
|
assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation)
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.True(t, reloadManagerCalled)
|
|
|
|
cluster = cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated
|
|
assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated
|
|
|
|
monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData = map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // unchanged
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
})
|
|
}
|
|
|
|
// super simple helper to see if the command matches
|
|
func isCommand(cargs []string, lookFor ...string) bool {
|
|
for i, want := range lookFor {
|
|
if cargs[i] != want {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func hasArg(args []string, lookFor string) bool {
|
|
for _, arg := range args {
|
|
if arg == lookFor {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
const (
|
|
// `ceph auth get-or-create-key client.admin-rotator` output
|
|
authCreateAdminRotatorOutput = `{
|
|
"entity": "client.admin-rotator",
|
|
"key": "ADMINROTATORADMINROTATORADMINROTATORADMINROTATORADMINROTATO=",
|
|
"caps": {
|
|
"mds": "allow *",
|
|
"mgr": "allow *",
|
|
"mon": "allow *",
|
|
"osd": "allow *"
|
|
}
|
|
}`
|
|
|
|
// `ceph auth ls` output from before client.admin rotation (after admin-rotator user created)
|
|
// output pared down to a few entries to keep CI test more readable without being too trivial
|
|
authLsOutputBeforeRotation = `{
|
|
"auth_dump": [
|
|
{
|
|
"entity": "client.admin",
|
|
"key": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
|
|
"caps": {
|
|
"mds": "allow *",
|
|
"mgr": "allow *",
|
|
"mon": "allow *",
|
|
"osd": "allow *"
|
|
}
|
|
},
|
|
{
|
|
"entity": "client.admin-rotator",
|
|
"key": "ADMINROTATORADMINROTATORADMINROTATORADMINROTATORADMINROTATO=",
|
|
"caps": {
|
|
"mds": "allow *",
|
|
"mgr": "allow *",
|
|
"mon": "allow *",
|
|
"osd": "allow *"
|
|
}
|
|
},
|
|
{
|
|
"entity": "client.csi-rbd-node.1",
|
|
"key": "AgASB6ZohDE6MCAANiC1Nt0tE6IjQi5nPeiFKvaNZLcZcIJ1bgYfRQ6hlPE=",
|
|
"caps": {
|
|
"mgr": "allow rw",
|
|
"mon": "profile rbd",
|
|
"osd": "profile rbd"
|
|
}
|
|
},
|
|
{
|
|
"entity": "client.csi-rbd-provisioner.1",
|
|
"key": "AgASB6ZoEIZ1IyAANmYrvZmHsseRtTb2iftoD0Jp3Un3Ob+6wa+HDW1Qh5I=",
|
|
"caps": {
|
|
"mgr": "allow rw",
|
|
"mon": "profile rbd, allow command 'osd blocklist'",
|
|
"osd": "profile rbd"
|
|
}
|
|
}
|
|
]
|
|
}`
|
|
|
|
// `ceph auth rotate client.admin` output
|
|
authRotateAdminOutput = `[{
|
|
"entity": "client.admin",
|
|
"key": "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=",
|
|
"caps": {
|
|
"mds": "allow *",
|
|
"mgr": "allow *",
|
|
"mon": "allow *",
|
|
"osd": "allow *"
|
|
}
|
|
}]`
|
|
|
|
authLsOutputAfterRotation = `{
|
|
"auth_dump": [
|
|
{
|
|
"entity": "client.admin",
|
|
"key": "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=",
|
|
"caps": {
|
|
"mds": "allow *",
|
|
"mgr": "allow *",
|
|
"mon": "allow *",
|
|
"osd": "allow *"
|
|
}
|
|
},
|
|
{
|
|
"entity": "client.admin-rotator",
|
|
"key": "ADMINROTATORADMINROTATORADMINROTATORADMINROTATORADMINROTATO=",
|
|
"caps": {
|
|
"mds": "allow *",
|
|
"mgr": "allow *",
|
|
"mon": "allow *",
|
|
"osd": "allow *"
|
|
}
|
|
},
|
|
{
|
|
"entity": "client.csi-rbd-node.1",
|
|
"key": "AgASB6ZohDE6MCAANiC1Nt0tE6IjQi5nPeiFKvaNZLcZcIJ1bgYfRQ6hlPE=",
|
|
"caps": {
|
|
"mgr": "allow rw",
|
|
"mon": "profile rbd",
|
|
"osd": "profile rbd"
|
|
}
|
|
},
|
|
{
|
|
"entity": "client.csi-rbd-provisioner.1",
|
|
"key": "AgASB6ZoEIZ1IyAANmYrvZmHsseRtTb2iftoD0Jp3Un3Ob+6wa+HDW1Qh5I=",
|
|
"caps": {
|
|
"mgr": "allow rw",
|
|
"mon": "profile rbd, allow command 'osd blocklist'",
|
|
"osd": "profile rbd"
|
|
}
|
|
}
|
|
]
|
|
}`
|
|
|
|
// `ceph auth ls` output from after client.admin-rotator deleted
|
|
// output pared down to a few entries to keep CI test more readable without being too trivial
|
|
authLsOutputAfterRotatorDeletion = `{
|
|
"auth_dump": [
|
|
{
|
|
"entity": "client.admin",
|
|
"key": "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=",
|
|
"caps": {
|
|
"mds": "allow *",
|
|
"mgr": "allow *",
|
|
"mon": "allow *",
|
|
"osd": "allow *"
|
|
}
|
|
},
|
|
{
|
|
"entity": "client.csi-rbd-node.1",
|
|
"key": "AgASB6ZohDE6MCAANiC1Nt0tE6IjQi5nPeiFKvaNZLcZcIJ1bgYfRQ6hlPE=",
|
|
"caps": {
|
|
"mgr": "allow rw",
|
|
"mon": "profile rbd",
|
|
"osd": "profile rbd"
|
|
}
|
|
},
|
|
{
|
|
"entity": "client.csi-rbd-provisioner.1",
|
|
"key": "AgASB6ZoEIZ1IyAANmYrvZmHsseRtTb2iftoD0Jp3Un3Ob+6wa+HDW1Qh5I=",
|
|
"caps": {
|
|
"mgr": "allow rw",
|
|
"mon": "profile rbd, allow command 'osd blocklist'",
|
|
"osd": "profile rbd"
|
|
}
|
|
}
|
|
]
|
|
}`
|
|
)
|
|
|
|
func Test_adminRotationLock(t *testing.T) {
|
|
var err error
|
|
|
|
err = claimAdminRotationLock("ns")
|
|
assert.NoError(t, err)
|
|
|
|
err = claimAdminRotationLock("ns")
|
|
assert.Error(t, err)
|
|
|
|
err = claimAdminRotationLock("other")
|
|
assert.NoError(t, err)
|
|
|
|
releaseAdminRotationLock("ns")
|
|
|
|
err = claimAdminRotationLock("ns")
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, map[string]struct{}{
|
|
"ns": {},
|
|
"other": {},
|
|
}, adminRotationInProgress)
|
|
|
|
releaseAdminRotationLock("ns")
|
|
releaseAdminRotationLock("other")
|
|
|
|
assert.Equal(t, map[string]struct{}{}, adminRotationInProgress)
|
|
}
|