1142 lines
43 KiB
Go
1142 lines
43 KiB
Go
/*
|
|
Copyright 2025 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package cluster
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"slices"
|
|
"testing"
|
|
|
|
"github.com/coreos/pkg/capnslog"
|
|
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
|
"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
|
|
"github.com/rook/rook/pkg/clusterd"
|
|
"github.com/rook/rook/pkg/daemon/ceph/client"
|
|
"github.com/rook/rook/pkg/operator/ceph/version"
|
|
"github.com/rook/rook/pkg/operator/k8sutil"
|
|
testop "github.com/rook/rook/pkg/operator/test"
|
|
exectest "github.com/rook/rook/pkg/util/exec/test"
|
|
"github.com/stretchr/testify/assert"
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
)
|
|
|
|
func Test_genKeyring(t *testing.T) {
|
|
twoCapsKeyring := `[client.them]
|
|
key = themkey==
|
|
caps any = "thing"
|
|
`
|
|
adminRotatorKeyring := `[client.admin-rotator]
|
|
key = adminrotatorkey==
|
|
caps mds = "allow *"
|
|
caps mon = "allow *"
|
|
caps osd = "allow *"
|
|
caps mgr = "allow *"
|
|
`
|
|
|
|
tests := []struct {
|
|
name string
|
|
clientName string
|
|
clientKey string
|
|
clientCaps []string
|
|
wantErr bool
|
|
want string
|
|
}{
|
|
{"no caps", "client.me", "mekey==", []string{}, false, "[client.me]\n key = mekey==\n"},
|
|
{"no caps, no key", "client.me", "", []string{}, true, ""},
|
|
{"one cap", "client.you", "youkey==", []string{"bad"}, true, ""},
|
|
{"two caps", "client.them", "themkey==", []string{"any", "thing"}, false, twoCapsKeyring},
|
|
{"three caps", "client.batman", "batmankey==", []string{"some", "thing", "else"}, true, ""},
|
|
{"admin rotator caps", "client.admin-rotator", "adminrotatorkey==", adminKeyAccessCaps, false, adminRotatorKeyring},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got, err := genKeyring(tt.clientName, tt.clientKey, tt.clientCaps)
|
|
if (err != nil) != tt.wantErr {
|
|
t.Errorf("genKeyring() error = %v, wantErr %v", err, tt.wantErr)
|
|
return
|
|
}
|
|
if got != tt.want {
|
|
t.Errorf("genKeyring() = %v, want %v", got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func Test_admin_key_rotation(t *testing.T) {
|
|
logger.SetLevel(capnslog.DEBUG) // TESTING TESTING TESTING
|
|
|
|
reloadManagerCalled := false
|
|
reloadManagerFunc = func() {
|
|
reloadManagerCalled = true
|
|
}
|
|
|
|
// this feature requires commands to execute in a specific order and often with specific arg(s)
|
|
// it also requires commands to not be skipped
|
|
// set up an executor that allows tests to expect commands in a certain order
|
|
type expect struct {
|
|
cargs []string
|
|
withArgs []string
|
|
returnFunc func(cmd string, args ...string) (string, error)
|
|
}
|
|
numCalls := 0
|
|
expectList := []expect{}
|
|
|
|
mockExecutor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(cmd string, args ...string) (string, error) {
|
|
cargs := append([]string{cmd}, args...) // cmd+args in one slice for easy comparison
|
|
t.Logf("mock command: %v", cargs)
|
|
|
|
// get current command expectations
|
|
if len(expectList) <= numCalls {
|
|
e := fmt.Errorf("execution iteration=%d: no expectation set for called %v", numCalls, cargs)
|
|
t.Error(e)
|
|
return "", e
|
|
}
|
|
currentExpect := expectList[numCalls]
|
|
numCalls++
|
|
|
|
// assert current command matches expectations
|
|
if !isCommand(cargs, currentExpect.cargs...) {
|
|
e := fmt.Errorf("execution iteration=%d: expected command %v does not match called %v", numCalls, currentExpect.cargs, cargs)
|
|
t.Error(e)
|
|
return "", e
|
|
}
|
|
if len(currentExpect.withArgs) > 0 {
|
|
for _, arg := range currentExpect.withArgs {
|
|
if !slices.Contains(cargs, arg) {
|
|
e := fmt.Errorf("execution iteration=%d: expected arg %q missing in called %v", numCalls, arg, cargs)
|
|
t.Error(e)
|
|
return "", e
|
|
}
|
|
}
|
|
}
|
|
|
|
// expectations met, call return func with args
|
|
return currentExpect.returnFunc(cmd, args...)
|
|
},
|
|
}
|
|
|
|
tmpCfgDir, err := os.MkdirTemp("", "")
|
|
t.Logf("temp dir: %q", tmpCfgDir)
|
|
assert.NoError(t, err)
|
|
defer func() { os.RemoveAll(tmpCfgDir) }()
|
|
|
|
ns := "ns"
|
|
clusterName := "my-cluster"
|
|
|
|
newTest := func() (*clusterd.Context, *client.ClusterInfo, *k8sutil.OwnerInfo, *cephv1.CephCluster) {
|
|
reloadManagerCalled = false
|
|
numCalls = 0
|
|
expectList = []expect{} // clear expect list
|
|
|
|
clusterInfo := &client.ClusterInfo{
|
|
Context: context.TODO(),
|
|
FSID: "00000000-0000-0000-0000-000000000000",
|
|
Namespace: ns,
|
|
CephVersion: version.CephVersion{Major: 20, Minor: 3, Extra: 0},
|
|
MonitorSecret: "MONSECRET/SHOULDNOTCHANGE=",
|
|
CephCred: client.CephCred{
|
|
Username: "client.admin",
|
|
Secret: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
|
|
},
|
|
OwnerInfo: &k8sutil.OwnerInfo{},
|
|
}
|
|
clusterInfo.SetName(clusterName)
|
|
|
|
cephCluster := &cephv1.CephCluster{ // create via controller-runtime client
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: ns,
|
|
Name: clusterName,
|
|
},
|
|
Spec: cephv1.ClusterSpec{
|
|
Security: cephv1.ClusterSecuritySpec{
|
|
CephX: cephv1.ClusterCephxConfig{
|
|
Daemon: cephv1.CephxConfig{
|
|
KeyRotationPolicy: "KeyGeneration",
|
|
KeyGeneration: 2,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Status: cephv1.ClusterStatus{
|
|
Cephx: cephv1.ClusterCephxStatus{
|
|
Admin: cephv1.CephxStatus{
|
|
KeyGeneration: 1,
|
|
KeyCephVersion: "19.2.3-0",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
monSecret := &corev1.Secret{ // create via clientset
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: "ns",
|
|
Name: "rook-ceph-mon",
|
|
},
|
|
Data: map[string][]byte{
|
|
"ceph-secret": []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="), // before rotation key
|
|
"ceph-username": []byte("client.admin"),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
},
|
|
}
|
|
|
|
clientset := testop.New(t, 3)
|
|
_, err := clientset.CoreV1().Secrets(ns).Create(context.TODO(), monSecret, metav1.CreateOptions{})
|
|
if err != nil {
|
|
panic(err) // test setup failed
|
|
}
|
|
|
|
s := scheme.Scheme
|
|
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{})
|
|
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(cephCluster).Build()
|
|
|
|
clusterdCtx := &clusterd.Context{
|
|
Clientset: clientset,
|
|
Client: cl,
|
|
Executor: mockExecutor,
|
|
ConfigDir: tmpCfgDir,
|
|
}
|
|
|
|
return clusterdCtx, clusterInfo, clusterInfo.OwnerInfo, cephCluster
|
|
}
|
|
|
|
// args that the default clusterInfo should use
|
|
defaultClientArgs := []string{"--name=client.admin", "--keyring=" + tmpCfgDir + "/ns/client.admin.keyring"}
|
|
|
|
// args that the temporary admin-rotator user should use
|
|
tmpRotatorClientArgs := []string{"--name=client.admin-rotator", "--keyring=" + tmpCfgDir + "/ns/admin-rotate/client.admin-rotator.keyring"}
|
|
|
|
// args that the new admin client should use temporarily to verify it works before final storage
|
|
tmpAdminClientArgs := []string{"--name=client.admin", "--keyring=" + tmpCfgDir + "/ns/admin-rotate/client.admin.keyring"}
|
|
|
|
t.Run("happy path rotation", func(t *testing.T) {
|
|
clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest()
|
|
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authCreateAdminRotatorOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authRotateAdminOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpAdminClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotatorDeletion, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "del", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", nil
|
|
},
|
|
})
|
|
|
|
err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster)
|
|
assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation)
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.True(t, reloadManagerCalled)
|
|
|
|
cluster := cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated
|
|
assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated
|
|
|
|
monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData := map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // changed
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
|
|
// after successful rotation, recovery should not occur
|
|
err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, len(expectList), numCalls) // same number calls from before
|
|
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged
|
|
assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged
|
|
|
|
monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, wantMonSecretData, monSec.Data) // unchanged
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
})
|
|
|
|
t.Run("fail create admin-rotator", func(t *testing.T) {
|
|
clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest()
|
|
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", fmt.Errorf("induced error")
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
// auth generation tries to modify caps if auth get-or-create-key fails
|
|
cargs: []string{"ceph", "auth", "caps", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", fmt.Errorf("induced error")
|
|
},
|
|
})
|
|
|
|
err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster)
|
|
assert.ErrorContains(t, err, "induced error")
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.False(t, reloadManagerCalled)
|
|
|
|
cluster := cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged
|
|
assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged
|
|
|
|
monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData := map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="), // unchanged
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
// nothing to recover before admin-rotator created and stored in secret
|
|
err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, len(expectList), numCalls) // same number calls from before
|
|
|
|
monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, wantMonSecretData, monSec.Data) // unchanged
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
})
|
|
|
|
t.Run("fail ls as admin rotator", func(t *testing.T) {
|
|
clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest()
|
|
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authCreateAdminRotatorOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", fmt.Errorf("induced error")
|
|
},
|
|
})
|
|
|
|
err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster)
|
|
assert.ErrorContains(t, err, "induced error")
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.False(t, reloadManagerCalled)
|
|
|
|
cluster := cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged
|
|
assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged
|
|
|
|
monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData := map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="), // unchanged
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.NoError(t, err) // rotator secret now persisted
|
|
|
|
// expect recovery to proceed
|
|
numCalls = 0
|
|
expectList = []expect{} // clear expect list
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
// client.admin can still ls, and admin-rotator is present
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authRotateAdminOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpAdminClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotatorDeletion, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "del", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", nil
|
|
},
|
|
})
|
|
|
|
err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns)
|
|
assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation)
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.True(t, reloadManagerCalled)
|
|
|
|
cluster = cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated
|
|
assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated
|
|
|
|
monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData = map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // changed
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
|
|
t.Run("recovery doesn't reoccur after successful recovery", func(t *testing.T) {
|
|
// only need to test this in one of the unit tests
|
|
err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, len(expectList), numCalls) // same number calls from before
|
|
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged
|
|
assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged
|
|
|
|
monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, wantMonSecretData, monSec.Data) // unchanged
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
})
|
|
})
|
|
|
|
t.Run("fail auth rotate", func(t *testing.T) {
|
|
clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest()
|
|
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authCreateAdminRotatorOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", fmt.Errorf("induced error")
|
|
},
|
|
})
|
|
|
|
err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster)
|
|
assert.ErrorContains(t, err, "induced error")
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.False(t, reloadManagerCalled)
|
|
|
|
cluster := cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged
|
|
assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged
|
|
|
|
monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData := map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="), // unchanged
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.NoError(t, err) // rotator secret now persisted
|
|
|
|
// expect recovery to proceed
|
|
numCalls = 0
|
|
expectList = []expect{} // clear expect list
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
// client.admin can still ls, and admin-rotator is present
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authRotateAdminOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpAdminClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotatorDeletion, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "del", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", nil
|
|
},
|
|
})
|
|
|
|
err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns)
|
|
assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation)
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.True(t, reloadManagerCalled)
|
|
|
|
cluster = cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated
|
|
assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated
|
|
|
|
monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData = map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // changed
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
})
|
|
|
|
t.Run("fail auth ls as temp rotated client.admin", func(t *testing.T) {
|
|
clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest()
|
|
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authCreateAdminRotatorOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authRotateAdminOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpAdminClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", fmt.Errorf("induced error")
|
|
},
|
|
})
|
|
|
|
err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster)
|
|
assert.ErrorContains(t, err, "induced error")
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.False(t, reloadManagerCalled)
|
|
|
|
cluster := cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged
|
|
assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged
|
|
|
|
monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData := map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="), // unchanged
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.NoError(t, err) // rotator secret now persisted
|
|
|
|
// expect recovery to proceed
|
|
numCalls = 0
|
|
expectList = []expect{} // clear expect list
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", fmt.Errorf("client.admin key stored rook-ceph-mon secret now doesn't work")
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authRotateAdminOutput, nil // auth rotate happens again
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpAdminClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotatorDeletion, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "del", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", nil
|
|
},
|
|
})
|
|
|
|
err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns)
|
|
assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation)
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.True(t, reloadManagerCalled)
|
|
|
|
cluster = cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated
|
|
assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated
|
|
|
|
monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData = map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // changed
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
})
|
|
|
|
t.Run("fail auth del admin-rotator", func(t *testing.T) {
|
|
clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest()
|
|
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authCreateAdminRotatorOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authRotateAdminOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpAdminClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotatorDeletion, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "del", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", fmt.Errorf("induced error")
|
|
},
|
|
})
|
|
|
|
err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster)
|
|
assert.ErrorContains(t, err, "induced error")
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.False(t, reloadManagerCalled)
|
|
|
|
cluster := cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged
|
|
assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged
|
|
|
|
monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData := map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // IS UPDATED!
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.NoError(t, err) // rotator secret still persisted
|
|
|
|
// expect recovery to proceed
|
|
numCalls = 0
|
|
expectList = []expect{} // clear expect list
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
// client.admin key stored rook-ceph-mon secret works, but Rook can't know if that
|
|
// is because it is un-rotated or because auth del failed
|
|
return authLsOutputAfterRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authRotateAdminOutput, nil // auth rotate happens again
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpAdminClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotatorDeletion, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "del", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", nil
|
|
},
|
|
})
|
|
|
|
err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns)
|
|
assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation)
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.True(t, reloadManagerCalled)
|
|
|
|
cluster = cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated
|
|
assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated
|
|
|
|
monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData = map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // rotated value
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
})
|
|
|
|
t.Run("recover cleanup", func(t *testing.T) {
|
|
clusterdCtx, clusterInfo, ownerInfo, cephCluster := newTest()
|
|
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "get-or-create-key", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authCreateAdminRotatorOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputBeforeRotation, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "rotate", "client.admin"},
|
|
withArgs: tmpRotatorClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authRotateAdminOutput, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: tmpAdminClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return authLsOutputAfterRotatorDeletion, nil
|
|
},
|
|
})
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "del", "client.admin-rotator"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
return "", fmt.Errorf("induced error")
|
|
},
|
|
})
|
|
|
|
err := rotateAdminCephxKey(clusterdCtx, clusterInfo, ownerInfo, cephCluster)
|
|
assert.ErrorContains(t, err, "induced error")
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.False(t, reloadManagerCalled)
|
|
|
|
cluster := cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(1), cluster.Status.Cephx.Admin.KeyGeneration) // unchanged
|
|
assert.Equal(t, "19.2.3-0", cluster.Status.Cephx.Admin.KeyCephVersion) // unchanged
|
|
|
|
monSec, err := clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData := map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // IS UPDATED!
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.NoError(t, err) // rotator secret still persisted
|
|
|
|
// expect recovery to proceed
|
|
numCalls = 0
|
|
expectList = []expect{} // clear expect list
|
|
expectList = append(expectList, expect{
|
|
cargs: []string{"ceph", "auth", "ls"},
|
|
withArgs: defaultClientArgs,
|
|
returnFunc: func(cmd string, args ...string) (string, error) {
|
|
// pretend that auth del successfully deleted auth-rotator
|
|
return authLsOutputAfterRotatorDeletion, nil
|
|
},
|
|
})
|
|
// don't expect any more commands, but just check that secrets are correct
|
|
|
|
err = recoverPriorAdminCephxKeyRotation(clusterdCtx, clusterInfo, ownerInfo, ns)
|
|
assert.ErrorIs(t, err, errSuccessfulAdminKeyRotation)
|
|
assert.Equal(t, len(expectList), numCalls) // any expected calls not called?
|
|
assert.True(t, reloadManagerCalled)
|
|
|
|
cluster = cephv1.CephCluster{}
|
|
err = clusterdCtx.Client.Get(context.TODO(), clusterInfo.NamespacedName(), &cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(2), cluster.Status.Cephx.Admin.KeyGeneration) // updated
|
|
assert.Equal(t, "20.3.0-0", cluster.Status.Cephx.Admin.KeyCephVersion) // updated
|
|
|
|
monSec, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-mon", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
wantMonSecretData = map[string][]byte{
|
|
"mon-secret": []byte("MONSECRET/SHOULDNOTCHANGE="),
|
|
"fsid": []byte("00000000-0000-0000-0000-000000000000"),
|
|
"ceph-username": []byte("client.admin"),
|
|
"ceph-secret": []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="), // unchanged
|
|
}
|
|
assert.Equal(t, wantMonSecretData, monSec.Data)
|
|
|
|
_, err = clusterdCtx.Clientset.CoreV1().Secrets(ns).Get(context.TODO(), "rook-ceph-admin-rotator-keyring", metav1.GetOptions{})
|
|
assert.ErrorContains(t, err, "not found") // rotator keyring not persisted
|
|
})
|
|
}
|
|
|
|
// super simple helper to see if the command matches
|
|
func isCommand(cargs []string, lookFor ...string) bool {
|
|
for i, want := range lookFor {
|
|
if cargs[i] != want {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
const (
|
|
// `ceph auth get-or-create-key client.admin-rotator` output
|
|
authCreateAdminRotatorOutput = `{
|
|
"entity": "client.admin-rotator",
|
|
"key": "ADMINROTATORADMINROTATORADMINROTATORADMINROTATORADMINROTATO=",
|
|
"caps": {
|
|
"mds": "allow *",
|
|
"mgr": "allow *",
|
|
"mon": "allow *",
|
|
"osd": "allow *"
|
|
}
|
|
}`
|
|
|
|
// `ceph auth ls` output from before client.admin rotation (after admin-rotator user created)
|
|
// output pared down to a few entries to keep CI test more readable without being too trivial
|
|
authLsOutputBeforeRotation = `{
|
|
"auth_dump": [
|
|
{
|
|
"entity": "client.admin",
|
|
"key": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
|
|
"caps": {
|
|
"mds": "allow *",
|
|
"mgr": "allow *",
|
|
"mon": "allow *",
|
|
"osd": "allow *"
|
|
}
|
|
},
|
|
{
|
|
"entity": "client.admin-rotator",
|
|
"key": "ADMINROTATORADMINROTATORADMINROTATORADMINROTATORADMINROTATO=",
|
|
"caps": {
|
|
"mds": "allow *",
|
|
"mgr": "allow *",
|
|
"mon": "allow *",
|
|
"osd": "allow *"
|
|
}
|
|
},
|
|
{
|
|
"entity": "client.csi-rbd-node.1",
|
|
"key": "AgASB6ZohDE6MCAANiC1Nt0tE6IjQi5nPeiFKvaNZLcZcIJ1bgYfRQ6hlPE=",
|
|
"caps": {
|
|
"mgr": "allow rw",
|
|
"mon": "profile rbd",
|
|
"osd": "profile rbd"
|
|
}
|
|
},
|
|
{
|
|
"entity": "client.csi-rbd-provisioner.1",
|
|
"key": "AgASB6ZoEIZ1IyAANmYrvZmHsseRtTb2iftoD0Jp3Un3Ob+6wa+HDW1Qh5I=",
|
|
"caps": {
|
|
"mgr": "allow rw",
|
|
"mon": "profile rbd, allow command 'osd blocklist'",
|
|
"osd": "profile rbd"
|
|
}
|
|
}
|
|
]
|
|
}`
|
|
|
|
// `ceph auth rotate client.admin` output
|
|
authRotateAdminOutput = `[{
|
|
"entity": "client.admin",
|
|
"key": "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=",
|
|
"caps": {
|
|
"mds": "allow *",
|
|
"mgr": "allow *",
|
|
"mon": "allow *",
|
|
"osd": "allow *"
|
|
}
|
|
}]`
|
|
|
|
authLsOutputAfterRotation = `{
|
|
"auth_dump": [
|
|
{
|
|
"entity": "client.admin",
|
|
"key": "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=",
|
|
"caps": {
|
|
"mds": "allow *",
|
|
"mgr": "allow *",
|
|
"mon": "allow *",
|
|
"osd": "allow *"
|
|
}
|
|
},
|
|
{
|
|
"entity": "client.admin-rotator",
|
|
"key": "ADMINROTATORADMINROTATORADMINROTATORADMINROTATORADMINROTATO=",
|
|
"caps": {
|
|
"mds": "allow *",
|
|
"mgr": "allow *",
|
|
"mon": "allow *",
|
|
"osd": "allow *"
|
|
}
|
|
},
|
|
{
|
|
"entity": "client.csi-rbd-node.1",
|
|
"key": "AgASB6ZohDE6MCAANiC1Nt0tE6IjQi5nPeiFKvaNZLcZcIJ1bgYfRQ6hlPE=",
|
|
"caps": {
|
|
"mgr": "allow rw",
|
|
"mon": "profile rbd",
|
|
"osd": "profile rbd"
|
|
}
|
|
},
|
|
{
|
|
"entity": "client.csi-rbd-provisioner.1",
|
|
"key": "AgASB6ZoEIZ1IyAANmYrvZmHsseRtTb2iftoD0Jp3Un3Ob+6wa+HDW1Qh5I=",
|
|
"caps": {
|
|
"mgr": "allow rw",
|
|
"mon": "profile rbd, allow command 'osd blocklist'",
|
|
"osd": "profile rbd"
|
|
}
|
|
}
|
|
]
|
|
}`
|
|
|
|
// `ceph auth ls` output from after client.admin-rotator deleted
|
|
// output pared down to a few entries to keep CI test more readable without being too trivial
|
|
authLsOutputAfterRotatorDeletion = `{
|
|
"auth_dump": [
|
|
{
|
|
"entity": "client.admin",
|
|
"key": "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=",
|
|
"caps": {
|
|
"mds": "allow *",
|
|
"mgr": "allow *",
|
|
"mon": "allow *",
|
|
"osd": "allow *"
|
|
}
|
|
},
|
|
{
|
|
"entity": "client.csi-rbd-node.1",
|
|
"key": "AgASB6ZohDE6MCAANiC1Nt0tE6IjQi5nPeiFKvaNZLcZcIJ1bgYfRQ6hlPE=",
|
|
"caps": {
|
|
"mgr": "allow rw",
|
|
"mon": "profile rbd",
|
|
"osd": "profile rbd"
|
|
}
|
|
},
|
|
{
|
|
"entity": "client.csi-rbd-provisioner.1",
|
|
"key": "AgASB6ZoEIZ1IyAANmYrvZmHsseRtTb2iftoD0Jp3Un3Ob+6wa+HDW1Qh5I=",
|
|
"caps": {
|
|
"mgr": "allow rw",
|
|
"mon": "profile rbd, allow command 'osd blocklist'",
|
|
"osd": "profile rbd"
|
|
}
|
|
}
|
|
]
|
|
}`
|
|
)
|
|
|
|
func Test_adminRotationLock(t *testing.T) {
|
|
var err error
|
|
|
|
err = claimAdminRotationLock("ns")
|
|
assert.NoError(t, err)
|
|
|
|
err = claimAdminRotationLock("ns")
|
|
assert.Error(t, err)
|
|
|
|
err = claimAdminRotationLock("other")
|
|
assert.NoError(t, err)
|
|
|
|
releaseAdminRotationLock("ns")
|
|
|
|
err = claimAdminRotationLock("ns")
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, map[string]struct{}{
|
|
"ns": {},
|
|
"other": {},
|
|
}, adminRotationInProgress)
|
|
|
|
releaseAdminRotationLock("ns")
|
|
releaseAdminRotationLock("other")
|
|
|
|
assert.Equal(t, map[string]struct{}{}, adminRotationInProgress)
|
|
}
|