rgw: make CephObjectRealm controller idempotent
The CephObjectRealm controller would fail all subsequent reconciles if the first reconcile created the Kubernetes Secret containing the access keys for the realm but where the radosgw-admin command failed to create the realm. This was the only idempotency issue found after reviewing the CephObjectRealm controller. Resolves #8954 Signed-off-by: Blaine Gardner <blaine.gardner@redhat.com>
This commit is contained in:
5 files changed
+276
-25
No files matched your search
@@ -38,6 +38,7 @@ import (
|
||||
v1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
)
|
||||
|
||||
@@ -293,32 +294,34 @@ func DecodeSecret(secret *v1.Secret, keyName string) (string, error) {
|
||||
realmKey, ok := secret.Data[keyName]
|
||||
|
||||
if !ok {
|
||||
return "", errors.New("key was not in secret data")
|
||||
return "", errors.New(fmt.Sprintf("failed to find key %q in secret %q data. ", keyName, secret.Name) +
|
||||
"user likely created or modified the secret manually and should add the missing key back into the secret")
|
||||
}
|
||||
|
||||
return string(realmKey), nil
|
||||
}
|
||||
|
||||
func GetRealmKeyArgs(clusterdContext *clusterd.Context, realmName, namespace string) (string, string, error) {
|
||||
func GetRealmKeySecret(clusterdContext *clusterd.Context, realmName types.NamespacedName) (*v1.Secret, error) {
|
||||
ctx := context.TODO()
|
||||
logger.Debugf("getting keys for realm %v", realmName)
|
||||
// get realm's access and secret keys
|
||||
realmSecretName := realmName + "-keys"
|
||||
realmSecret, err := clusterdContext.Clientset.CoreV1().Secrets(namespace).Get(ctx, realmSecretName, metav1.GetOptions{})
|
||||
realmSecretName := realmName.Name + "-keys"
|
||||
realmSecret, err := clusterdContext.Clientset.CoreV1().Secrets(realmName.Namespace).Get(ctx, realmSecretName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return "", "", errors.Wrapf(err, "failed to get realm %q keys secret", realmName)
|
||||
return nil, errors.Wrapf(err, "failed to get CephObjectRealm %q keys secret", realmName.String())
|
||||
}
|
||||
logger.Debugf("found keys secret for realm %v", realmName)
|
||||
logger.Debugf("found keys secret for CephObjectRealm %q", realmName.String())
|
||||
return realmSecret, nil
|
||||
}
|
||||
|
||||
func GetRealmKeyArgsFromSecret(realmSecret *v1.Secret, realmName types.NamespacedName) (string, string, error) {
|
||||
accessKey, err := DecodeSecret(realmSecret, AccessKeyName)
|
||||
if err != nil {
|
||||
return "", "", errors.Wrapf(err, "failed to decode realm %q access key", realmName)
|
||||
return "", "", errors.Wrapf(err, "failed to decode CephObjectRealm %q access key from secret %q", realmName.String(), realmSecret.Name)
|
||||
}
|
||||
secretKey, err := DecodeSecret(realmSecret, SecretKeyName)
|
||||
if err != nil {
|
||||
return "", "", errors.Wrapf(err, "failed to decode realm %q access key", realmName)
|
||||
return "", "", errors.Wrapf(err, "failed to decode CephObjectRealm %q access key from secret %q", realmName.String(), realmSecret.Name)
|
||||
}
|
||||
logger.Debugf("decoded keys for realm %v", realmName)
|
||||
logger.Debugf("decoded keys for realm %q", realmName.String())
|
||||
|
||||
accessKeyArg := fmt.Sprintf("--access-key=%s", accessKey)
|
||||
secretKeyArg := fmt.Sprintf("--secret-key=%s", secretKey)
|
||||
@@ -326,6 +329,18 @@ func GetRealmKeyArgs(clusterdContext *clusterd.Context, realmName, namespace str
|
||||
return accessKeyArg, secretKeyArg, nil
|
||||
}
|
||||
|
||||
func GetRealmKeyArgs(clusterdContext *clusterd.Context, realmName, namespace string) (string, string, error) {
|
||||
realmNsName := types.NamespacedName{Namespace: namespace, Name: realmName}
|
||||
logger.Debugf("getting keys for realm %q", realmNsName.String())
|
||||
|
||||
secret, err := GetRealmKeySecret(clusterdContext, realmNsName)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
return GetRealmKeyArgsFromSecret(secret, realmNsName)
|
||||
}
|
||||
|
||||
func getZoneEndpoints(objContext *Context, serviceEndpoint string) ([]string, error) {
|
||||
logger.Debugf("getting current endpoints for zone %v", objContext.Zone)
|
||||
realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm)
|
||||
|
||||
@@ -32,6 +32,10 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
exectest "github.com/rook/rook/pkg/util/exec/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
k8sfake "k8s.io/client-go/kubernetes/fake"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -550,3 +554,152 @@ func Test_createMultisite(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRealmKeySecret(t *testing.T) {
|
||||
ns := "my-ns"
|
||||
realmName := "my-realm"
|
||||
|
||||
t.Run("secret exists", func(t *testing.T) {
|
||||
secret := &v1.Secret{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "Secret",
|
||||
APIVersion: v1.SchemeGroupVersion.String(),
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: ns,
|
||||
Name: realmName + "-keys",
|
||||
},
|
||||
// should not care about data presence just to get the secret
|
||||
}
|
||||
|
||||
c := &clusterd.Context{
|
||||
Clientset: k8sfake.NewSimpleClientset(secret),
|
||||
}
|
||||
|
||||
secret, err := GetRealmKeySecret(c, types.NamespacedName{Namespace: ns, Name: realmName})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, secret)
|
||||
})
|
||||
|
||||
t.Run("secret doesn't exist", func(t *testing.T) {
|
||||
c := &clusterd.Context{
|
||||
Clientset: k8sfake.NewSimpleClientset(),
|
||||
}
|
||||
|
||||
secret, err := GetRealmKeySecret(c, types.NamespacedName{Namespace: ns, Name: realmName})
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, secret)
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetRealmKeyArgsFromSecret(t *testing.T) {
|
||||
ns := "my-ns"
|
||||
realmName := "my-realm"
|
||||
realmNsName := types.NamespacedName{Namespace: ns, Name: realmName}
|
||||
|
||||
baseSecret := &v1.Secret{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "Secret",
|
||||
APIVersion: v1.SchemeGroupVersion.String(),
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: ns,
|
||||
Name: realmName + "-keys",
|
||||
},
|
||||
Data: map[string][]byte{},
|
||||
}
|
||||
|
||||
t.Run("all secret data exists", func(t *testing.T) {
|
||||
s := baseSecret.DeepCopy()
|
||||
s.Data["access-key"] = []byte("my-access-key")
|
||||
s.Data["secret-key"] = []byte("my-secret-key")
|
||||
|
||||
access, secret, err := GetRealmKeyArgsFromSecret(s, realmNsName)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "--access-key=my-access-key", access)
|
||||
assert.Equal(t, "--secret-key=my-secret-key", secret)
|
||||
})
|
||||
|
||||
t.Run("access-key missing", func(t *testing.T) {
|
||||
s := baseSecret.DeepCopy()
|
||||
// missing s.Data["access-key"]
|
||||
s.Data["secret-key"] = []byte("my-secret-key")
|
||||
|
||||
access, secret, err := GetRealmKeyArgsFromSecret(s, realmNsName)
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, "", access)
|
||||
assert.Equal(t, "", secret)
|
||||
})
|
||||
|
||||
t.Run("secret-key missing", func(t *testing.T) {
|
||||
s := baseSecret.DeepCopy()
|
||||
s.Data["access-key"] = []byte("my-access-key")
|
||||
// missing s.Data["secret-key"]
|
||||
|
||||
access, secret, err := GetRealmKeyArgsFromSecret(s, realmNsName)
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, "", access)
|
||||
assert.Equal(t, "", secret)
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetRealmKeyArgs(t *testing.T) {
|
||||
ns := "my-ns"
|
||||
realmName := "my-realm"
|
||||
|
||||
baseSecret := &v1.Secret{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "Secret",
|
||||
APIVersion: v1.SchemeGroupVersion.String(),
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: ns,
|
||||
Name: realmName + "-keys",
|
||||
},
|
||||
Data: map[string][]byte{},
|
||||
}
|
||||
|
||||
// No need to test every case since this is a combination of GetRealmKeySecret and
|
||||
// GetRealmKeyArgsFromSecret and those are both thoroughly unit tested. Just check the success
|
||||
// case and cases where either sub-function fails.
|
||||
|
||||
t.Run("secret exists with all data", func(t *testing.T) {
|
||||
s := baseSecret.DeepCopy()
|
||||
s.Data["access-key"] = []byte("my-access-key")
|
||||
s.Data["secret-key"] = []byte("my-secret-key")
|
||||
|
||||
c := &clusterd.Context{
|
||||
Clientset: k8sfake.NewSimpleClientset(s),
|
||||
}
|
||||
|
||||
access, secret, err := GetRealmKeyArgs(c, realmName, ns)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "--access-key=my-access-key", access)
|
||||
assert.Equal(t, "--secret-key=my-secret-key", secret)
|
||||
})
|
||||
|
||||
t.Run("secret doesn't exist", func(t *testing.T) {
|
||||
c := &clusterd.Context{
|
||||
Clientset: k8sfake.NewSimpleClientset(),
|
||||
}
|
||||
|
||||
access, secret, err := GetRealmKeyArgs(c, realmName, ns)
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, "", access)
|
||||
assert.Equal(t, "", secret)
|
||||
})
|
||||
|
||||
t.Run("secret exists but is missing data", func(t *testing.T) {
|
||||
s := baseSecret.DeepCopy()
|
||||
// missing all data
|
||||
|
||||
c := &clusterd.Context{
|
||||
Clientset: k8sfake.NewSimpleClientset(s),
|
||||
}
|
||||
|
||||
access, secret, err := GetRealmKeyArgs(c, realmName, ns)
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, "", access)
|
||||
assert.Equal(t, "", secret)
|
||||
})
|
||||
}
|
||||
@@ -120,7 +120,7 @@ func (r *ReconcileObjectRealm) Reconcile(context context.Context, request reconc
|
||||
// workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface
|
||||
reconcileResponse, err := r.reconcile(request)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to reconcile: %v", err)
|
||||
logger.Errorf("failed to reconcile CephObjectRealm %q. %v", request.NamespacedName.String(), err)
|
||||
}
|
||||
|
||||
return reconcileResponse, err
|
||||
@@ -132,7 +132,7 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R
|
||||
err := r.client.Get(r.opManagerContext, request.NamespacedName, cephObjectRealm)
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Debug("CephObjectRealm resource not found. Ignoring since object must be deleted.")
|
||||
logger.Debug("CephObjectRealm %q resource not found. Ignoring since object must be deleted", request.NamespacedName.String())
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
// Error reading the object - requeue the request.
|
||||
@@ -181,7 +181,7 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R
|
||||
|
||||
// Create/Pull Ceph Realm
|
||||
if cephObjectRealm.Spec.IsPullRealm() {
|
||||
logger.Debug("pull section in spec found")
|
||||
logger.Debug("pull section in realm %q spec found", request.NamespacedName)
|
||||
_, err = r.pullCephRealm(cephObjectRealm)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, err
|
||||
@@ -202,14 +202,14 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R
|
||||
r.updateStatus(r.client, request.NamespacedName, k8sutil.ReadyStatus)
|
||||
|
||||
// Return and do not requeue
|
||||
logger.Debug("realm done reconciling")
|
||||
logger.Debug("realm %q done reconciling", request.NamespacedName)
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
func (r *ReconcileObjectRealm) pullCephRealm(realm *cephv1.CephObjectRealm) (reconcile.Result, error) {
|
||||
realmArg := fmt.Sprintf("--rgw-realm=%s", realm.Name)
|
||||
urlArg := fmt.Sprintf("--url=%s", realm.Spec.Pull.Endpoint)
|
||||
logger.Debug("getting keys to pull realm")
|
||||
logger.Debug("getting keys to pull realm for CephObjectRealm %q", realm.Name)
|
||||
accessKeyArg, secretKeyArg, err := object.GetRealmKeyArgs(r.context, realm.Name, realm.Namespace)
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
@@ -217,7 +217,7 @@ func (r *ReconcileObjectRealm) pullCephRealm(realm *cephv1.CephObjectRealm) (rec
|
||||
}
|
||||
return waitForRequeueIfRealmNotReady, errors.Wrap(err, "failed to get keys for realm")
|
||||
}
|
||||
logger.Debugf("keys found to pull realm, getting ready to pull from endpoint %q", realm.Spec.Pull.Endpoint)
|
||||
logger.Debugf("keys found to pull realm for CephObjectRealm %q, getting ready to pull from endpoint %q", realm.Name, realm.Spec.Pull.Endpoint)
|
||||
|
||||
objContext := object.NewContext(r.context, r.clusterInfo, realm.Name)
|
||||
output, err := object.RunAdminCommandNoMultisite(objContext, false, "realm", "pull", realmArg, urlArg, accessKeyArg, secretKeyArg)
|
||||
@@ -253,7 +253,21 @@ func (r *ReconcileObjectRealm) createCephRealm(realm *cephv1.CephObjectRealm) (r
|
||||
}
|
||||
|
||||
func (r *ReconcileObjectRealm) createRealmKeys(realm *cephv1.CephObjectRealm) (reconcile.Result, error) {
|
||||
logger.Debugf("generating access and secret keys for new realm %q", realm.Name)
|
||||
realmName := types.NamespacedName{Namespace: realm.Namespace, Name: realm.Name}
|
||||
logger.Debugf("generating access and secret keys for new realm %q", realmName.String())
|
||||
secretName := realm.Name + "-keys"
|
||||
|
||||
// Check if the secret exists first, and check that it has the access information needed.
|
||||
secret, err := object.GetRealmKeySecret(r.context, realmName)
|
||||
if err == nil {
|
||||
// secret exists, now verify access info. We don't need the args, but we do want to get the
|
||||
// error if the args can't be built
|
||||
_, _, err := object.GetRealmKeyArgsFromSecret(secret, realmName)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, errors.Wrapf(err, "failed to ensure secret keys for CephObjectRealm %q", realmName.String())
|
||||
}
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
// the realm's secret key and access key are randomly generated and then encoded to base64
|
||||
accessKey, err := mgr.GeneratePassword(accessKeyLength)
|
||||
@@ -275,8 +289,7 @@ func (r *ReconcileObjectRealm) createRealmKeys(realm *cephv1.CephObjectRealm) (r
|
||||
object.SecretKeyName: []byte(secretKey),
|
||||
}
|
||||
|
||||
secretName := realm.Name + "-keys"
|
||||
secret := &v1.Secret{
|
||||
secret = &v1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: secretName,
|
||||
Namespace: realm.Namespace,
|
||||
@@ -292,7 +305,7 @@ func (r *ReconcileObjectRealm) createRealmKeys(realm *cephv1.CephObjectRealm) (r
|
||||
if _, err = r.context.Clientset.CoreV1().Secrets(realm.Namespace).Create(r.opManagerContext, secret, metav1.CreateOptions{}); err != nil {
|
||||
return reconcile.Result{}, errors.Wrap(err, "failed to save rgw secrets")
|
||||
}
|
||||
logger.Infof("secrets for keys have been created for realm %q", realm.Name)
|
||||
logger.Infof("secret keys have been created for realm %q in secret %q", realm.Name, secret.Name)
|
||||
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
@@ -318,7 +331,7 @@ func (r *ReconcileObjectRealm) updateStatus(client client.Client, name types.Nam
|
||||
objectRealm := &cephv1.CephObjectRealm{}
|
||||
if err := client.Get(r.opManagerContext, name, objectRealm); err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Debug("CephObjectRealm resource not found. Ignoring since object must be deleted.")
|
||||
logger.Debug("CephObjectRealm %q resource not found. Ignoring since object must be deleted", name)
|
||||
return
|
||||
}
|
||||
logger.Warningf("failed to retrieve object realm %q to update status to %q. %v", name, status, err)
|
||||
|
||||
@@ -24,17 +24,17 @@ import (
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
|
||||
"github.com/rook/rook/pkg/operator/test"
|
||||
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/operator/test"
|
||||
exectest "github.com/rook/rook/pkg/util/exec/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
k8sfake "k8s.io/client-go/kubernetes/fake"
|
||||
"k8s.io/client-go/kubernetes/scheme"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
@@ -260,3 +260,73 @@ func getObjectRealmAndReconcileObjectRealm(t *testing.T) (*ReconcileObjectRealm,
|
||||
|
||||
return r, objectRealm
|
||||
}
|
||||
|
||||
func TestReconcileObjectRealm_createRealmKeys(t *testing.T) {
|
||||
ctx := context.TODO()
|
||||
realmName := "my-realm"
|
||||
ns := "my-ns"
|
||||
|
||||
scheme := scheme.Scheme
|
||||
scheme.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectRealm{}, &cephv1.CephCluster{}, &cephv1.CephClusterList{})
|
||||
|
||||
realm := &cephv1.CephObjectRealm{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: realmName,
|
||||
Namespace: ns,
|
||||
},
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "CephObjectRealm",
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("should be idempotent", func(t *testing.T) {
|
||||
r := ReconcileObjectRealm{
|
||||
context: &clusterd.Context{
|
||||
Clientset: k8sfake.NewSimpleClientset(),
|
||||
},
|
||||
scheme: scheme,
|
||||
}
|
||||
|
||||
for _, tName := range []string{"first reconcile", "second reconcile"} {
|
||||
// the output should be the same on the first and subsequent reconciles
|
||||
t.Run(tName, func(t *testing.T) {
|
||||
res, err := r.createRealmKeys(realm)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, res.IsZero())
|
||||
|
||||
secret, err := r.context.Clientset.CoreV1().Secrets(ns).Get(ctx, realmName+"-keys", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.Contains(t, secret.Data, "access-key")
|
||||
assert.Contains(t, secret.Data, "secret-key")
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("should fail if the secret doesn't have the necessary keys", func(t *testing.T) {
|
||||
secret := &v1.Secret{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "Secret",
|
||||
APIVersion: v1.SchemeGroupVersion.String(),
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: ns,
|
||||
Name: realmName + "-keys",
|
||||
},
|
||||
Data: map[string][]byte{
|
||||
"access-key": []byte("my-access-key"),
|
||||
// missing "secret-key"
|
||||
},
|
||||
}
|
||||
|
||||
r := ReconcileObjectRealm{
|
||||
context: &clusterd.Context{
|
||||
Clientset: k8sfake.NewSimpleClientset(secret),
|
||||
},
|
||||
scheme: scheme,
|
||||
}
|
||||
|
||||
_, err := r.createRealmKeys(realm)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "user likely created or modified the secret manually and should add the missing key back into the secret")
|
||||
})
|
||||
}
|
||||
@@ -282,7 +282,7 @@ function wait_for_operator_log_message() {
|
||||
|
||||
function restart_operator () {
|
||||
local namespace="${1:-rook-ceph}" # optional param 1: the namespace of the CephCluster (default: rook-ceph)
|
||||
kubectl --namespace "$namespace" delete pod --selector app=rook-ceph=operator
|
||||
kubectl --namespace "$namespace" delete pod --selector app=rook-ceph-operator
|
||||
# wait for new pod to be running
|
||||
get_pod_cmd=(kubectl --namespace "$namespace" get pod --selector app=rook-ceph-operator --no-headers)
|
||||
timeout 20 bash -c \
|
||||
|
||||
Reference in new issue
Block a user