From 65c49720385ece5295f09723ea2bce38cf1f13f0 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Tue, 12 Oct 2021 13:08:36 -0600 Subject: [PATCH] rgw: make CephObjectRealm controller idempotent The CephObjectRealm controller would fail all subsequent reconciles if the first reconcile created the Kubernetes Secret containing the access keys for the realm but where the radosgw-admin command failed to create the realm. This was the only idempotency issue found after reviewing the CephObjectRealm controller. Resolves #8954 Signed-off-by: Blaine Gardner --- pkg/operator/ceph/object/objectstore.go | 37 +++-- pkg/operator/ceph/object/objectstore_test.go | 153 ++++++++++++++++++ pkg/operator/ceph/object/realm/controller.go | 35 ++-- .../ceph/object/realm/controller_test.go | 74 ++++++++- tests/scripts/github-action-helper.sh | 2 +- 5 files changed, 276 insertions(+), 25 deletions(-) diff --git a/pkg/operator/ceph/object/objectstore.go b/pkg/operator/ceph/object/objectstore.go index 7356423e3..73ee510fc 100644 --- a/pkg/operator/ceph/object/objectstore.go +++ b/pkg/operator/ceph/object/objectstore.go @@ -38,6 +38,7 @@ import ( v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" ) @@ -293,32 +294,34 @@ func DecodeSecret(secret *v1.Secret, keyName string) (string, error) { realmKey, ok := secret.Data[keyName] if !ok { - return "", errors.New("key was not in secret data") + return "", errors.New(fmt.Sprintf("failed to find key %q in secret %q data. ", keyName, secret.Name) + + "user likely created or modified the secret manually and should add the missing key back into the secret") } return string(realmKey), nil } -func GetRealmKeyArgs(clusterdContext *clusterd.Context, realmName, namespace string) (string, string, error) { +func GetRealmKeySecret(clusterdContext *clusterd.Context, realmName types.NamespacedName) (*v1.Secret, error) { ctx := context.TODO() - logger.Debugf("getting keys for realm %v", realmName) - // get realm's access and secret keys - realmSecretName := realmName + "-keys" - realmSecret, err := clusterdContext.Clientset.CoreV1().Secrets(namespace).Get(ctx, realmSecretName, metav1.GetOptions{}) + realmSecretName := realmName.Name + "-keys" + realmSecret, err := clusterdContext.Clientset.CoreV1().Secrets(realmName.Namespace).Get(ctx, realmSecretName, metav1.GetOptions{}) if err != nil { - return "", "", errors.Wrapf(err, "failed to get realm %q keys secret", realmName) + return nil, errors.Wrapf(err, "failed to get CephObjectRealm %q keys secret", realmName.String()) } - logger.Debugf("found keys secret for realm %v", realmName) + logger.Debugf("found keys secret for CephObjectRealm %q", realmName.String()) + return realmSecret, nil +} +func GetRealmKeyArgsFromSecret(realmSecret *v1.Secret, realmName types.NamespacedName) (string, string, error) { accessKey, err := DecodeSecret(realmSecret, AccessKeyName) if err != nil { - return "", "", errors.Wrapf(err, "failed to decode realm %q access key", realmName) + return "", "", errors.Wrapf(err, "failed to decode CephObjectRealm %q access key from secret %q", realmName.String(), realmSecret.Name) } secretKey, err := DecodeSecret(realmSecret, SecretKeyName) if err != nil { - return "", "", errors.Wrapf(err, "failed to decode realm %q access key", realmName) + return "", "", errors.Wrapf(err, "failed to decode CephObjectRealm %q access key from secret %q", realmName.String(), realmSecret.Name) } - logger.Debugf("decoded keys for realm %v", realmName) + logger.Debugf("decoded keys for realm %q", realmName.String()) accessKeyArg := fmt.Sprintf("--access-key=%s", accessKey) secretKeyArg := fmt.Sprintf("--secret-key=%s", secretKey) @@ -326,6 +329,18 @@ func GetRealmKeyArgs(clusterdContext *clusterd.Context, realmName, namespace str return accessKeyArg, secretKeyArg, nil } +func GetRealmKeyArgs(clusterdContext *clusterd.Context, realmName, namespace string) (string, string, error) { + realmNsName := types.NamespacedName{Namespace: namespace, Name: realmName} + logger.Debugf("getting keys for realm %q", realmNsName.String()) + + secret, err := GetRealmKeySecret(clusterdContext, realmNsName) + if err != nil { + return "", "", err + } + + return GetRealmKeyArgsFromSecret(secret, realmNsName) +} + func getZoneEndpoints(objContext *Context, serviceEndpoint string) ([]string, error) { logger.Debugf("getting current endpoints for zone %v", objContext.Zone) realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm) diff --git a/pkg/operator/ceph/object/objectstore_test.go b/pkg/operator/ceph/object/objectstore_test.go index 51cbec7c4..0ee96831e 100644 --- a/pkg/operator/ceph/object/objectstore_test.go +++ b/pkg/operator/ceph/object/objectstore_test.go @@ -32,6 +32,10 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" ) const ( @@ -550,3 +554,152 @@ func Test_createMultisite(t *testing.T) { }) } } + +func TestGetRealmKeySecret(t *testing.T) { + ns := "my-ns" + realmName := "my-realm" + + t.Run("secret exists", func(t *testing.T) { + secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: v1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: realmName + "-keys", + }, + // should not care about data presence just to get the secret + } + + c := &clusterd.Context{ + Clientset: k8sfake.NewSimpleClientset(secret), + } + + secret, err := GetRealmKeySecret(c, types.NamespacedName{Namespace: ns, Name: realmName}) + assert.NoError(t, err) + assert.NotNil(t, secret) + }) + + t.Run("secret doesn't exist", func(t *testing.T) { + c := &clusterd.Context{ + Clientset: k8sfake.NewSimpleClientset(), + } + + secret, err := GetRealmKeySecret(c, types.NamespacedName{Namespace: ns, Name: realmName}) + assert.Error(t, err) + assert.Nil(t, secret) + }) +} + +func TestGetRealmKeyArgsFromSecret(t *testing.T) { + ns := "my-ns" + realmName := "my-realm" + realmNsName := types.NamespacedName{Namespace: ns, Name: realmName} + + baseSecret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: v1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: realmName + "-keys", + }, + Data: map[string][]byte{}, + } + + t.Run("all secret data exists", func(t *testing.T) { + s := baseSecret.DeepCopy() + s.Data["access-key"] = []byte("my-access-key") + s.Data["secret-key"] = []byte("my-secret-key") + + access, secret, err := GetRealmKeyArgsFromSecret(s, realmNsName) + assert.NoError(t, err) + assert.Equal(t, "--access-key=my-access-key", access) + assert.Equal(t, "--secret-key=my-secret-key", secret) + }) + + t.Run("access-key missing", func(t *testing.T) { + s := baseSecret.DeepCopy() + // missing s.Data["access-key"] + s.Data["secret-key"] = []byte("my-secret-key") + + access, secret, err := GetRealmKeyArgsFromSecret(s, realmNsName) + assert.Error(t, err) + assert.Equal(t, "", access) + assert.Equal(t, "", secret) + }) + + t.Run("secret-key missing", func(t *testing.T) { + s := baseSecret.DeepCopy() + s.Data["access-key"] = []byte("my-access-key") + // missing s.Data["secret-key"] + + access, secret, err := GetRealmKeyArgsFromSecret(s, realmNsName) + assert.Error(t, err) + assert.Equal(t, "", access) + assert.Equal(t, "", secret) + }) +} + +func TestGetRealmKeyArgs(t *testing.T) { + ns := "my-ns" + realmName := "my-realm" + + baseSecret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: v1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: realmName + "-keys", + }, + Data: map[string][]byte{}, + } + + // No need to test every case since this is a combination of GetRealmKeySecret and + // GetRealmKeyArgsFromSecret and those are both thoroughly unit tested. Just check the success + // case and cases where either sub-function fails. + + t.Run("secret exists with all data", func(t *testing.T) { + s := baseSecret.DeepCopy() + s.Data["access-key"] = []byte("my-access-key") + s.Data["secret-key"] = []byte("my-secret-key") + + c := &clusterd.Context{ + Clientset: k8sfake.NewSimpleClientset(s), + } + + access, secret, err := GetRealmKeyArgs(c, realmName, ns) + assert.NoError(t, err) + assert.Equal(t, "--access-key=my-access-key", access) + assert.Equal(t, "--secret-key=my-secret-key", secret) + }) + + t.Run("secret doesn't exist", func(t *testing.T) { + c := &clusterd.Context{ + Clientset: k8sfake.NewSimpleClientset(), + } + + access, secret, err := GetRealmKeyArgs(c, realmName, ns) + assert.Error(t, err) + assert.Equal(t, "", access) + assert.Equal(t, "", secret) + }) + + t.Run("secret exists but is missing data", func(t *testing.T) { + s := baseSecret.DeepCopy() + // missing all data + + c := &clusterd.Context{ + Clientset: k8sfake.NewSimpleClientset(s), + } + + access, secret, err := GetRealmKeyArgs(c, realmName, ns) + assert.Error(t, err) + assert.Equal(t, "", access) + assert.Equal(t, "", secret) + }) +} diff --git a/pkg/operator/ceph/object/realm/controller.go b/pkg/operator/ceph/object/realm/controller.go index 2aa3af1a7..43b36637d 100644 --- a/pkg/operator/ceph/object/realm/controller.go +++ b/pkg/operator/ceph/object/realm/controller.go @@ -120,7 +120,7 @@ func (r *ReconcileObjectRealm) Reconcile(context context.Context, request reconc // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { - logger.Errorf("failed to reconcile: %v", err) + logger.Errorf("failed to reconcile CephObjectRealm %q. %v", request.NamespacedName.String(), err) } return reconcileResponse, err @@ -132,7 +132,7 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R err := r.client.Get(r.opManagerContext, request.NamespacedName, cephObjectRealm) if err != nil { if kerrors.IsNotFound(err) { - logger.Debug("CephObjectRealm resource not found. Ignoring since object must be deleted.") + logger.Debug("CephObjectRealm %q resource not found. Ignoring since object must be deleted", request.NamespacedName.String()) return reconcile.Result{}, nil } // Error reading the object - requeue the request. @@ -181,7 +181,7 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R // Create/Pull Ceph Realm if cephObjectRealm.Spec.IsPullRealm() { - logger.Debug("pull section in spec found") + logger.Debug("pull section in realm %q spec found", request.NamespacedName) _, err = r.pullCephRealm(cephObjectRealm) if err != nil { return reconcile.Result{}, err @@ -202,14 +202,14 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R r.updateStatus(r.client, request.NamespacedName, k8sutil.ReadyStatus) // Return and do not requeue - logger.Debug("realm done reconciling") + logger.Debug("realm %q done reconciling", request.NamespacedName) return reconcile.Result{}, nil } func (r *ReconcileObjectRealm) pullCephRealm(realm *cephv1.CephObjectRealm) (reconcile.Result, error) { realmArg := fmt.Sprintf("--rgw-realm=%s", realm.Name) urlArg := fmt.Sprintf("--url=%s", realm.Spec.Pull.Endpoint) - logger.Debug("getting keys to pull realm") + logger.Debug("getting keys to pull realm for CephObjectRealm %q", realm.Name) accessKeyArg, secretKeyArg, err := object.GetRealmKeyArgs(r.context, realm.Name, realm.Namespace) if err != nil { if kerrors.IsNotFound(err) { @@ -217,7 +217,7 @@ func (r *ReconcileObjectRealm) pullCephRealm(realm *cephv1.CephObjectRealm) (rec } return waitForRequeueIfRealmNotReady, errors.Wrap(err, "failed to get keys for realm") } - logger.Debugf("keys found to pull realm, getting ready to pull from endpoint %q", realm.Spec.Pull.Endpoint) + logger.Debugf("keys found to pull realm for CephObjectRealm %q, getting ready to pull from endpoint %q", realm.Name, realm.Spec.Pull.Endpoint) objContext := object.NewContext(r.context, r.clusterInfo, realm.Name) output, err := object.RunAdminCommandNoMultisite(objContext, false, "realm", "pull", realmArg, urlArg, accessKeyArg, secretKeyArg) @@ -253,7 +253,21 @@ func (r *ReconcileObjectRealm) createCephRealm(realm *cephv1.CephObjectRealm) (r } func (r *ReconcileObjectRealm) createRealmKeys(realm *cephv1.CephObjectRealm) (reconcile.Result, error) { - logger.Debugf("generating access and secret keys for new realm %q", realm.Name) + realmName := types.NamespacedName{Namespace: realm.Namespace, Name: realm.Name} + logger.Debugf("generating access and secret keys for new realm %q", realmName.String()) + secretName := realm.Name + "-keys" + + // Check if the secret exists first, and check that it has the access information needed. + secret, err := object.GetRealmKeySecret(r.context, realmName) + if err == nil { + // secret exists, now verify access info. We don't need the args, but we do want to get the + // error if the args can't be built + _, _, err := object.GetRealmKeyArgsFromSecret(secret, realmName) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to ensure secret keys for CephObjectRealm %q", realmName.String()) + } + return reconcile.Result{}, nil + } // the realm's secret key and access key are randomly generated and then encoded to base64 accessKey, err := mgr.GeneratePassword(accessKeyLength) @@ -275,8 +289,7 @@ func (r *ReconcileObjectRealm) createRealmKeys(realm *cephv1.CephObjectRealm) (r object.SecretKeyName: []byte(secretKey), } - secretName := realm.Name + "-keys" - secret := &v1.Secret{ + secret = &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: secretName, Namespace: realm.Namespace, @@ -292,7 +305,7 @@ func (r *ReconcileObjectRealm) createRealmKeys(realm *cephv1.CephObjectRealm) (r if _, err = r.context.Clientset.CoreV1().Secrets(realm.Namespace).Create(r.opManagerContext, secret, metav1.CreateOptions{}); err != nil { return reconcile.Result{}, errors.Wrap(err, "failed to save rgw secrets") } - logger.Infof("secrets for keys have been created for realm %q", realm.Name) + logger.Infof("secret keys have been created for realm %q in secret %q", realm.Name, secret.Name) return reconcile.Result{}, nil } @@ -318,7 +331,7 @@ func (r *ReconcileObjectRealm) updateStatus(client client.Client, name types.Nam objectRealm := &cephv1.CephObjectRealm{} if err := client.Get(r.opManagerContext, name, objectRealm); err != nil { if kerrors.IsNotFound(err) { - logger.Debug("CephObjectRealm resource not found. Ignoring since object must be deleted.") + logger.Debug("CephObjectRealm %q resource not found. Ignoring since object must be deleted", name) return } logger.Warningf("failed to retrieve object realm %q to update status to %q. %v", name, status, err) diff --git a/pkg/operator/ceph/object/realm/controller_test.go b/pkg/operator/ceph/object/realm/controller_test.go index 057271642..73ba09ade 100644 --- a/pkg/operator/ceph/object/realm/controller_test.go +++ b/pkg/operator/ceph/object/realm/controller_test.go @@ -24,17 +24,17 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" - "github.com/rook/rook/pkg/operator/test" - "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -260,3 +260,73 @@ func getObjectRealmAndReconcileObjectRealm(t *testing.T) (*ReconcileObjectRealm, return r, objectRealm } + +func TestReconcileObjectRealm_createRealmKeys(t *testing.T) { + ctx := context.TODO() + realmName := "my-realm" + ns := "my-ns" + + scheme := scheme.Scheme + scheme.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectRealm{}, &cephv1.CephCluster{}, &cephv1.CephClusterList{}) + + realm := &cephv1.CephObjectRealm{ + ObjectMeta: metav1.ObjectMeta{ + Name: realmName, + Namespace: ns, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "CephObjectRealm", + }, + } + + t.Run("should be idempotent", func(t *testing.T) { + r := ReconcileObjectRealm{ + context: &clusterd.Context{ + Clientset: k8sfake.NewSimpleClientset(), + }, + scheme: scheme, + } + + for _, tName := range []string{"first reconcile", "second reconcile"} { + // the output should be the same on the first and subsequent reconciles + t.Run(tName, func(t *testing.T) { + res, err := r.createRealmKeys(realm) + assert.NoError(t, err) + assert.True(t, res.IsZero()) + + secret, err := r.context.Clientset.CoreV1().Secrets(ns).Get(ctx, realmName+"-keys", metav1.GetOptions{}) + assert.NoError(t, err) + assert.Contains(t, secret.Data, "access-key") + assert.Contains(t, secret.Data, "secret-key") + }) + } + }) + + t.Run("should fail if the secret doesn't have the necessary keys", func(t *testing.T) { + secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: v1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: realmName + "-keys", + }, + Data: map[string][]byte{ + "access-key": []byte("my-access-key"), + // missing "secret-key" + }, + } + + r := ReconcileObjectRealm{ + context: &clusterd.Context{ + Clientset: k8sfake.NewSimpleClientset(secret), + }, + scheme: scheme, + } + + _, err := r.createRealmKeys(realm) + assert.Error(t, err) + assert.Contains(t, err.Error(), "user likely created or modified the secret manually and should add the missing key back into the secret") + }) +} diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 5f8b7592f..80c659119 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -282,7 +282,7 @@ function wait_for_operator_log_message() { function restart_operator () { local namespace="${1:-rook-ceph}" # optional param 1: the namespace of the CephCluster (default: rook-ceph) - kubectl --namespace "$namespace" delete pod --selector app=rook-ceph=operator + kubectl --namespace "$namespace" delete pod --selector app=rook-ceph-operator # wait for new pod to be running get_pod_cmd=(kubectl --namespace "$namespace" get pod --selector app=rook-ceph-operator --no-headers) timeout 20 bash -c \