diff --git a/pkg/operator/ceph/object/objectstore.go b/pkg/operator/ceph/object/objectstore.go index 7356423e3..73ee510fc 100644 --- a/pkg/operator/ceph/object/objectstore.go +++ b/pkg/operator/ceph/object/objectstore.go @@ -38,6 +38,7 @@ import ( v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" ) @@ -293,32 +294,34 @@ func DecodeSecret(secret *v1.Secret, keyName string) (string, error) { realmKey, ok := secret.Data[keyName] if !ok { - return "", errors.New("key was not in secret data") + return "", errors.New(fmt.Sprintf("failed to find key %q in secret %q data. ", keyName, secret.Name) + + "user likely created or modified the secret manually and should add the missing key back into the secret") } return string(realmKey), nil } -func GetRealmKeyArgs(clusterdContext *clusterd.Context, realmName, namespace string) (string, string, error) { +func GetRealmKeySecret(clusterdContext *clusterd.Context, realmName types.NamespacedName) (*v1.Secret, error) { ctx := context.TODO() - logger.Debugf("getting keys for realm %v", realmName) - // get realm's access and secret keys - realmSecretName := realmName + "-keys" - realmSecret, err := clusterdContext.Clientset.CoreV1().Secrets(namespace).Get(ctx, realmSecretName, metav1.GetOptions{}) + realmSecretName := realmName.Name + "-keys" + realmSecret, err := clusterdContext.Clientset.CoreV1().Secrets(realmName.Namespace).Get(ctx, realmSecretName, metav1.GetOptions{}) if err != nil { - return "", "", errors.Wrapf(err, "failed to get realm %q keys secret", realmName) + return nil, errors.Wrapf(err, "failed to get CephObjectRealm %q keys secret", realmName.String()) } - logger.Debugf("found keys secret for realm %v", realmName) + logger.Debugf("found keys secret for CephObjectRealm %q", realmName.String()) + return realmSecret, nil +} +func GetRealmKeyArgsFromSecret(realmSecret *v1.Secret, realmName types.NamespacedName) (string, string, error) { accessKey, err := DecodeSecret(realmSecret, AccessKeyName) if err != nil { - return "", "", errors.Wrapf(err, "failed to decode realm %q access key", realmName) + return "", "", errors.Wrapf(err, "failed to decode CephObjectRealm %q access key from secret %q", realmName.String(), realmSecret.Name) } secretKey, err := DecodeSecret(realmSecret, SecretKeyName) if err != nil { - return "", "", errors.Wrapf(err, "failed to decode realm %q access key", realmName) + return "", "", errors.Wrapf(err, "failed to decode CephObjectRealm %q access key from secret %q", realmName.String(), realmSecret.Name) } - logger.Debugf("decoded keys for realm %v", realmName) + logger.Debugf("decoded keys for realm %q", realmName.String()) accessKeyArg := fmt.Sprintf("--access-key=%s", accessKey) secretKeyArg := fmt.Sprintf("--secret-key=%s", secretKey) @@ -326,6 +329,18 @@ func GetRealmKeyArgs(clusterdContext *clusterd.Context, realmName, namespace str return accessKeyArg, secretKeyArg, nil } +func GetRealmKeyArgs(clusterdContext *clusterd.Context, realmName, namespace string) (string, string, error) { + realmNsName := types.NamespacedName{Namespace: namespace, Name: realmName} + logger.Debugf("getting keys for realm %q", realmNsName.String()) + + secret, err := GetRealmKeySecret(clusterdContext, realmNsName) + if err != nil { + return "", "", err + } + + return GetRealmKeyArgsFromSecret(secret, realmNsName) +} + func getZoneEndpoints(objContext *Context, serviceEndpoint string) ([]string, error) { logger.Debugf("getting current endpoints for zone %v", objContext.Zone) realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm) diff --git a/pkg/operator/ceph/object/objectstore_test.go b/pkg/operator/ceph/object/objectstore_test.go index 51cbec7c4..0ee96831e 100644 --- a/pkg/operator/ceph/object/objectstore_test.go +++ b/pkg/operator/ceph/object/objectstore_test.go @@ -32,6 +32,10 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" ) const ( @@ -550,3 +554,152 @@ func Test_createMultisite(t *testing.T) { }) } } + +func TestGetRealmKeySecret(t *testing.T) { + ns := "my-ns" + realmName := "my-realm" + + t.Run("secret exists", func(t *testing.T) { + secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: v1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: realmName + "-keys", + }, + // should not care about data presence just to get the secret + } + + c := &clusterd.Context{ + Clientset: k8sfake.NewSimpleClientset(secret), + } + + secret, err := GetRealmKeySecret(c, types.NamespacedName{Namespace: ns, Name: realmName}) + assert.NoError(t, err) + assert.NotNil(t, secret) + }) + + t.Run("secret doesn't exist", func(t *testing.T) { + c := &clusterd.Context{ + Clientset: k8sfake.NewSimpleClientset(), + } + + secret, err := GetRealmKeySecret(c, types.NamespacedName{Namespace: ns, Name: realmName}) + assert.Error(t, err) + assert.Nil(t, secret) + }) +} + +func TestGetRealmKeyArgsFromSecret(t *testing.T) { + ns := "my-ns" + realmName := "my-realm" + realmNsName := types.NamespacedName{Namespace: ns, Name: realmName} + + baseSecret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: v1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: realmName + "-keys", + }, + Data: map[string][]byte{}, + } + + t.Run("all secret data exists", func(t *testing.T) { + s := baseSecret.DeepCopy() + s.Data["access-key"] = []byte("my-access-key") + s.Data["secret-key"] = []byte("my-secret-key") + + access, secret, err := GetRealmKeyArgsFromSecret(s, realmNsName) + assert.NoError(t, err) + assert.Equal(t, "--access-key=my-access-key", access) + assert.Equal(t, "--secret-key=my-secret-key", secret) + }) + + t.Run("access-key missing", func(t *testing.T) { + s := baseSecret.DeepCopy() + // missing s.Data["access-key"] + s.Data["secret-key"] = []byte("my-secret-key") + + access, secret, err := GetRealmKeyArgsFromSecret(s, realmNsName) + assert.Error(t, err) + assert.Equal(t, "", access) + assert.Equal(t, "", secret) + }) + + t.Run("secret-key missing", func(t *testing.T) { + s := baseSecret.DeepCopy() + s.Data["access-key"] = []byte("my-access-key") + // missing s.Data["secret-key"] + + access, secret, err := GetRealmKeyArgsFromSecret(s, realmNsName) + assert.Error(t, err) + assert.Equal(t, "", access) + assert.Equal(t, "", secret) + }) +} + +func TestGetRealmKeyArgs(t *testing.T) { + ns := "my-ns" + realmName := "my-realm" + + baseSecret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: v1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: realmName + "-keys", + }, + Data: map[string][]byte{}, + } + + // No need to test every case since this is a combination of GetRealmKeySecret and + // GetRealmKeyArgsFromSecret and those are both thoroughly unit tested. Just check the success + // case and cases where either sub-function fails. + + t.Run("secret exists with all data", func(t *testing.T) { + s := baseSecret.DeepCopy() + s.Data["access-key"] = []byte("my-access-key") + s.Data["secret-key"] = []byte("my-secret-key") + + c := &clusterd.Context{ + Clientset: k8sfake.NewSimpleClientset(s), + } + + access, secret, err := GetRealmKeyArgs(c, realmName, ns) + assert.NoError(t, err) + assert.Equal(t, "--access-key=my-access-key", access) + assert.Equal(t, "--secret-key=my-secret-key", secret) + }) + + t.Run("secret doesn't exist", func(t *testing.T) { + c := &clusterd.Context{ + Clientset: k8sfake.NewSimpleClientset(), + } + + access, secret, err := GetRealmKeyArgs(c, realmName, ns) + assert.Error(t, err) + assert.Equal(t, "", access) + assert.Equal(t, "", secret) + }) + + t.Run("secret exists but is missing data", func(t *testing.T) { + s := baseSecret.DeepCopy() + // missing all data + + c := &clusterd.Context{ + Clientset: k8sfake.NewSimpleClientset(s), + } + + access, secret, err := GetRealmKeyArgs(c, realmName, ns) + assert.Error(t, err) + assert.Equal(t, "", access) + assert.Equal(t, "", secret) + }) +} diff --git a/pkg/operator/ceph/object/realm/controller.go b/pkg/operator/ceph/object/realm/controller.go index 2aa3af1a7..43b36637d 100644 --- a/pkg/operator/ceph/object/realm/controller.go +++ b/pkg/operator/ceph/object/realm/controller.go @@ -120,7 +120,7 @@ func (r *ReconcileObjectRealm) Reconcile(context context.Context, request reconc // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { - logger.Errorf("failed to reconcile: %v", err) + logger.Errorf("failed to reconcile CephObjectRealm %q. %v", request.NamespacedName.String(), err) } return reconcileResponse, err @@ -132,7 +132,7 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R err := r.client.Get(r.opManagerContext, request.NamespacedName, cephObjectRealm) if err != nil { if kerrors.IsNotFound(err) { - logger.Debug("CephObjectRealm resource not found. Ignoring since object must be deleted.") + logger.Debug("CephObjectRealm %q resource not found. Ignoring since object must be deleted", request.NamespacedName.String()) return reconcile.Result{}, nil } // Error reading the object - requeue the request. @@ -181,7 +181,7 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R // Create/Pull Ceph Realm if cephObjectRealm.Spec.IsPullRealm() { - logger.Debug("pull section in spec found") + logger.Debug("pull section in realm %q spec found", request.NamespacedName) _, err = r.pullCephRealm(cephObjectRealm) if err != nil { return reconcile.Result{}, err @@ -202,14 +202,14 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R r.updateStatus(r.client, request.NamespacedName, k8sutil.ReadyStatus) // Return and do not requeue - logger.Debug("realm done reconciling") + logger.Debug("realm %q done reconciling", request.NamespacedName) return reconcile.Result{}, nil } func (r *ReconcileObjectRealm) pullCephRealm(realm *cephv1.CephObjectRealm) (reconcile.Result, error) { realmArg := fmt.Sprintf("--rgw-realm=%s", realm.Name) urlArg := fmt.Sprintf("--url=%s", realm.Spec.Pull.Endpoint) - logger.Debug("getting keys to pull realm") + logger.Debug("getting keys to pull realm for CephObjectRealm %q", realm.Name) accessKeyArg, secretKeyArg, err := object.GetRealmKeyArgs(r.context, realm.Name, realm.Namespace) if err != nil { if kerrors.IsNotFound(err) { @@ -217,7 +217,7 @@ func (r *ReconcileObjectRealm) pullCephRealm(realm *cephv1.CephObjectRealm) (rec } return waitForRequeueIfRealmNotReady, errors.Wrap(err, "failed to get keys for realm") } - logger.Debugf("keys found to pull realm, getting ready to pull from endpoint %q", realm.Spec.Pull.Endpoint) + logger.Debugf("keys found to pull realm for CephObjectRealm %q, getting ready to pull from endpoint %q", realm.Name, realm.Spec.Pull.Endpoint) objContext := object.NewContext(r.context, r.clusterInfo, realm.Name) output, err := object.RunAdminCommandNoMultisite(objContext, false, "realm", "pull", realmArg, urlArg, accessKeyArg, secretKeyArg) @@ -253,7 +253,21 @@ func (r *ReconcileObjectRealm) createCephRealm(realm *cephv1.CephObjectRealm) (r } func (r *ReconcileObjectRealm) createRealmKeys(realm *cephv1.CephObjectRealm) (reconcile.Result, error) { - logger.Debugf("generating access and secret keys for new realm %q", realm.Name) + realmName := types.NamespacedName{Namespace: realm.Namespace, Name: realm.Name} + logger.Debugf("generating access and secret keys for new realm %q", realmName.String()) + secretName := realm.Name + "-keys" + + // Check if the secret exists first, and check that it has the access information needed. + secret, err := object.GetRealmKeySecret(r.context, realmName) + if err == nil { + // secret exists, now verify access info. We don't need the args, but we do want to get the + // error if the args can't be built + _, _, err := object.GetRealmKeyArgsFromSecret(secret, realmName) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to ensure secret keys for CephObjectRealm %q", realmName.String()) + } + return reconcile.Result{}, nil + } // the realm's secret key and access key are randomly generated and then encoded to base64 accessKey, err := mgr.GeneratePassword(accessKeyLength) @@ -275,8 +289,7 @@ func (r *ReconcileObjectRealm) createRealmKeys(realm *cephv1.CephObjectRealm) (r object.SecretKeyName: []byte(secretKey), } - secretName := realm.Name + "-keys" - secret := &v1.Secret{ + secret = &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: secretName, Namespace: realm.Namespace, @@ -292,7 +305,7 @@ func (r *ReconcileObjectRealm) createRealmKeys(realm *cephv1.CephObjectRealm) (r if _, err = r.context.Clientset.CoreV1().Secrets(realm.Namespace).Create(r.opManagerContext, secret, metav1.CreateOptions{}); err != nil { return reconcile.Result{}, errors.Wrap(err, "failed to save rgw secrets") } - logger.Infof("secrets for keys have been created for realm %q", realm.Name) + logger.Infof("secret keys have been created for realm %q in secret %q", realm.Name, secret.Name) return reconcile.Result{}, nil } @@ -318,7 +331,7 @@ func (r *ReconcileObjectRealm) updateStatus(client client.Client, name types.Nam objectRealm := &cephv1.CephObjectRealm{} if err := client.Get(r.opManagerContext, name, objectRealm); err != nil { if kerrors.IsNotFound(err) { - logger.Debug("CephObjectRealm resource not found. Ignoring since object must be deleted.") + logger.Debug("CephObjectRealm %q resource not found. Ignoring since object must be deleted", name) return } logger.Warningf("failed to retrieve object realm %q to update status to %q. %v", name, status, err) diff --git a/pkg/operator/ceph/object/realm/controller_test.go b/pkg/operator/ceph/object/realm/controller_test.go index 057271642..73ba09ade 100644 --- a/pkg/operator/ceph/object/realm/controller_test.go +++ b/pkg/operator/ceph/object/realm/controller_test.go @@ -24,17 +24,17 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" - "github.com/rook/rook/pkg/operator/test" - "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -260,3 +260,73 @@ func getObjectRealmAndReconcileObjectRealm(t *testing.T) (*ReconcileObjectRealm, return r, objectRealm } + +func TestReconcileObjectRealm_createRealmKeys(t *testing.T) { + ctx := context.TODO() + realmName := "my-realm" + ns := "my-ns" + + scheme := scheme.Scheme + scheme.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectRealm{}, &cephv1.CephCluster{}, &cephv1.CephClusterList{}) + + realm := &cephv1.CephObjectRealm{ + ObjectMeta: metav1.ObjectMeta{ + Name: realmName, + Namespace: ns, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "CephObjectRealm", + }, + } + + t.Run("should be idempotent", func(t *testing.T) { + r := ReconcileObjectRealm{ + context: &clusterd.Context{ + Clientset: k8sfake.NewSimpleClientset(), + }, + scheme: scheme, + } + + for _, tName := range []string{"first reconcile", "second reconcile"} { + // the output should be the same on the first and subsequent reconciles + t.Run(tName, func(t *testing.T) { + res, err := r.createRealmKeys(realm) + assert.NoError(t, err) + assert.True(t, res.IsZero()) + + secret, err := r.context.Clientset.CoreV1().Secrets(ns).Get(ctx, realmName+"-keys", metav1.GetOptions{}) + assert.NoError(t, err) + assert.Contains(t, secret.Data, "access-key") + assert.Contains(t, secret.Data, "secret-key") + }) + } + }) + + t.Run("should fail if the secret doesn't have the necessary keys", func(t *testing.T) { + secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: v1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: realmName + "-keys", + }, + Data: map[string][]byte{ + "access-key": []byte("my-access-key"), + // missing "secret-key" + }, + } + + r := ReconcileObjectRealm{ + context: &clusterd.Context{ + Clientset: k8sfake.NewSimpleClientset(secret), + }, + scheme: scheme, + } + + _, err := r.createRealmKeys(realm) + assert.Error(t, err) + assert.Contains(t, err.Error(), "user likely created or modified the secret manually and should add the missing key back into the secret") + }) +} diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 5f8b7592f..80c659119 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -282,7 +282,7 @@ function wait_for_operator_log_message() { function restart_operator () { local namespace="${1:-rook-ceph}" # optional param 1: the namespace of the CephCluster (default: rook-ceph) - kubectl --namespace "$namespace" delete pod --selector app=rook-ceph=operator + kubectl --namespace "$namespace" delete pod --selector app=rook-ceph-operator # wait for new pod to be running get_pod_cmd=(kubectl --namespace "$namespace" get pod --selector app=rook-ceph-operator --no-headers) timeout 20 bash -c \