From 26c8fd9bd14178d74dcc28f009e5c2f658224436 Mon Sep 17 00:00:00 2001 From: Satoru Takeuchi Date: Thu, 18 Feb 2021 13:10:11 +0000 Subject: [PATCH] ceph: improve owner reference management It's better to validate ownerReferences when setting them. In addition, we should use controllerrutil.Set{Controller,Owner}Reference, that have such validation, as possible. Signed-off-by: Satoru Takeuchi --- cmd/rook/ceph/mgr.go | 3 +- cmd/rook/ceph/osd.go | 6 +- .../flexvolume/manager/ceph/manager_test.go | 10 +- pkg/daemon/ceph/client/info.go | 23 ++++- pkg/daemon/ceph/client/test/info.go | 4 +- pkg/daemon/ceph/osd/kms/k8s.go | 16 ++- pkg/operator/ceph/client/controller.go | 2 +- pkg/operator/ceph/cluster/cluster.go | 14 +-- pkg/operator/ceph/cluster/cluster_external.go | 21 ++-- pkg/operator/ceph/cluster/controller.go | 13 +-- pkg/operator/ceph/cluster/crash/crash.go | 2 +- pkg/operator/ceph/cluster/crash/keyring.go | 9 +- pkg/operator/ceph/cluster/mgr/config.go | 2 +- pkg/operator/ceph/cluster/mgr/dashboard.go | 10 +- .../ceph/cluster/mgr/dashboard_test.go | 5 +- pkg/operator/ceph/cluster/mgr/mgr.go | 14 ++- pkg/operator/ceph/cluster/mgr/mgr_test.go | 7 +- pkg/operator/ceph/cluster/mgr/spec.go | 35 ++++--- pkg/operator/ceph/cluster/mgr/spec_test.go | 21 ++-- pkg/operator/ceph/cluster/mon/config.go | 17 ++-- pkg/operator/ceph/cluster/mon/config_test.go | 11 ++- pkg/operator/ceph/cluster/mon/health_test.go | 13 ++- pkg/operator/ceph/cluster/mon/mon.go | 21 ++-- pkg/operator/ceph/cluster/mon/mon_test.go | 10 +- pkg/operator/ceph/cluster/mon/node_test.go | 5 +- pkg/operator/ceph/cluster/mon/service.go | 10 +- pkg/operator/ceph/cluster/mon/service_test.go | 3 +- pkg/operator/ceph/cluster/mon/spec.go | 10 +- pkg/operator/ceph/cluster/mon/spec_test.go | 17 ++-- pkg/operator/ceph/cluster/osd/deviceSet.go | 11 ++- pkg/operator/ceph/cluster/osd/envs.go | 2 +- pkg/operator/ceph/cluster/osd/osd.go | 2 +- pkg/operator/ceph/cluster/osd/osd_pvc_test.go | 1 + pkg/operator/ceph/cluster/osd/osd_test.go | 4 + .../ceph/cluster/osd/provision_spec.go | 5 +- pkg/operator/ceph/cluster/osd/spec.go | 5 +- pkg/operator/ceph/cluster/osd/spec_test.go | 12 ++- pkg/operator/ceph/cluster/osd/status_test.go | 3 +- pkg/operator/ceph/cluster/rbd/config.go | 4 +- pkg/operator/ceph/cluster/rbd/mirror.go | 5 +- pkg/operator/ceph/cluster/rbd/spec_test.go | 7 +- pkg/operator/ceph/cluster/utils.go | 12 ++- pkg/operator/ceph/cluster/version.go | 2 +- .../ceph/config/keyring/admin_test.go | 9 +- pkg/operator/ceph/config/keyring/store.go | 11 ++- .../ceph/config/keyring/store_test.go | 13 +-- pkg/operator/ceph/config/store.go | 15 +-- pkg/operator/ceph/config/store_test.go | 8 +- .../ceph/controller/controller_utils.go | 2 + pkg/operator/ceph/csi/cluster_config.go | 9 +- pkg/operator/ceph/csi/csi.go | 7 +- pkg/operator/ceph/csi/secrets.go | 9 +- pkg/operator/ceph/csi/spec.go | 41 ++++++-- pkg/operator/ceph/file/controller.go | 19 +--- pkg/operator/ceph/file/filesystem.go | 12 +-- pkg/operator/ceph/file/filesystem_test.go | 21 ++-- pkg/operator/ceph/file/mds/config.go | 2 +- pkg/operator/ceph/file/mds/mds.go | 17 ++-- pkg/operator/ceph/file/mds/spec.go | 7 +- pkg/operator/ceph/file/mds/spec_test.go | 13 ++- pkg/operator/ceph/file/mirror/config.go | 6 +- pkg/operator/ceph/file/mirror/mirror.go | 14 +-- pkg/operator/ceph/file/mirror/spec.go | 5 +- pkg/operator/ceph/file/mirror/spec_test.go | 6 +- pkg/operator/ceph/nfs/config.go | 4 +- pkg/operator/ceph/nfs/nfs.go | 4 +- pkg/operator/ceph/nfs/spec.go | 2 +- pkg/operator/ceph/object/config.go | 2 +- pkg/operator/ceph/object/controller.go | 4 +- pkg/operator/ceph/object/mime.go | 2 +- pkg/operator/ceph/object/realm/controller.go | 2 +- pkg/operator/ceph/object/rgw.go | 18 +--- pkg/operator/ceph/object/rgw_test.go | 12 ++- pkg/operator/ceph/object/spec.go | 9 +- pkg/operator/ceph/object/spec_test.go | 6 +- pkg/operator/ceph/object/user/controller.go | 2 +- pkg/operator/ceph/operator.go | 5 +- pkg/operator/ceph/pool/config.go | 2 +- pkg/operator/ceph/test/ownerinfo/resources.go | 39 ++++++++ .../k8sutil/cmdreporter/cmdreporter.go | 16 +-- pkg/operator/k8sutil/kvstore.go | 11 ++- pkg/operator/k8sutil/kvstore_test.go | 5 +- pkg/operator/k8sutil/resources.go | 97 ++++++++++++++++++- pkg/operator/k8sutil/resources_test.go | 40 ++++++++ tests/framework/installer/ceph_installer.go | 3 +- 85 files changed, 631 insertions(+), 309 deletions(-) create mode 100644 pkg/operator/ceph/test/ownerinfo/resources.go diff --git a/cmd/rook/ceph/mgr.go b/cmd/rook/ceph/mgr.go index e4bf21c11..ceb94bd96 100644 --- a/cmd/rook/ceph/mgr.go +++ b/cmd/rook/ceph/mgr.go @@ -25,6 +25,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/mgr" "github.com/rook/rook/pkg/operator/ceph/cluster/mon" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/flags" "github.com/spf13/cobra" ) @@ -70,7 +71,7 @@ func runMgrSidecar(cmd *cobra.Command, args []string) error { rook.LogStartupInfo(mgrSidecarCmd.Flags()) ownerRef := opcontroller.ClusterOwnerRef(clusterName, ownerRefID) - clusterInfo.OwnerRef = ownerRef + clusterInfo.OwnerInfo = k8sutil.NewOwnerInfoWithOwnerRef(&ownerRef, clusterInfo.Namespace) if err := client.WriteCephConfig(context, &clusterInfo); err != nil { rook.TerminateFatal(err) diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index 38fa7b4d1..e37fb23bb 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -220,9 +220,11 @@ func prepareOSD(cmd *cobra.Command, args []string) error { logger.Infof("crush location of osd: %s", crushLocation) forceFormat := false + ownerRef := opcontroller.ClusterOwnerRef(clusterName, ownerRefID) - clusterInfo.OwnerRef = ownerRef - kv := k8sutil.NewConfigMapKVStore(clusterInfo.Namespace, context.Clientset, ownerRef) + ownerInfo := k8sutil.NewOwnerInfoWithOwnerRef(&ownerRef, clusterInfo.Namespace) + clusterInfo.OwnerInfo = ownerInfo + kv := k8sutil.NewConfigMapKVStore(clusterInfo.Namespace, context.Clientset, ownerInfo) agent := osddaemon.NewAgent(context, dataDevices, cfg.metadataDevice, forceFormat, cfg.storeConfig, &clusterInfo, cfg.nodeName, kv, cfg.pvcBacked) diff --git a/pkg/daemon/ceph/agent/flexvolume/manager/ceph/manager_test.go b/pkg/daemon/ceph/agent/flexvolume/manager/ceph/manager_test.go index db0512692..6d1ae9f1e 100644 --- a/pkg/daemon/ceph/agent/flexvolume/manager/ceph/manager_test.go +++ b/pkg/daemon/ceph/agent/flexvolume/manager/ceph/manager_test.go @@ -26,6 +26,7 @@ import ( "time" "github.com/rook/rook/pkg/clusterd" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test" "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/test" @@ -160,7 +161,8 @@ func TestAttach(t *testing.T) { called: 0, }, } - _, _, _, err = mon.CreateOrLoadClusterInfo(context, clusterNamespace, &metav1.OwnerReference{}) + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + _, _, _, err = mon.CreateOrLoadClusterInfo(context, clusterNamespace, ownerInfo) assert.NoError(t, err) devicePath, err := vm.Attach("image1", "testpool", "admin", "never-gonna-give-you-up", clusterNamespace) assert.Equal(t, "/dev/rbd3", devicePath) @@ -241,7 +243,8 @@ func TestDetach(t *testing.T) { called: 0, }, } - _, _, _, err = mon.CreateOrLoadClusterInfo(context, clusterNamespace, &metav1.OwnerReference{}) + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + _, _, _, err = mon.CreateOrLoadClusterInfo(context, clusterNamespace, ownerInfo) assert.NoError(t, err) err = vm.Detach("image1", "testpool", "admin", "", clusterNamespace, false) assert.Nil(t, err) @@ -296,7 +299,8 @@ func TestDetachCustomKeyring(t *testing.T) { called: 0, }, } - _, _, _, err = mon.CreateOrLoadClusterInfo(context, clusterNamespace, &metav1.OwnerReference{}) + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + _, _, _, err = mon.CreateOrLoadClusterInfo(context, clusterNamespace, ownerInfo) assert.NoError(t, err) err = vm.Detach("image1", "testpool", "user1", "", clusterNamespace, false) assert.Nil(t, err) diff --git a/pkg/daemon/ceph/client/info.go b/pkg/daemon/ceph/client/info.go index fae7c254e..06dc1f9cd 100644 --- a/pkg/daemon/ceph/client/info.go +++ b/pkg/daemon/ceph/client/info.go @@ -19,10 +19,15 @@ package client import ( "fmt" "net" + "testing" "time" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" cephver "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ) @@ -35,7 +40,7 @@ type ClusterInfo struct { Monitors map[string]*MonInfo CephVersion cephver.CephVersion Namespace string - OwnerRef metav1.OwnerReference + OwnerInfo *k8sutil.OwnerInfo // Hide the name of the cluster since in 99% of uses we want to use the cluster namespace. // If the CR name is needed, access it through the NamespacedName() method. name string @@ -76,12 +81,14 @@ func (c *ClusterInfo) NamespacedName() types.NamespacedName { // so this clusterInfo cannot be used to generate the mon config or request the // namespacedName. A full cluster info must be populated for those operations. func AdminClusterInfo(namespace string) *ClusterInfo { + ownerInfo := k8sutil.NewOwnerInfoWithOwnerRef(&metav1.OwnerReference{}, "") return &ClusterInfo{ Namespace: namespace, CephCred: CephCred{ Username: AdminUsername, }, - name: "testing", + name: "testing", + OwnerInfo: ownerInfo, } } @@ -123,3 +130,15 @@ func (c *ClusterInfo) IsInitialized(logError bool) bool { func NewMonInfo(name, ip string, port int32) *MonInfo { return &MonInfo{Name: name, Endpoint: net.JoinHostPort(ip, fmt.Sprintf("%d", port))} } + +func NewMinimumOwnerInfo(t *testing.T) *k8sutil.OwnerInfo { + cluster := &cephv1.CephCluster{} + scheme := runtime.NewScheme() + err := cephv1.AddToScheme(scheme) + assert.NoError(t, err) + return k8sutil.NewOwnerInfo(cluster, scheme) +} + +func NewMinimumOwnerInfoWithOwnerRef() *k8sutil.OwnerInfo { + return k8sutil.NewOwnerInfoWithOwnerRef(&metav1.OwnerReference{}, "") +} diff --git a/pkg/daemon/ceph/client/test/info.go b/pkg/daemon/ceph/client/test/info.go index 9f4243c80..efb03c956 100644 --- a/pkg/daemon/ceph/client/test/info.go +++ b/pkg/daemon/ceph/client/test/info.go @@ -42,6 +42,7 @@ func CreateConfigDir(configDir string) error { // CreateTestClusterInfo creates a test cluster info // This would be best in a test package, but is included here to avoid cyclic dependencies func CreateTestClusterInfo(monCount int) *client.ClusterInfo { + ownerInfo := client.NewMinimumOwnerInfoWithOwnerRef() c := &client.ClusterInfo{ FSID: "12345", Namespace: "default", @@ -50,7 +51,8 @@ func CreateTestClusterInfo(monCount int) *client.ClusterInfo { Username: client.AdminUsername, Secret: "adminkey", }, - Monitors: map[string]*client.MonInfo{}, + Monitors: map[string]*client.MonInfo{}, + OwnerInfo: ownerInfo, } mons := []string{"a", "b", "c", "d", "e"} for i := 0; i < monCount; i++ { diff --git a/pkg/daemon/ceph/osd/kms/k8s.go b/pkg/daemon/ceph/osd/kms/k8s.go index c11e048e4..d1e693314 100644 --- a/pkg/daemon/ceph/osd/kms/k8s.go +++ b/pkg/daemon/ceph/osd/kms/k8s.go @@ -43,10 +43,13 @@ const ( // storeSecretInKubernetes stores the dmcrypt key in a Kubernetes Secret func (c *Config) storeSecretInKubernetes(pvcName, key string) error { ctx := context.TODO() - s := generateOSDEncryptedKeySecret(pvcName, key, c.clusterInfo) + s, err := generateOSDEncryptedKeySecret(pvcName, key, c.clusterInfo) + if err != nil { + return err + } // Create the Kubernetes Secret - _, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Create(ctx, s, metav1.CreateOptions{}) + _, err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Create(ctx, s, metav1.CreateOptions{}) if err != nil && !kerrors.IsAlreadyExists(err) { return errors.Wrapf(err, "failed to save ceph osd encryption key as a secret for pvc %q", pvcName) } @@ -54,7 +57,7 @@ func (c *Config) storeSecretInKubernetes(pvcName, key string) error { return nil } -func generateOSDEncryptedKeySecret(pvcName, key string, clusterInfo *cephclient.ClusterInfo) *v1.Secret { +func generateOSDEncryptedKeySecret(pvcName, key string, clusterInfo *cephclient.ClusterInfo) (*v1.Secret, error) { s := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: GenerateOSDEncryptionSecretName(pvcName), @@ -70,9 +73,12 @@ func generateOSDEncryptedKeySecret(pvcName, key string, clusterInfo *cephclient. } // Set the ownerref to the Secret - k8sutil.SetOwnerRef(&s.ObjectMeta, &clusterInfo.OwnerRef) + err := clusterInfo.OwnerInfo.SetControllerReference(s) + if err != nil { + return nil, errors.Wrapf(err, "failed to set owner reference to osd encryption key secret %q", s.Name) + } - return s + return s, nil } // GenerateOSDEncryptionSecretName generate the Kubernetes Secret name of the encrypted key diff --git a/pkg/operator/ceph/client/controller.go b/pkg/operator/ceph/client/controller.go index eb74c2e18..691ad07de 100644 --- a/pkg/operator/ceph/client/controller.go +++ b/pkg/operator/ceph/client/controller.go @@ -263,7 +263,7 @@ func (r *ReconcileCephClient) createOrUpdateClient(cephClient *cephv1.CephClient // Set CephClient owner ref to the Secret err = controllerutil.SetControllerReference(cephClient, secret, r.scheme) if err != nil { - return errors.Wrap(err, "failed to set owner reference to ceph client secret") + return errors.Wrapf(err, "failed to set owner reference to ceph client secret %q", secret.Name) } // Create or Update Kubernetes Secret diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index e39b23008..365ac2d0a 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -61,7 +61,7 @@ type cluster struct { mons *mon.Cluster stopCh chan struct{} closedStopCh bool - ownerRef metav1.OwnerReference + ownerInfo *k8sutil.OwnerInfo isUpgrade bool watchersActivated bool monitoringChannels map[string]*clusterHealth @@ -72,7 +72,7 @@ type clusterHealth struct { monitoringRunning bool } -func newCluster(c *cephv1.CephCluster, context *clusterd.Context, csiMutex *sync.Mutex, ownerRef *metav1.OwnerReference) *cluster { +func newCluster(c *cephv1.CephCluster, context *clusterd.Context, csiMutex *sync.Mutex, ownerInfo *k8sutil.OwnerInfo) *cluster { return &cluster{ // at this phase of the cluster creation process, the identity components of the cluster are // not yet established. we reserve this struct which is filled in as soon as the cluster's @@ -84,15 +84,15 @@ func newCluster(c *cephv1.CephCluster, context *clusterd.Context, csiMutex *sync namespacedName: types.NamespacedName{Namespace: c.Namespace, Name: c.Name}, monitoringChannels: make(map[string]*clusterHealth), stopCh: make(chan struct{}), - ownerRef: *ownerRef, - mons: mon.New(context, c.Namespace, c.Spec, *ownerRef, csiMutex), + ownerInfo: ownerInfo, + mons: mon.New(context, c.Namespace, c.Spec, ownerInfo, csiMutex), } } func (c *cluster) doOrchestration(rookImage string, cephVersion cephver.CephVersion, spec *cephv1.ClusterSpec) error { // Create a configmap for overriding ceph config settings // These settings should only be modified by a user after they are initialized - err := populateConfigOverrideConfigMap(c.context, c.Namespace, c.ownerRef) + err := populateConfigOverrideConfigMap(c.context, c.Namespace, c.ownerInfo) if err != nil { return errors.Wrap(err, "failed to populate config override config map") } @@ -103,7 +103,7 @@ func (c *cluster) doOrchestration(rookImage string, cephVersion cephver.CephVers if err != nil { return errors.Wrap(err, "failed to start ceph monitors") } - clusterInfo.OwnerRef = c.ownerRef + clusterInfo.OwnerInfo = c.ownerInfo clusterInfo.SetName(c.namespacedName.Name) c.ClusterInfo = clusterInfo @@ -186,7 +186,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *ceph if err != nil { logger.Infof("clusterInfo not yet found, must be a new cluster") } else { - clusterInfo.OwnerRef = cluster.ownerRef + clusterInfo.OwnerInfo = cluster.ownerInfo clusterInfo.SetName(c.namespacedName.Name) cluster.ClusterInfo = clusterInfo } diff --git a/pkg/operator/ceph/cluster/cluster_external.go b/pkg/operator/ceph/cluster/cluster_external.go index 2d92950ee..c288ee318 100644 --- a/pkg/operator/ceph/cluster/cluster_external.go +++ b/pkg/operator/ceph/cluster/cluster_external.go @@ -47,7 +47,8 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error // loop until we find the secret necessary to connect to the external cluster // then populate clusterInfo - cluster.ClusterInfo = mon.PopulateExternalClusterInfo(c.context, c.namespacedName.Namespace, cluster.ownerRef) + + cluster.ClusterInfo = mon.PopulateExternalClusterInfo(c.context, c.namespacedName.Namespace, cluster.ownerInfo) cluster.ClusterInfo.SetName(c.namespacedName.Name) if !client.IsKeyringBase64Encoded(cluster.ClusterInfo.CephCred.Secret) { @@ -75,13 +76,13 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error // // Only do this when doing a bit of management... logger.Infof("creating %q configmap", k8sutil.ConfigOverrideName) - err = populateConfigOverrideConfigMap(c.context, c.namespacedName.Namespace, cluster.ClusterInfo.OwnerRef) + err = populateConfigOverrideConfigMap(c.context, c.namespacedName.Namespace, cluster.ClusterInfo.OwnerInfo) if err != nil { return errors.Wrap(err, "failed to populate config override config map") } logger.Infof("creating %q secret", config.StoreName) - err = config.GetStore(c.context, c.namespacedName.Namespace, &cluster.ClusterInfo.OwnerRef).CreateOrUpdate(cluster.ClusterInfo) + err = config.GetStore(c.context, c.namespacedName.Namespace, cluster.ClusterInfo.OwnerInfo).CreateOrUpdate(cluster.ClusterInfo) if err != nil { return errors.Wrap(err, "failed to update the global config") } @@ -102,7 +103,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error } // Create CSI config map - err = csi.CreateCsiConfigMap(c.namespacedName.Namespace, c.context.Clientset, &cluster.ownerRef) + err = csi.CreateCsiConfigMap(c.namespacedName.Namespace, c.context.Clientset, cluster.ownerInfo) if err != nil { return errors.Wrap(err, "failed to create csi config map") } @@ -204,16 +205,22 @@ func (c *ClusterController) configureExternalClusterMonitoring(cluster *cluster) ) // Create external monitoring Service - service := manager.MakeMetricsService(mgr.ExternalMgrAppName, "", mgr.ServiceExternalMetricName) + service, err := manager.MakeMetricsService(mgr.ExternalMgrAppName, "", mgr.ServiceExternalMetricName) + if err != nil { + return err + } logger.Info("creating mgr external monitoring service") - _, err := k8sutil.CreateOrUpdateService(c.context.Clientset, cluster.Namespace, service) + _, err = k8sutil.CreateOrUpdateService(c.context.Clientset, cluster.Namespace, service) if err != nil && !kerrors.IsAlreadyExists(err) { return errors.Wrap(err, "failed to create or update mgr service") } logger.Info("mgr external metrics service created") // Create external monitoring Endpoints - endpoint := mgr.CreateExternalMetricsEndpoints(cluster.Namespace, cluster.Spec.Monitoring, cluster.ownerRef) + endpoint, err := mgr.CreateExternalMetricsEndpoints(cluster.Namespace, cluster.Spec.Monitoring, cluster.ownerInfo) + if err != nil { + return err + } logger.Info("creating mgr external monitoring endpoints") _, err = k8sutil.CreateOrUpdateEndpoint(c.context.Clientset, c.namespacedName.Namespace, endpoint) if err != nil { diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 00a2e1492..bd6024ba6 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -293,14 +293,9 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R return reconcile.Result{}, nil } - // Create the controller owner ref - ref, err := opcontroller.GetControllerObjectOwnerReference(cephCluster, r.scheme) - if err != nil || ref == nil { - return reconcile.Result{}, errors.Wrapf(err, "failed to get controller %q owner reference", cephCluster.Name) - } - // Do reconcile here! - if err := r.clusterController.onAdd(cephCluster, ref); err != nil { + ownerInfo := k8sutil.NewOwnerInfo(cephCluster, r.scheme) + if err := r.clusterController.onAdd(cephCluster, ownerInfo); err != nil { return reconcile.Result{}, errors.Wrapf(err, "failed to reconcile cluster %q", cephCluster.Name) } @@ -321,7 +316,7 @@ func NewClusterController(context *clusterd.Context, rookImage string, volumeAtt } } -func (c *ClusterController) onAdd(clusterObj *cephv1.CephCluster, ref *metav1.OwnerReference) error { +func (c *ClusterController) onAdd(clusterObj *cephv1.CephCluster, ownerInfo *k8sutil.OwnerInfo) error { if clusterObj.Spec.CleanupPolicy.HasDataDirCleanPolicy() { logger.Infof("skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set", clusterObj.Name, clusterObj.Namespace) return nil @@ -330,7 +325,7 @@ func (c *ClusterController) onAdd(clusterObj *cephv1.CephCluster, ref *metav1.Ow cluster, ok := c.clusterMap[clusterObj.Namespace] if !ok { // It's a new cluster so let's populate the struct - cluster = newCluster(clusterObj, c.context, c.csiConfigMutex, ref) + cluster = newCluster(clusterObj, c.context, c.csiConfigMutex, ownerInfo) } // Note that this lock is held through the callback process, as this creates CSI resources, but we must lock in diff --git a/pkg/operator/ceph/cluster/crash/crash.go b/pkg/operator/ceph/cluster/crash/crash.go index 26db46901..814a980f8 100644 --- a/pkg/operator/ceph/cluster/crash/crash.go +++ b/pkg/operator/ceph/cluster/crash/crash.go @@ -57,7 +57,7 @@ func (r *ReconcileNode) createOrUpdateCephCrash(node corev1.Node, tolerations [] } err := controllerutil.SetControllerReference(&cephCluster, deploy, r.scheme) if err != nil { - return controllerutil.OperationResultNone, errors.Errorf("failed to set owner reference of deployment %q", deploy.Name) + return controllerutil.OperationResultNone, errors.Errorf("failed to set owner reference of crashcollector deployment %q", deploy.Name) } volumes := controller.DaemonVolumesBase(config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath), "") diff --git a/pkg/operator/ceph/cluster/crash/keyring.go b/pkg/operator/ceph/cluster/crash/keyring.go index 5b616f60b..a6400f8b0 100644 --- a/pkg/operator/ceph/cluster/crash/keyring.go +++ b/pkg/operator/ceph/cluster/crash/keyring.go @@ -40,7 +40,7 @@ const ( // CreateCrashCollectorSecret creates the Kubernetes Crash Collector Secret func CreateCrashCollectorSecret(context *clusterd.Context, clusterInfo *client.ClusterInfo) error { - k := keyring.GetSecretStore(context, clusterInfo, &clusterInfo.OwnerRef) + k := keyring.GetSecretStore(context, clusterInfo, clusterInfo.OwnerInfo) // Create CrashCollector Ceph key crashCollectorSecretKey, err := createCrashCollectorKeyring(k) @@ -88,10 +88,13 @@ func createOrUpdateCrashCollectorSecret(clusterInfo *client.ClusterInfo, crashCo Data: crashCollectorSecret, Type: k8sutil.RookType, } - k8sutil.SetOwnerRef(&s.ObjectMeta, &clusterInfo.OwnerRef) + err := clusterInfo.OwnerInfo.SetControllerReference(s) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference to crash controller secret %q", s.Name) + } // Create Kubernetes Secret - err := k.CreateSecret(s) + err = k.CreateSecret(s) if err != nil { return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", crashCollectorSecret, clusterInfo.Namespace) } diff --git a/pkg/operator/ceph/cluster/mgr/config.go b/pkg/operator/ceph/cluster/mgr/config.go index 3088392cc..ae69cf77c 100644 --- a/pkg/operator/ceph/cluster/mgr/config.go +++ b/pkg/operator/ceph/cluster/mgr/config.go @@ -60,7 +60,7 @@ func (c *Cluster) generateKeyring(m *mgrConfig) (string, error) { ctx := context.TODO() user := fmt.Sprintf("mgr.%s", m.DaemonID) access := []string{"mon", "allow profile mgr", "mds", "allow *", "osd", "allow *"} - s := keyring.GetSecretStore(c.context, c.clusterInfo, &c.clusterInfo.OwnerRef) + s := keyring.GetSecretStore(c.context, c.clusterInfo, c.clusterInfo.OwnerInfo) key, err := s.GenerateKey(user, access) if err != nil { diff --git a/pkg/operator/ceph/cluster/mgr/dashboard.go b/pkg/operator/ceph/cluster/mgr/dashboard.go index 0b9b39106..3e5d7ce40 100755 --- a/pkg/operator/ceph/cluster/mgr/dashboard.go +++ b/pkg/operator/ceph/cluster/mgr/dashboard.go @@ -51,7 +51,10 @@ var ( func (c *Cluster) configureDashboardService(activeDaemon string) error { ctx := context.TODO() - dashboardService := c.makeDashboardService(AppName, activeDaemon) + dashboardService, err := c.makeDashboardService(AppName, activeDaemon) + if err != nil { + return err + } if c.spec.Dashboard.Enabled { // expose the dashboard service if _, err := k8sutil.CreateOrUpdateService(c.context.Clientset, c.clusterInfo.Namespace, dashboardService); err != nil { @@ -242,7 +245,10 @@ func (c *Cluster) getOrGenerateDashboardPassword() (string, error) { Data: secrets, Type: k8sutil.RookType, } - k8sutil.SetOwnerRef(&secret.ObjectMeta, &c.clusterInfo.OwnerRef) + err = c.clusterInfo.OwnerInfo.SetControllerReference(secret) + if err != nil { + return "", errors.Wrapf(err, "failed to set owner reference to dashboard secret %q", secret.Name) + } _, err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Create(ctx, secret, metav1.CreateOptions{}) if err != nil { diff --git a/pkg/operator/ceph/cluster/mgr/dashboard_test.go b/pkg/operator/ceph/cluster/mgr/dashboard_test.go index 735650aa3..cccafe798 100644 --- a/pkg/operator/ceph/cluster/mgr/dashboard_test.go +++ b/pkg/operator/ceph/cluster/mgr/dashboard_test.go @@ -53,7 +53,8 @@ func TestGeneratePassword(t *testing.T) { func TestGetOrGeneratePassword(t *testing.T) { ctx := context.TODO() clientset := test.New(t, 3) - clusterInfo := &client.ClusterInfo{Namespace: "myns"} + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + clusterInfo := &client.ClusterInfo{Namespace: "myns", OwnerInfo: ownerInfo} c := &Cluster{context: &clusterd.Context{Clientset: clientset}, clusterInfo: clusterInfo} _, err := c.context.Clientset.CoreV1().Secrets(clusterInfo.Namespace).Get(ctx, dashboardPasswordName, metav1.GetOptions{}) assert.True(t, kerrors.IsNotFound(err)) @@ -109,9 +110,11 @@ func TestStartSecureDashboard(t *testing.T) { return executor.MockExecuteCommandWithOutputFile(command, outfileArg, arg...) } + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() clusterInfo := &cephclient.ClusterInfo{ Namespace: "myns", CephVersion: cephver.Nautilus, + OwnerInfo: ownerInfo, } c := &Cluster{clusterInfo: clusterInfo, context: &clusterd.Context{Clientset: clientset, Executor: executor}, spec: cephv1.ClusterSpec{ diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go index 36f18351c..87ee94890 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr.go +++ b/pkg/operator/ceph/cluster/mgr/mgr.go @@ -278,7 +278,10 @@ func (c *Cluster) reconcileService(activeDaemon string) error { } // create the metrics service - service := c.MakeMetricsService(AppName, activeDaemon, serviceMetricName) + service, err := c.MakeMetricsService(AppName, activeDaemon, serviceMetricName) + if err != nil { + return err + } if _, err := k8sutil.CreateOrUpdateService(c.context.Clientset, c.clusterInfo.Namespace, service); err != nil { return errors.Wrap(err, "failed to create mgr metrics service") } @@ -442,10 +445,13 @@ func (c *Cluster) EnableServiceMonitor(activeDaemon string) error { if c.spec.External.Enable { serviceMonitor.Spec.Endpoints[0].Port = ServiceExternalMetricName } - k8sutil.SetOwnerRef(&serviceMonitor.ObjectMeta, &c.clusterInfo.OwnerRef) + err = c.clusterInfo.OwnerInfo.SetControllerReference(serviceMonitor) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference to service monitor %q", serviceMonitor.Name) + } serviceMonitor.Spec.NamespaceSelector.MatchNames = []string{c.clusterInfo.Namespace} serviceMonitor.Spec.Selector.MatchLabels = c.selectorLabels(activeDaemon) - if _, err := k8sutil.CreateOrUpdateServiceMonitor(serviceMonitor); err != nil { + if _, err = k8sutil.CreateOrUpdateServiceMonitor(serviceMonitor); err != nil { return errors.Wrap(err, "service monitor could not be enabled") } return nil @@ -463,7 +469,7 @@ func (c *Cluster) DeployPrometheusRule(name, namespace string) error { } prometheusRule.SetName(name) prometheusRule.SetNamespace(namespace) - owners := append(prometheusRule.GetOwnerReferences(), c.clusterInfo.OwnerRef) + owners := append(prometheusRule.GetOwnerReferences(), *c.clusterInfo.OwnerInfo.GetOwnerRef()) k8sutil.SetOwnerRefs(&prometheusRule.ObjectMeta, owners) if _, err := k8sutil.CreateOrUpdatePrometheusRule(prometheusRule); err != nil { return errors.Wrap(err, "prometheus rule could not be deployed") diff --git a/pkg/operator/ceph/cluster/mgr/mgr_test.go b/pkg/operator/ceph/cluster/mgr/mgr_test.go index a0ce7a9a6..0831d26d3 100755 --- a/pkg/operator/ceph/cluster/mgr/mgr_test.go +++ b/pkg/operator/ceph/cluster/mgr/mgr_test.go @@ -28,6 +28,7 @@ import ( "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" cephver "github.com/rook/rook/pkg/operator/ceph/version" + testopk8s "github.com/rook/rook/pkg/operator/k8sutil/test" testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" @@ -63,7 +64,8 @@ func TestStartMgr(t *testing.T) { ConfigDir: configDir, Clientset: clientset, RequestCancelOrchestration: abool.New()} - clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", FSID: "myfsid"} + ownerInfo := cephclient.NewMinimumOwnerInfo(t) + clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", FSID: "myfsid", OwnerInfo: ownerInfo} clusterInfo.SetName("test") clusterSpec := cephv1.ClusterSpec{ Annotations: map[rookv1.KeyType]rookv1.Annotations{cephv1.KeyMgr: {"my": "annotation"}}, @@ -195,7 +197,8 @@ func TestMgrSidecarReconcile(t *testing.T) { ConfigDir: configDir, Clientset: clientset, } - clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"} + ownerInfo := cephclient.NewMinimumOwnerInfo(t) + clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", OwnerInfo: ownerInfo} clusterInfo.SetName("test") c := &Cluster{spec: spec, context: ctx, clusterInfo: clusterInfo} diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 6b4d9bf6f..f474d9d89 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -21,6 +21,7 @@ import ( "os" "strconv" + "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookcephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" @@ -122,7 +123,10 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) k8sutil.AddRookVersionLabelToDeployment(d) cephv1.GetMgrLabels(c.spec.Labels).ApplyToObjectMeta(&d.ObjectMeta) controller.AddCephVersionLabelToDeployment(c.clusterInfo.CephVersion, d) - k8sutil.SetOwnerRef(&d.ObjectMeta, &c.clusterInfo.OwnerRef) + err := c.clusterInfo.OwnerInfo.SetControllerReference(d) + if err != nil { + return nil, errors.Wrapf(err, "failed to set owner reference to mgr deployment %q", d.Name) + } return d, nil } @@ -196,7 +200,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { func (c *Cluster) makeMgrSidecarContainer(mgrConfig *mgrConfig) v1.Container { envVars := []v1.EnvVar{ - {Name: "ROOK_CLUSTER_ID", Value: string(c.clusterInfo.OwnerRef.UID)}, + {Name: "ROOK_CLUSTER_ID", Value: string(c.clusterInfo.OwnerInfo.GetUID())}, {Name: "ROOK_CLUSTER_NAME", Value: string(c.clusterInfo.NamespacedName().Name)}, k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), @@ -241,7 +245,7 @@ func getDefaultMgrLivenessProbe() *v1.Probe { } // MakeMetricsService generates the Kubernetes service object for the monitoring service -func (c *Cluster) MakeMetricsService(name, activeDaemon, servicePortMetricName string) *v1.Service { +func (c *Cluster) MakeMetricsService(name, activeDaemon, servicePortMetricName string) (*v1.Service, error) { labels := controller.AppLabels(AppName, c.clusterInfo.Namespace) svc := &v1.Service{ @@ -267,11 +271,14 @@ func (c *Cluster) MakeMetricsService(name, activeDaemon, servicePortMetricName s svc.Spec.Selector = c.selectorLabels(activeDaemon) } - k8sutil.SetOwnerRef(&svc.ObjectMeta, &c.clusterInfo.OwnerRef) - return svc + err := c.clusterInfo.OwnerInfo.SetControllerReference(svc) + if err != nil { + return nil, errors.Wrapf(err, "failed to set owner reference to monitoring service %q", svc.Name) + } + return svc, nil } -func (c *Cluster) makeDashboardService(name, activeDaemon string) *v1.Service { +func (c *Cluster) makeDashboardService(name, activeDaemon string) (*v1.Service, error) { labels := controller.AppLabels(AppName, c.clusterInfo.Namespace) selectorLabels := controller.AppLabels(AppName, c.clusterInfo.Namespace) selectorLabels[controller.DaemonIDLabel] = activeDaemon @@ -298,8 +305,11 @@ func (c *Cluster) makeDashboardService(name, activeDaemon string) *v1.Service { }, }, } - k8sutil.SetOwnerRef(&svc.ObjectMeta, &c.clusterInfo.OwnerRef) - return svc + err := c.clusterInfo.OwnerInfo.SetControllerReference(svc) + if err != nil { + return nil, errors.Wrapf(err, "failed to set owner reference to dashboard service %q", svc.Name) + } + return svc, nil } func (c *Cluster) getPodLabels(daemonName string, includeNewLabels bool) map[string]string { @@ -332,7 +342,7 @@ func (c *Cluster) cephMgrOrchestratorModuleEnvs() []v1.EnvVar { } // CreateExternalMetricsEndpoints creates external metric endpoint -func CreateExternalMetricsEndpoints(namespace string, monitoringSpec cephv1.MonitoringSpec, ownerRef metav1.OwnerReference) *v1.Endpoints { +func CreateExternalMetricsEndpoints(namespace string, monitoringSpec cephv1.MonitoringSpec, ownerInfo *k8sutil.OwnerInfo) (*v1.Endpoints, error) { labels := controller.AppLabels(AppName, namespace) endpoints := &v1.Endpoints{ @@ -355,8 +365,11 @@ func CreateExternalMetricsEndpoints(namespace string, monitoringSpec cephv1.Moni }, } - k8sutil.SetOwnerRef(&endpoints.ObjectMeta, &ownerRef) - return endpoints + err := ownerInfo.SetControllerReference(endpoints) + if err != nil { + return nil, errors.Wrapf(err, "failed to set owner reference to metric endpoints %q", endpoints.Name) + } + return endpoints, nil } func (c *Cluster) selectorLabels(activeDaemon string) map[string]string { diff --git a/pkg/operator/ceph/cluster/mgr/spec_test.go b/pkg/operator/ceph/cluster/mgr/spec_test.go index e6dc0f47b..8ef9f8afc 100644 --- a/pkg/operator/ceph/cluster/mgr/spec_test.go +++ b/pkg/operator/ceph/cluster/mgr/spec_test.go @@ -25,7 +25,7 @@ import ( cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/controller" - cephtest "github.com/rook/rook/pkg/operator/ceph/test" + "github.com/rook/rook/pkg/operator/ceph/test" optest "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" @@ -34,7 +34,8 @@ import ( func TestPodSpec(t *testing.T) { clientset := optest.New(t, 1) - clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", FSID: "myfsid"} + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", FSID: "myfsid", OwnerInfo: ownerInfo} clusterInfo.SetName("test") clusterSpec := cephv1.ClusterSpec{ CephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:myceph"}, @@ -65,10 +66,10 @@ func TestPodSpec(t *testing.T) { assert.NoError(t, err) // Deployment should have Ceph labels - cephtest.AssertLabelsContainCephRequirements(t, d.ObjectMeta.Labels, + test.AssertLabelsContainCephRequirements(t, d.ObjectMeta.Labels, config.MgrType, "a", AppName, "ns") - podTemplate := cephtest.NewPodTemplateSpecTester(t, &d.Spec.Template) + podTemplate := test.NewPodTemplateSpecTester(t, &d.Spec.Template) podTemplate.Spec().Containers().RequireAdditionalEnvVars( "ROOK_OPERATOR_NAMESPACE", "ROOK_CEPH_CLUSTER_CRD_VERSION", "ROOK_CEPH_CLUSTER_CRD_NAME") @@ -80,11 +81,13 @@ func TestPodSpec(t *testing.T) { func TestServiceSpec(t *testing.T) { clientset := optest.New(t, 1) - clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", FSID: "myfsid"} + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", FSID: "myfsid", OwnerInfo: ownerInfo} clusterSpec := cephv1.ClusterSpec{} c := New(&clusterd.Context{Clientset: clientset}, clusterInfo, clusterSpec, "myversion") - s := c.MakeMetricsService("rook-mgr", "foo", serviceMetricName) + s, err := c.MakeMetricsService("rook-mgr", "foo", serviceMetricName) + assert.NoError(t, err) assert.NotNil(t, s) assert.Equal(t, "rook-mgr", s.Name) assert.Equal(t, 1, len(s.Spec.Ports)) @@ -95,7 +98,8 @@ func TestServiceSpec(t *testing.T) { func TestHostNetwork(t *testing.T) { clientset := optest.New(t, 1) - clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", FSID: "myfsid"} + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", FSID: "myfsid", OwnerInfo: ownerInfo} clusterInfo.SetName("test") clusterSpec := cephv1.ClusterSpec{ Network: cephv1.NetworkSpec{HostNetwork: true}, @@ -123,7 +127,8 @@ func TestApplyPrometheusAnnotations(t *testing.T) { clusterSpec := cephv1.ClusterSpec{ DataDirHostPath: "/var/lib/rook/", } - clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", FSID: "myfsid"} + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", FSID: "myfsid", OwnerInfo: ownerInfo} clusterInfo.SetName("test") c := New(&clusterd.Context{Clientset: clientset}, clusterInfo, clusterSpec, "myversion") diff --git a/pkg/operator/ceph/cluster/mon/config.go b/pkg/operator/ceph/cluster/mon/config.go index a5e39ce0f..16e1ff6a3 100644 --- a/pkg/operator/ceph/cluster/mon/config.go +++ b/pkg/operator/ceph/cluster/mon/config.go @@ -83,7 +83,7 @@ func LoadClusterInfo(context *clusterd.Context, namespace string) (*cephclient.C } // CreateOrLoadClusterInfo constructs or loads a clusterinfo and returns it along with the maxMonID -func CreateOrLoadClusterInfo(clusterdContext *clusterd.Context, namespace string, ownerRef *metav1.OwnerReference) (*cephclient.ClusterInfo, int, *Mapping, error) { +func CreateOrLoadClusterInfo(clusterdContext *clusterd.Context, namespace string, ownerInfo *k8sutil.OwnerInfo) (*cephclient.ClusterInfo, int, *Mapping, error) { ctx := context.TODO() var clusterInfo *cephclient.ClusterInfo maxMonID := -1 @@ -96,7 +96,7 @@ func CreateOrLoadClusterInfo(clusterdContext *clusterd.Context, namespace string if !kerrors.IsNotFound(err) { return nil, maxMonID, monMapping, errors.Wrap(err, "failed to get mon secrets") } - if ownerRef == nil { + if ownerInfo == nil { return nil, maxMonID, monMapping, errors.New("not expected to create new cluster info and did not find existing secret") } @@ -105,7 +105,7 @@ func CreateOrLoadClusterInfo(clusterdContext *clusterd.Context, namespace string return nil, maxMonID, monMapping, errors.Wrap(err, "failed to create mon secrets") } - err = createClusterAccessSecret(clusterdContext.Clientset, namespace, clusterInfo, ownerRef) + err = createClusterAccessSecret(clusterdContext.Clientset, namespace, clusterInfo, ownerInfo) if err != nil { return nil, maxMonID, monMapping, err } @@ -263,7 +263,7 @@ func loadMonConfig(clientset kubernetes.Interface, namespace string) (map[string return monEndpointMap, maxMonID, monMapping, nil } -func createClusterAccessSecret(clientset kubernetes.Interface, namespace string, clusterInfo *cephclient.ClusterInfo, ownerRef *metav1.OwnerReference) error { +func createClusterAccessSecret(clientset kubernetes.Interface, namespace string, clusterInfo *cephclient.ClusterInfo, ownerInfo *k8sutil.OwnerInfo) error { ctx := context.TODO() logger.Infof("creating mon secrets for a new cluster") var err error @@ -283,7 +283,10 @@ func createClusterAccessSecret(clientset kubernetes.Interface, namespace string, Data: secrets, Type: k8sutil.RookType, } - k8sutil.SetOwnerRef(&secret.ObjectMeta, ownerRef) + err = ownerInfo.SetControllerReference(secret) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference to mon secret %q", secret.Name) + } if _, err = clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { return errors.Wrap(err, "failed to save mon secrets") } @@ -367,7 +370,7 @@ func ExtractKey(contents string) (string, error) { } // PopulateExternalClusterInfo Add validation in the code to fail if the external cluster has no OSDs keep waiting -func PopulateExternalClusterInfo(context *clusterd.Context, namespace string, ownerRef metav1.OwnerReference) *cephclient.ClusterInfo { +func PopulateExternalClusterInfo(context *clusterd.Context, namespace string, ownerInfo *k8sutil.OwnerInfo) *cephclient.ClusterInfo { for { clusterInfo, _, _, err := LoadClusterInfo(context, namespace) if err != nil { @@ -377,7 +380,7 @@ func PopulateExternalClusterInfo(context *clusterd.Context, namespace string, ow continue } logger.Infof("found the cluster info to connect to the external cluster. will use %q to check health and monitor status. mons=%+v", clusterInfo.CephCred.Username, clusterInfo.Monitors) - clusterInfo.OwnerRef = ownerRef + clusterInfo.OwnerInfo = ownerInfo return clusterInfo } } diff --git a/pkg/operator/ceph/cluster/mon/config_test.go b/pkg/operator/ceph/cluster/mon/config_test.go index 6558fac75..b31bdc859 100644 --- a/pkg/operator/ceph/cluster/mon/config_test.go +++ b/pkg/operator/ceph/cluster/mon/config_test.go @@ -27,6 +27,7 @@ import ( "github.com/stretchr/testify/require" "github.com/rook/rook/pkg/clusterd" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -55,8 +56,8 @@ func TestCreateClusterSecrets(t *testing.T) { Executor: executor, } namespace := "ns" - ownerRef := &metav1.OwnerReference{} - info, maxID, mapping, err := CreateOrLoadClusterInfo(context, namespace, ownerRef) + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + info, maxID, mapping, err := CreateOrLoadClusterInfo(context, namespace, ownerInfo) assert.NoError(t, err) assert.Equal(t, -1, maxID) require.NotNil(t, info) @@ -79,7 +80,7 @@ func TestCreateClusterSecrets(t *testing.T) { assert.NoError(t, err) // Check that the cluster info can now be loaded - info, _, _, err = CreateOrLoadClusterInfo(context, namespace, ownerRef) + info, _, _, err = CreateOrLoadClusterInfo(context, namespace, ownerInfo) assert.NoError(t, err) assert.Equal(t, "client.admin", info.CephCred.Username) assert.Equal(t, adminSecret, info.CephCred.Secret) @@ -88,7 +89,7 @@ func TestCreateClusterSecrets(t *testing.T) { secret.Data[adminSecretNameKey] = []byte(adminSecretNameKey) _, err = clientset.CoreV1().Secrets(namespace).Update(ctx, secret, metav1.UpdateOptions{}) assert.NoError(t, err) - _, _, _, err = CreateOrLoadClusterInfo(context, namespace, ownerRef) + _, _, _, err = CreateOrLoadClusterInfo(context, namespace, ownerInfo) assert.Error(t, err) // Load the external cluster with the legacy external creds @@ -99,7 +100,7 @@ func TestCreateClusterSecrets(t *testing.T) { } _, err = clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) assert.NoError(t, err) - info, _, _, err = CreateOrLoadClusterInfo(context, namespace, ownerRef) + info, _, _, err = CreateOrLoadClusterInfo(context, namespace, ownerInfo) assert.NoError(t, err) assert.Equal(t, "testid", info.CephCred.Username) assert.Equal(t, "testkey", info.CephCred.Secret) diff --git a/pkg/operator/ceph/cluster/mon/health_test.go b/pkg/operator/ceph/cluster/mon/health_test.go index 773f771cf..9e6ffd2c4 100644 --- a/pkg/operator/ceph/cluster/mon/health_test.go +++ b/pkg/operator/ceph/cluster/mon/health_test.go @@ -29,6 +29,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test" "github.com/rook/rook/pkg/operator/ceph/config" testopk8s "github.com/rook/rook/pkg/operator/k8sutil/test" @@ -61,7 +62,8 @@ func TestCheckHealth(t *testing.T) { Executor: executor, RequestCancelOrchestration: abool.New(), } - c := New(context, "ns", cephv1.ClusterSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{}) // clusterInfo is nil so we return err err := c.checkHealth() assert.NotNil(t, err) @@ -152,7 +154,8 @@ func TestScaleMonDeployment(t *testing.T) { ctx := context.TODO() clientset := test.New(t, 1) context := &clusterd.Context{Clientset: clientset} - c := New(context, "ns", cephv1.ClusterSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{}) setCommonMonProperties(c, 1, cephv1.MonSpec{Count: 0, AllowMultiplePerNode: true}, "myversion") name := "a" @@ -199,7 +202,8 @@ func TestCheckHealthNotFound(t *testing.T) { Executor: executor, RequestCancelOrchestration: abool.New(), } - c := New(context, "ns", cephv1.ClusterSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{}) setCommonMonProperties(c, 2, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") c.waitForStart = false defer os.RemoveAll(c.context.ConfigDir) @@ -257,7 +261,8 @@ func TestAddRemoveMons(t *testing.T) { Executor: executor, RequestCancelOrchestration: abool.New(), } - c := New(context, "ns", cephv1.ClusterSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{}) setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 5, AllowMultiplePerNode: true}, "myversion") c.maxMonID = 0 // "a" is max mon id c.waitForStart = false diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index db60f3683..5a6bb15ef 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -120,7 +120,7 @@ type Cluster struct { waitForStart bool monTimeoutList map[string]time.Time mapping *Mapping - ownerRef metav1.OwnerReference + ownerInfo *k8sutil.OwnerInfo csiConfigMutex *sync.Mutex isUpgrade bool arbiterMon string @@ -165,7 +165,7 @@ type SchedulingResult struct { } // New creates an instance of a mon cluster -func New(context *clusterd.Context, namespace string, spec cephv1.ClusterSpec, ownerRef metav1.OwnerReference, csiConfigMutex *sync.Mutex) *Cluster { +func New(context *clusterd.Context, namespace string, spec cephv1.ClusterSpec, ownerInfo *k8sutil.OwnerInfo, csiConfigMutex *sync.Mutex) *Cluster { return &Cluster{ context: context, spec: spec, @@ -176,7 +176,7 @@ func New(context *clusterd.Context, namespace string, spec cephv1.ClusterSpec, o mapping: &Mapping{ Schedule: map[string]*MonScheduleInfo{}, }, - ownerRef: ownerRef, + ownerInfo: ownerInfo, csiConfigMutex: csiConfigMutex, } } @@ -188,7 +188,7 @@ func (c *Cluster) Start(clusterInfo *cephclient.ClusterInfo, rookVersion string, c.acquireOrchestrationLock() defer c.releaseOrchestrationLock() - clusterInfo.OwnerRef = c.ownerRef + clusterInfo.OwnerInfo = c.ownerInfo c.ClusterInfo = clusterInfo c.rookVersion = rookVersion c.spec = spec @@ -517,20 +517,20 @@ func (c *Cluster) initClusterInfo(cephVersion cephver.CephVersion) error { var err error // get the cluster info from secret - c.ClusterInfo, c.maxMonID, c.mapping, err = CreateOrLoadClusterInfo(c.context, c.Namespace, &c.ownerRef) + c.ClusterInfo, c.maxMonID, c.mapping, err = CreateOrLoadClusterInfo(c.context, c.Namespace, c.ownerInfo) if err != nil { return errors.Wrap(err, "failed to get cluster info") } c.ClusterInfo.CephVersion = cephVersion - c.ClusterInfo.OwnerRef = c.ownerRef + c.ClusterInfo.OwnerInfo = c.ownerInfo // save cluster monitor config if err = c.saveMonConfig(); err != nil { return errors.Wrap(err, "failed to save mons") } - k := keyring.GetSecretStore(c.context, c.ClusterInfo, &c.ownerRef) + k := keyring.GetSecretStore(c.context, c.ClusterInfo, c.ownerInfo) // store the keyring which all mons share if err := k.CreateOrUpdate(keyringStoreName, c.genMonSharedKeyring()); err != nil { return errors.Wrap(err, "failed to save mon keyring secret") @@ -1054,7 +1054,10 @@ func (c *Cluster) saveMonConfig() error { Namespace: c.Namespace, }, } - k8sutil.SetOwnerRef(&configMap.ObjectMeta, &c.ownerRef) + err := c.ownerInfo.SetControllerReference(configMap) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference mon configmap %q", configMap.Name) + } monMapping, err := json.Marshal(c.mapping) if err != nil { @@ -1099,7 +1102,7 @@ func (c *Cluster) saveMonConfig() error { // Every time the mon config is updated, must also update the global config so that all daemons // have the most updated version if they restart. - if err := config.GetStore(c.context, c.Namespace, &c.ownerRef).CreateOrUpdate(c.ClusterInfo); err != nil { + if err := config.GetStore(c.context, c.Namespace, c.ownerInfo).CreateOrUpdate(c.ClusterInfo); err != nil { return errors.Wrap(err, "failed to update the global config") } diff --git a/pkg/operator/ceph/cluster/mon/mon_test.go b/pkg/operator/ceph/cluster/mon/mon_test.go index e2310a918..124e87d7c 100644 --- a/pkg/operator/ceph/cluster/mon/mon_test.go +++ b/pkg/operator/ceph/cluster/mon/mon_test.go @@ -34,6 +34,7 @@ import ( rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test" "github.com/rook/rook/pkg/operator/ceph/config" cephver "github.com/rook/rook/pkg/operator/ceph/version" @@ -106,6 +107,7 @@ func newTestStartClusterWithQuorumResponse(t *testing.T, namespace string, monRe } func newCluster(context *clusterd.Context, namespace string, allowMultiplePerNode bool, resources v1.ResourceRequirements) *Cluster { + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() return &Cluster{ ClusterInfo: nil, context: context, @@ -124,7 +126,7 @@ func newCluster(context *clusterd.Context, namespace string, allowMultiplePerNod mapping: &Mapping{ Schedule: map[string]*MonScheduleInfo{}, }, - ownerRef: metav1.OwnerReference{}, + ownerInfo: ownerInfo, } } @@ -235,7 +237,8 @@ func TestSaveMonEndpoints(t *testing.T) { clientset := test.New(t, 1) configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", cephv1.ClusterSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{}) setCommonMonProperties(c, 1, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") // create the initial config map @@ -282,7 +285,8 @@ func TestMaxMonID(t *testing.T) { clientset := test.New(t, 1) configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", cephv1.ClusterSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{}) // when the configmap is not found, the maxMonID is -1 maxMonID, err := c.getStoredMaxMonID() diff --git a/pkg/operator/ceph/cluster/mon/node_test.go b/pkg/operator/ceph/cluster/mon/node_test.go index 479e046f1..5e99ffba1 100644 --- a/pkg/operator/ceph/cluster/mon/node_test.go +++ b/pkg/operator/ceph/cluster/mon/node_test.go @@ -27,6 +27,7 @@ import ( "github.com/rook/rook/pkg/clusterd" clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test" cephver "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" @@ -37,7 +38,7 @@ import ( func TestNodeAffinity(t *testing.T) { ctx := context.TODO() clientset := test.New(t, 4) - c := New(&clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) + c := New(&clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{}, &k8sutil.OwnerInfo{}, &sync.Mutex{}) setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") c.spec.Placement = map[rookv1.KeyType]rookv1.Placement{} @@ -167,7 +168,7 @@ func TestPodMemory(t *testing.T) { func TestHostNetwork(t *testing.T) { clientset := test.New(t, 3) - c := New(&clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) + c := New(&clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{}, &k8sutil.OwnerInfo{}, &sync.Mutex{}) setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") c.spec.Network.HostNetwork = true diff --git a/pkg/operator/ceph/cluster/mon/service.go b/pkg/operator/ceph/cluster/mon/service.go index d79a50bab..acfde8ff0 100644 --- a/pkg/operator/ceph/cluster/mon/service.go +++ b/pkg/operator/ceph/cluster/mon/service.go @@ -32,8 +32,9 @@ func (c *Cluster) createService(mon *monConfig) (string, error) { ctx := context.TODO() svcDef := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: mon.ResourceName, - Labels: c.getLabels(mon, false, true), + Name: mon.ResourceName, + Namespace: c.Namespace, + Labels: c.getLabels(mon, false, true), }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ @@ -49,7 +50,10 @@ func (c *Cluster) createService(mon *monConfig) (string, error) { Selector: c.getLabels(mon, false, false), }, } - k8sutil.SetOwnerRef(&svcDef.ObjectMeta, &c.ownerRef) + err := c.ownerInfo.SetOwnerReference(svcDef) + if err != nil { + return "", errors.Wrapf(err, "failed to set owner reference to mon service %q", svcDef.Name) + } // If deploying Nautilus or newer we need a new port for the monitor service addServicePort(svcDef, "tcp-msgr2", DefaultMsgr2Port) diff --git a/pkg/operator/ceph/cluster/mon/service_test.go b/pkg/operator/ceph/cluster/mon/service_test.go index 8f54e923e..95f525a8d 100644 --- a/pkg/operator/ceph/cluster/mon/service_test.go +++ b/pkg/operator/ceph/cluster/mon/service_test.go @@ -23,6 +23,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,7 +32,7 @@ import ( func TestCreateService(t *testing.T) { ctx := context.TODO() clientset := test.New(t, 1) - c := New(&clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) + c := New(&clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{}, &k8sutil.OwnerInfo{}, &sync.Mutex{}) m := &monConfig{ResourceName: "rook-ceph-mon-b", DaemonName: "b"} clusterIP, err := c.createService(m) assert.NoError(t, err) diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 64e503505..2ecd42ceb 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -90,7 +90,10 @@ func (c *Cluster) makeDeployment(monConfig *monConfig, canary bool) (*apps.Deplo cephv1.GetMonAnnotations(c.spec.Annotations).ApplyToObjectMeta(&d.ObjectMeta) cephv1.GetMonLabels(c.spec.Labels).ApplyToObjectMeta(&d.ObjectMeta) controller.AddCephVersionLabelToDeployment(c.ClusterInfo.CephVersion, d) - k8sutil.SetOwnerRef(&d.ObjectMeta, &c.ownerRef) + err := c.ownerInfo.SetControllerReference(d) + if err != nil { + return nil, errors.Wrapf(err, "failed to set owner reference to mon deployment %q", d.Name) + } pod, err := c.makeMonPod(monConfig, canary) if err != nil { @@ -135,7 +138,10 @@ func (c *Cluster) makeDeploymentPVC(m *monConfig, canary bool) (*v1.PersistentVo k8sutil.AddRookVersionLabelToObjectMeta(&pvc.ObjectMeta) cephv1.GetMonAnnotations(c.spec.Annotations).ApplyToObjectMeta(&pvc.ObjectMeta) controller.AddCephVersionLabelToObjectMeta(c.ClusterInfo.CephVersion, &pvc.ObjectMeta) - k8sutil.SetOwnerRef(&pvc.ObjectMeta, &c.ownerRef) + err := c.ownerInfo.SetControllerReference(pvc) + if err != nil { + return nil, errors.Wrapf(err, "failed to set owner reference to mon pvc %q", pvc.Name) + } // k8s uses limit as the resource request fallback if _, ok := pvc.Spec.Resources.Limits[v1.ResourceStorage]; ok { diff --git a/pkg/operator/ceph/cluster/mon/spec_test.go b/pkg/operator/ceph/cluster/mon/spec_test.go index d9d8efc47..d7a49760a 100644 --- a/pkg/operator/ceph/cluster/mon/spec_test.go +++ b/pkg/operator/ceph/cluster/mon/spec_test.go @@ -23,14 +23,15 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" "github.com/rook/rook/pkg/clusterd" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/controller" - cephtest "github.com/rook/rook/pkg/operator/ceph/test" + "github.com/rook/rook/pkg/operator/ceph/test" + "github.com/rook/rook/pkg/operator/k8sutil" testop "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestPodSpecs(t *testing.T) { @@ -42,11 +43,12 @@ func TestPodSpecs(t *testing.T) { func testPodSpec(t *testing.T, monID string, pvc bool) { clientset := testop.New(t, 1) + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() c := New( &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook"}, "ns", cephv1.ClusterSpec{}, - metav1.OwnerReference{}, + ownerInfo, &sync.Mutex{}, ) setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "rook/rook:myversion") @@ -80,10 +82,10 @@ func testPodSpec(t *testing.T, monID string, pvc bool) { } // Deployment should have Ceph labels - cephtest.AssertLabelsContainCephRequirements(t, d.ObjectMeta.Labels, + test.AssertLabelsContainCephRequirements(t, d.ObjectMeta.Labels, config.MonType, monID, AppName, "ns") - podTemplate := cephtest.NewPodTemplateSpecTester(t, &d.Spec.Template) + podTemplate := test.NewPodTemplateSpecTester(t, &d.Spec.Template) podTemplate.RunFullSuite(config.MonType, monID, AppName, "ns", "ceph/ceph:myceph", "200", "100", "1337", "500", /* resources */ "my-priority-class") @@ -91,11 +93,12 @@ func testPodSpec(t *testing.T, monID string, pvc bool) { func TestDeploymentPVCSpec(t *testing.T) { clientset := testop.New(t, 1) + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() c := New( &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook"}, "ns", cephv1.ClusterSpec{}, - metav1.OwnerReference{}, + ownerInfo, &sync.Mutex{}, ) setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "rook/rook:myversion") @@ -155,7 +158,7 @@ func testRequiredDuringScheduling(t *testing.T, hostNetwork, allowMultiplePerNod &clusterd.Context{}, "ns", cephv1.ClusterSpec{}, - metav1.OwnerReference{}, + &k8sutil.OwnerInfo{}, &sync.Mutex{}, ) diff --git a/pkg/operator/ceph/cluster/osd/deviceSet.go b/pkg/operator/ceph/cluster/osd/deviceSet.go index e91e6f932..7e867ac29 100644 --- a/pkg/operator/ceph/cluster/osd/deviceSet.go +++ b/pkg/operator/ceph/cluster/osd/deviceSet.go @@ -27,7 +27,6 @@ import ( rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/ceph/controller" - "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -156,8 +155,11 @@ func (c *Cluster) createDeviceSetPVC(existingPVCs map[string]*v1.PersistentVolum pvcID = deviceSetPVCID(deviceSetName, pvcTemplate.GetName(), setIndex) existingPVC = existingPVCs[pvcID] } - pvc := makeDeviceSetPVC(deviceSetName, pvcID, setIndex, pvcTemplate) - k8sutil.SetOwnerRef(&pvc.ObjectMeta, &c.clusterInfo.OwnerRef) + pvc := makeDeviceSetPVC(deviceSetName, pvcID, setIndex, pvcTemplate, c.clusterInfo.Namespace) + err := c.clusterInfo.OwnerInfo.SetControllerReference(pvc) + if err != nil { + return nil, errors.Wrapf(err, "failed to set owner reference to osd pvc %q", pvc.Name) + } if existingPVC != nil { logger.Infof("OSD PVC %q already exists", existingPVC.Name) @@ -199,7 +201,7 @@ func (c *Cluster) updatePVCIfChanged(desiredPVC *v1.PersistentVolumeClaim, curre } } -func makeDeviceSetPVC(deviceSetName, pvcID string, setIndex int, pvcTemplate v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim { +func makeDeviceSetPVC(deviceSetName, pvcID string, setIndex int, pvcTemplate v1.PersistentVolumeClaim, namespace string) *v1.PersistentVolumeClaim { pvcLabels := makeStorageClassDeviceSetPVCLabel(deviceSetName, pvcID, setIndex) // Add user provided labels to pvcTemplates @@ -213,6 +215,7 @@ func makeDeviceSetPVC(deviceSetName, pvcID string, setIndex int, pvcTemplate v1. // Use a generated name to avoid the possibility of two OSDs being created with the same ID. // If one is removed and a new one is created later with the same ID, the OSD would fail to start. GenerateName: pvcID, + Namespace: namespace, Labels: pvcLabels, Annotations: pvcTemplate.Annotations, }, diff --git a/pkg/operator/ceph/cluster/osd/envs.go b/pkg/operator/ceph/cluster/osd/envs.go index db7ef3ad5..fd23d10d0 100644 --- a/pkg/operator/ceph/cluster/osd/envs.go +++ b/pkg/operator/ceph/cluster/osd/envs.go @@ -56,7 +56,7 @@ var ( func (c *Cluster) getConfigEnvVars(osdProps osdProperties, dataDir string) []v1.EnvVar { envVars := []v1.EnvVar{ nodeNameEnvVar(osdProps.crushHostname), - {Name: "ROOK_CLUSTER_ID", Value: string(c.clusterInfo.OwnerRef.UID)}, + {Name: "ROOK_CLUSTER_ID", Value: string(c.clusterInfo.OwnerInfo.GetUID())}, {Name: "ROOK_CLUSTER_NAME", Value: string(c.clusterInfo.NamespacedName().Name)}, k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 1b3627b8d..2a494512b 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -95,7 +95,7 @@ func New(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, spec ce clusterInfo: clusterInfo, spec: spec, rookVersion: rookVersion, - kv: k8sutil.NewConfigMapKVStore(clusterInfo.Namespace, context.Clientset, clusterInfo.OwnerRef), + kv: k8sutil.NewConfigMapKVStore(clusterInfo.Namespace, context.Clientset, clusterInfo.OwnerInfo), } } diff --git a/pkg/operator/ceph/cluster/osd/osd_pvc_test.go b/pkg/operator/ceph/cluster/osd/osd_pvc_test.go index 3e920b6b9..324ee4da4 100644 --- a/pkg/operator/ceph/cluster/osd/osd_pvc_test.go +++ b/pkg/operator/ceph/cluster/osd/osd_pvc_test.go @@ -225,6 +225,7 @@ func testOSDsOnPVC(t *testing.T) { CephVersion: cephver.Nautilus, } clusterInfo.SetName("mycluster") + clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) executor := osdPVCTestExecutor(t, clientset, namespace) context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: executor, RequestCancelOrchestration: abool.New()} diff --git a/pkg/operator/ceph/cluster/osd/osd_test.go b/pkg/operator/ceph/cluster/osd/osd_test.go index 78b85cfa7..96a1126a4 100644 --- a/pkg/operator/ceph/cluster/osd/osd_test.go +++ b/pkg/operator/ceph/cluster/osd/osd_test.go @@ -142,6 +142,7 @@ func TestAddRemoveNode(t *testing.T) { CephVersion: cephver.Nautilus, } clusterInfo.SetName("rook-ceph-test") + clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) generateKey := "expected key" executor := &exectest.MockExecutor{ MockExecuteCommandWithOutputFile: func(command string, outFileArg string, args ...string) (string, error) { @@ -296,6 +297,7 @@ func TestAddNodeFailure(t *testing.T) { CephVersion: cephver.Nautilus, } clusterInfo.SetName("testcluster") + clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}, RequestCancelOrchestration: abool.New()} spec := cephv1.ClusterSpec{ DataDirHostPath: context.ConfigDir, @@ -383,6 +385,7 @@ func TestGetPVCHostName(t *testing.T) { func TestGetOSDInfo(t *testing.T) { clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"} clusterInfo.SetName("test") + clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) context := &clusterd.Context{} spec := cephv1.ClusterSpec{DataDirHostPath: "/rook"} c := New(context, clusterInfo, spec, "myversion") @@ -520,6 +523,7 @@ func TestDetectCrushLocation(t *testing.T) { func TestGetOSDInfoWithCustomRoot(t *testing.T) { clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"} clusterInfo.SetName("test") + clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) context := &clusterd.Context{} spec := cephv1.ClusterSpec{ DataDirHostPath: "/rook", diff --git a/pkg/operator/ceph/cluster/osd/provision_spec.go b/pkg/operator/ceph/cluster/osd/provision_spec.go index da124c53c..a18346be7 100644 --- a/pkg/operator/ceph/cluster/osd/provision_spec.go +++ b/pkg/operator/ceph/cluster/osd/provision_spec.go @@ -76,7 +76,10 @@ func (c *Cluster) makeJob(osdProps osdProperties, provisionConfig *provisionConf k8sutil.AddRookVersionLabelToJob(job) controller.AddCephVersionLabelToJob(c.clusterInfo.CephVersion, job) - k8sutil.SetOwnerRef(&job.ObjectMeta, &c.clusterInfo.OwnerRef) + err = c.clusterInfo.OwnerInfo.SetControllerReference(job) + if err != nil { + return nil, err + } // override the resources of all the init containers and main container with the expected osd prepare resources c.applyResourcesToAllContainers(&podSpec.Spec, cephv1.GetPrepareOSDResources(c.spec.Resources)) diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 473c277e3..7802ce899 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -653,7 +653,10 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC cephv1.GetOSDLabels(c.spec.Labels).ApplyToObjectMeta(&deployment.Spec.Template.ObjectMeta) controller.AddCephVersionLabelToDeployment(c.clusterInfo.CephVersion, deployment) controller.AddCephVersionLabelToDeployment(c.clusterInfo.CephVersion, deployment) - k8sutil.SetOwnerRef(&deployment.ObjectMeta, &c.clusterInfo.OwnerRef) + err := c.clusterInfo.OwnerInfo.SetControllerReference(deployment) + if err != nil { + return nil, errors.Wrapf(err, "failed to set owner reference to osd deployment %q", deployment.Name) + } // we are passing false in case of osd and prepare pods because we don't want to overlap placement of osd on PVC's and non-PVC's. p := cephv1.GetOSDPlacement(c.spec.Placement) diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index 60dcfe45f..016a89afd 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -29,6 +29,7 @@ import ( opconfig "github.com/rook/rook/pkg/operator/ceph/config" operatortest "github.com/rook/rook/pkg/operator/ceph/test" cephver "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/rook/rook/pkg/operator/k8sutil" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" @@ -38,6 +39,7 @@ import ( func TestPodContainer(t *testing.T) { cluster := &Cluster{rookVersion: "23", clusterInfo: client.AdminClusterInfo("myosd")} + cluster.clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) osdProps := osdProperties{ crushHostname: "node", devices: []rookv1.Device{}, @@ -89,6 +91,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { CephVersion: cephver.Nautilus, } clusterInfo.SetName("test") + clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}} spec := cephv1.ClusterSpec{ CephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v15"}, @@ -432,6 +435,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { CephVersion: cephver.Octopus, } clusterInfo.SetName("test") + clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) c := New(context, clusterInfo, spec, "rook/rook:myversion") deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.NoError(t, err) @@ -467,6 +471,7 @@ func TestStorageSpecConfig(t *testing.T) { CephVersion: cephver.Nautilus, } clusterInfo.SetName("testing") + clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}} spec := cephv1.ClusterSpec{ DataDirHostPath: context.ConfigDir, @@ -549,6 +554,8 @@ func TestHostNetwork(t *testing.T) { CephVersion: cephver.Nautilus, } clusterInfo.SetName("test") + + clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}} spec := cephv1.ClusterSpec{ Storage: storageSpec, @@ -588,6 +595,7 @@ func TestOsdPrepareResources(t *testing.T) { context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}} clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"} clusterInfo.SetName("test") + clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) spec := cephv1.ClusterSpec{ Resources: map[string]v1.ResourceRequirements{"prepareosd": { Limits: v1.ResourceList{ @@ -609,7 +617,7 @@ func TestOsdPrepareResources(t *testing.T) { } func TestClusterGetPVCEncryptionOpenInitContainerActivate(t *testing.T) { - c := New(&clusterd.Context{}, &cephclient.ClusterInfo{}, cephv1.ClusterSpec{}, "rook/rook:myversion") + c := New(&clusterd.Context{}, &cephclient.ClusterInfo{OwnerInfo: &k8sutil.OwnerInfo{}}, cephv1.ClusterSpec{}, "rook/rook:myversion") osdProperties := osdProperties{ pvc: v1.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc1", @@ -633,7 +641,7 @@ func TestClusterGetPVCEncryptionOpenInitContainerActivate(t *testing.T) { } func TestClusterGetPVCEncryptionInitContainerActivate(t *testing.T) { - c := New(&clusterd.Context{}, &cephclient.ClusterInfo{}, cephv1.ClusterSpec{}, "rook/rook:myversion") + c := New(&clusterd.Context{}, &cephclient.ClusterInfo{OwnerInfo: &k8sutil.OwnerInfo{}}, cephv1.ClusterSpec{}, "rook/rook:myversion") osdProperties := osdProperties{ pvc: v1.PersistentVolumeClaimVolumeSource{ ClaimName: "pvc1", diff --git a/pkg/operator/ceph/cluster/osd/status_test.go b/pkg/operator/ceph/cluster/osd/status_test.go index 971755e0d..9a0320855 100644 --- a/pkg/operator/ceph/cluster/osd/status_test.go +++ b/pkg/operator/ceph/cluster/osd/status_test.go @@ -45,7 +45,8 @@ func TestOrchestrationStatus(t *testing.T) { context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}} spec := cephv1.ClusterSpec{} c := New(context, clusterInfo, spec, "myversion") - kv := k8sutil.NewConfigMapKVStore(c.clusterInfo.Namespace, clientset, metav1.OwnerReference{}) + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + kv := k8sutil.NewConfigMapKVStore(c.clusterInfo.Namespace, clientset, ownerInfo) nodeName := "mynode" cmName := fmt.Sprintf(orchestrationStatusMapName, nodeName) diff --git a/pkg/operator/ceph/cluster/rbd/config.go b/pkg/operator/ceph/cluster/rbd/config.go index 12eb31ea7..c03fefbcc 100644 --- a/pkg/operator/ceph/cluster/rbd/config.go +++ b/pkg/operator/ceph/cluster/rbd/config.go @@ -64,7 +64,7 @@ type daemonConfig struct { ResourceName string // the name rook gives to mirror resources in k8s metadata DaemonID string // the ID of the Ceph daemon ("a", "b", ...) DataPathMap *config.DataPathMap // location to store data in container - ownerRef metav1.OwnerReference + ownerInfo *k8sutil.OwnerInfo } // PeerToken is the content of the peer token @@ -79,7 +79,7 @@ func (r *ReconcileCephRBDMirror) generateKeyring(clusterInfo *client.ClusterInfo ctx := context.TODO() user := fullDaemonName(daemonConfig.DaemonID) access := []string{"mon", "profile rbd-mirror", "osd", "profile rbd"} - s := keyring.GetSecretStore(r.context, clusterInfo, &daemonConfig.ownerRef) + s := keyring.GetSecretStore(r.context, clusterInfo, daemonConfig.ownerInfo) key, err := s.GenerateKey(user, access) if err != nil { diff --git a/pkg/operator/ceph/cluster/rbd/mirror.go b/pkg/operator/ceph/cluster/rbd/mirror.go index 672ed4c8c..4aee2151d 100644 --- a/pkg/operator/ceph/cluster/rbd/mirror.go +++ b/pkg/operator/ceph/cluster/rbd/mirror.go @@ -62,13 +62,14 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro logger.Infof("configure rbd-mirroring with %d workers", cephRBDMirror.Spec.Count) + ownerInfo := k8sutil.NewOwnerInfo(cephRBDMirror, r.scheme) daemonID := k8sutil.IndexToName(0) resourceName := fmt.Sprintf("%s-%s", AppName, daemonID) daemonConf := &daemonConfig{ DaemonID: daemonID, ResourceName: resourceName, DataPathMap: config.NewDatalessDaemonDataPathMap(cephRBDMirror.Namespace, r.cephClusterSpec.DataDirHostPath), - ownerRef: *ref, + ownerInfo: ownerInfo, } _, err = r.generateKeyring(r.clusterInfo, daemonConf) @@ -85,7 +86,7 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro // Set owner ref to cephRBDMirror object err = controllerutil.SetControllerReference(cephRBDMirror, d, r.scheme) if err != nil { - return errors.Wrapf(err, "failed to set owner reference for ceph rbd-mirror %q secret", d.Name) + return errors.Wrapf(err, "failed to set owner reference for ceph rbd-mirror deployment %q", d.Name) } // Set the deployment hash as an annotation diff --git a/pkg/operator/ceph/cluster/rbd/spec_test.go b/pkg/operator/ceph/cluster/rbd/spec_test.go index 2fe9bc3da..ea60e9e9c 100644 --- a/pkg/operator/ceph/cluster/rbd/spec_test.go +++ b/pkg/operator/ceph/cluster/rbd/spec_test.go @@ -23,7 +23,8 @@ import ( "github.com/rook/rook/pkg/client/clientset/versioned/scheme" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/config" - cephtest "github.com/rook/rook/pkg/operator/ceph/test" + + "github.com/rook/rook/pkg/operator/ceph/test" cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" @@ -91,10 +92,10 @@ func TestPodSpec(t *testing.T) { assert.Equal(t, 4, len(d.Spec.Template.Spec.Containers[0].VolumeMounts)) // Deployment should have Ceph labels - cephtest.AssertLabelsContainCephRequirements(t, d.ObjectMeta.Labels, + test.AssertLabelsContainCephRequirements(t, d.ObjectMeta.Labels, config.RbdMirrorType, "a", AppName, "ns") - podTemplate := cephtest.NewPodTemplateSpecTester(t, &d.Spec.Template) + podTemplate := test.NewPodTemplateSpecTester(t, &d.Spec.Template) podTemplate.RunFullSuite(config.RbdMirrorType, "a", AppName, "ns", "ceph/ceph:myceph", "200", "100", "600", "300", /* resources */ "my-priority-class") diff --git a/pkg/operator/ceph/cluster/utils.go b/pkg/operator/ceph/cluster/utils.go index 0aa177979..65ba8307b 100644 --- a/pkg/operator/ceph/cluster/utils.go +++ b/pkg/operator/ceph/cluster/utils.go @@ -30,7 +30,7 @@ import ( // populateConfigOverrideConfigMap creates the "rook-config-override" config map // Its content allows modifying Ceph configuration flags -func populateConfigOverrideConfigMap(clusterdContext *clusterd.Context, namespace string, ownerRef metav1.OwnerReference) error { +func populateConfigOverrideConfigMap(clusterdContext *clusterd.Context, namespace string, ownerInfo *k8sutil.OwnerInfo) error { ctx := context.TODO() placeholderConfig := map[string]string{ k8sutil.ConfigOverrideVal: "", @@ -38,13 +38,17 @@ func populateConfigOverrideConfigMap(clusterdContext *clusterd.Context, namespac cm := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: k8sutil.ConfigOverrideName, + Name: k8sutil.ConfigOverrideName, + Namespace: namespace, }, Data: placeholderConfig, } - k8sutil.SetOwnerRef(&cm.ObjectMeta, &ownerRef) - _, err := clusterdContext.Clientset.CoreV1().ConfigMaps(namespace).Create(ctx, cm, metav1.CreateOptions{}) + err := ownerInfo.SetControllerReference(cm) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference to override configmap %q", cm.Name) + } + _, err = clusterdContext.Clientset.CoreV1().ConfigMaps(namespace).Create(ctx, cm, metav1.CreateOptions{}) if err != nil && !kerrors.IsAlreadyExists(err) { return errors.Wrapf(err, "failed to create override configmap %s", namespace) } diff --git a/pkg/operator/ceph/cluster/version.go b/pkg/operator/ceph/cluster/version.go index 7b08af5c2..948acfdb9 100644 --- a/pkg/operator/ceph/cluster/version.go +++ b/pkg/operator/ceph/cluster/version.go @@ -119,7 +119,7 @@ func diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion, func (c *cluster) detectCephVersion(rookImage, cephImage string, timeout time.Duration) (*cephver.CephVersion, error) { logger.Infof("detecting the ceph image version for image %s...", cephImage) versionReporter, err := cmdreporter.New( - c.context.Clientset, &c.ownerRef, + c.context.Clientset, c.ownerInfo, detectVersionName, detectVersionName, c.Namespace, []string{"ceph"}, []string{"--version"}, rookImage, cephImage) diff --git a/pkg/operator/ceph/config/keyring/admin_test.go b/pkg/operator/ceph/config/keyring/admin_test.go index 595c9f074..beee9a2e7 100644 --- a/pkg/operator/ceph/config/keyring/admin_test.go +++ b/pkg/operator/ceph/config/keyring/admin_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/rook/rook/pkg/clusterd" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test" "github.com/rook/rook/pkg/operator/k8sutil" testop "github.com/rook/rook/pkg/operator/test" @@ -37,10 +38,10 @@ func TestAdminKeyringStore(t *testing.T) { Clientset: clientset, } ns := "test-ns" - owner := metav1.OwnerReference{} + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() clusterInfo := clienttest.CreateTestClusterInfo(1) clusterInfo.Namespace = ns - k := GetSecretStore(ctx, clusterInfo, &owner) + k := GetSecretStore(ctx, clusterInfo, ownerInfo) assertKeyringData := func(expectedKeyring string) { s, e := clientset.CoreV1().Secrets(ns).Get(ctxt, "rook-ceph-admin-keyring", metav1.GetOptions{}) @@ -68,9 +69,9 @@ func TestAdminVolumeAndMount(t *testing.T) { ctx := &clusterd.Context{ Clientset: clientset, } - owner := metav1.OwnerReference{} + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() clusterInfo := clienttest.CreateTestClusterInfo(1) - s := GetSecretStore(ctx, clusterInfo, &owner) + s := GetSecretStore(ctx, clusterInfo, ownerInfo) clusterInfo.CephCred.Secret = "adminsecretkey" err := s.Admin().CreateOrUpdate(clusterInfo) diff --git a/pkg/operator/ceph/config/keyring/store.go b/pkg/operator/ceph/config/keyring/store.go index e0aa9ed79..799bca9bb 100644 --- a/pkg/operator/ceph/config/keyring/store.go +++ b/pkg/operator/ceph/config/keyring/store.go @@ -42,15 +42,15 @@ const ( type SecretStore struct { context *clusterd.Context clusterInfo *client.ClusterInfo - ownerRef *metav1.OwnerReference + ownerInfo *k8sutil.OwnerInfo } // GetSecretStore returns a new SecretStore struct. -func GetSecretStore(context *clusterd.Context, clusterInfo *client.ClusterInfo, ownerRef *metav1.OwnerReference) *SecretStore { +func GetSecretStore(context *clusterd.Context, clusterInfo *client.ClusterInfo, ownerInfo *k8sutil.OwnerInfo) *SecretStore { return &SecretStore{ context: context, clusterInfo: clusterInfo, - ownerRef: ownerRef, + ownerInfo: ownerInfo, } } @@ -92,7 +92,10 @@ func (k *SecretStore) CreateOrUpdate(resourceName string, keyring string) error }, Type: k8sutil.RookType, } - k8sutil.SetOwnerRef(&secret.ObjectMeta, k.ownerRef) + err := k.ownerInfo.SetControllerReference(secret) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference to keyring secret %q", secret.Name) + } return k.CreateSecret(secret) } diff --git a/pkg/operator/ceph/config/keyring/store_test.go b/pkg/operator/ceph/config/keyring/store_test.go index a9772d6cb..4d2d288e8 100644 --- a/pkg/operator/ceph/config/keyring/store_test.go +++ b/pkg/operator/ceph/config/keyring/store_test.go @@ -24,6 +24,7 @@ import ( "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/k8sutil" testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" @@ -49,8 +50,8 @@ func TestGenerateKey(t *testing.T) { Executor: executor, } ns := "rook-ceph" - owner := metav1.OwnerReference{} - s := GetSecretStore(ctx, &client.ClusterInfo{Namespace: ns}, &owner) + ownerInfo := k8sutil.OwnerInfo{} + s := GetSecretStore(ctx, &client.ClusterInfo{Namespace: ns}, &ownerInfo) generateKey = "generatedsecretkey" failGenerateKey = false @@ -77,9 +78,9 @@ func TestKeyringStore(t *testing.T) { ctx := &clusterd.Context{ Clientset: clientset, } - owner := metav1.OwnerReference{} + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() ns := "rook-ceph" - k := GetSecretStore(ctx, &client.ClusterInfo{Namespace: ns}, &owner) + k := GetSecretStore(ctx, &client.ClusterInfo{Namespace: ns}, ownerInfo) assertKeyringData := func(keyringName, expectedKeyring string) { s, e := clientset.CoreV1().Secrets(ns).Get(ctxt, keyringName, metav1.GetOptions{}) @@ -123,8 +124,8 @@ func TestResourceVolumeAndMount(t *testing.T) { ctx := &clusterd.Context{ Clientset: clientset, } - owner := metav1.OwnerReference{} - k := GetSecretStore(ctx, &client.ClusterInfo{Namespace: "ns"}, &owner) + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + k := GetSecretStore(ctx, &client.ClusterInfo{Namespace: "ns"}, ownerInfo) err := k.CreateOrUpdate("test-resource", "qwertyuiop") assert.NoError(t, err) err = k.CreateOrUpdate("second-resource", "asdfgyhujkl") diff --git a/pkg/operator/ceph/config/store.go b/pkg/operator/ceph/config/store.go index b04da950f..6ab6c8bfa 100644 --- a/pkg/operator/ceph/config/store.go +++ b/pkg/operator/ceph/config/store.go @@ -47,16 +47,16 @@ type Store struct { configMapStore *k8sutil.ConfigMapKVStore namespace string context *clusterd.Context - ownerRef *metav1.OwnerReference + ownerInfo *k8sutil.OwnerInfo } // GetStore returns the Store for the cluster. -func GetStore(context *clusterd.Context, namespace string, ownerRef *metav1.OwnerReference) *Store { +func GetStore(context *clusterd.Context, namespace string, ownerInfo *k8sutil.OwnerInfo) *Store { return &Store{ - configMapStore: k8sutil.NewConfigMapKVStore(namespace, context.Clientset, *ownerRef), + configMapStore: k8sutil.NewConfigMapKVStore(namespace, context.Clientset, ownerInfo), namespace: namespace, context: context, - ownerRef: ownerRef, + ownerInfo: ownerInfo, } } @@ -92,9 +92,12 @@ func (s *Store) createOrUpdateMonHostSecrets(ctx context.Context, clusterInfo *c Type: k8sutil.RookType, } clientset := s.context.Clientset - k8sutil.SetOwnerRef(&secret.ObjectMeta, s.ownerRef) + err := s.ownerInfo.SetControllerReference(secret) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference to moh host secret %q", secret.Name) + } - _, err := clientset.CoreV1().Secrets(s.namespace).Get(ctx, StoreName, metav1.GetOptions{}) + _, err = clientset.CoreV1().Secrets(s.namespace).Get(ctx, StoreName, metav1.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { logger.Debugf("creating config secret %+v", secret) diff --git a/pkg/operator/ceph/config/store_test.go b/pkg/operator/ceph/config/store_test.go index cf6f3498a..735f55999 100644 --- a/pkg/operator/ceph/config/store_test.go +++ b/pkg/operator/ceph/config/store_test.go @@ -37,9 +37,9 @@ func TestStore(t *testing.T) { Clientset: clientset, } ns := "rook-ceph" - owner := metav1.OwnerReference{} + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() - s := GetStore(ctx, ns, &owner) + s := GetStore(ctx, ns, ownerInfo) assertConfigStore := func(ci *cephclient.ClusterInfo) { sec, e := clientset.CoreV1().Secrets(ns).Get(ctxt, StoreName, metav1.GetOptions{}) @@ -81,9 +81,9 @@ func TestEnvVarsAndFlags(t *testing.T) { Clientset: clientset, } ns := "rook-ceph" - owner := metav1.OwnerReference{} + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() - s := GetStore(ctx, ns, &owner) + s := GetStore(ctx, ns, ownerInfo) err := s.CreateOrUpdate(clienttest.CreateTestClusterInfo(3)) assert.NoError(t, err) diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index d03e6cc21..611dce029 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -136,12 +136,14 @@ func IsReadyToReconcile(c client.Client, clustercontext *clusterd.Context, names // ClusterOwnerRef represents the owner reference of the CephCluster CR func ClusterOwnerRef(clusterName, clusterID string) metav1.OwnerReference { blockOwner := true + controller := true return metav1.OwnerReference{ APIVersion: fmt.Sprintf("%s/%s", ClusterResource.Group, ClusterResource.Version), Kind: ClusterResource.Kind, Name: clusterName, UID: types.UID(clusterID), BlockOwnerDeletion: &blockOwner, + Controller: &controller, } } diff --git a/pkg/operator/ceph/csi/cluster_config.go b/pkg/operator/ceph/csi/cluster_config.go index df45b4f74..083f309b2 100644 --- a/pkg/operator/ceph/csi/cluster_config.go +++ b/pkg/operator/ceph/csi/cluster_config.go @@ -121,7 +121,7 @@ func UpdateCsiClusterConfig( // CreateCsiConfigMap creates an empty config map that will be later used // to provide cluster configuration to ceph-csi. If a config map already // exists, it will return it. -func CreateCsiConfigMap(namespace string, clientset kubernetes.Interface, ownerRef *metav1.OwnerReference) error { +func CreateCsiConfigMap(namespace string, clientset kubernetes.Interface, ownerInfo *k8sutil.OwnerInfo) error { ctx := context.TODO() configMap := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -133,8 +133,11 @@ func CreateCsiConfigMap(namespace string, clientset kubernetes.Interface, ownerR ConfigKey: "[]", } - k8sutil.SetOwnerRef(&configMap.ObjectMeta, ownerRef) - _, err := clientset.CoreV1().ConfigMaps(namespace).Create(ctx, configMap, metav1.CreateOptions{}) + err := ownerInfo.SetControllerReference(configMap) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference to csi configmap %q", configMap.Name) + } + _, err = clientset.CoreV1().ConfigMaps(namespace).Create(ctx, configMap, metav1.CreateOptions{}) if err != nil { if !k8serrors.IsAlreadyExists(err) { return errors.Wrapf(err, "failed to create initial csi config map %q (in %q)", configMap.Name, namespace) diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index cd138c390..99c83a125 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -23,16 +23,15 @@ import ( "github.com/rook/rook/pkg/clusterd" controllerutil "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/kubernetes" ) -func ValidateAndConfigureDrivers(context *clusterd.Context, namespace, rookImage, securityAccount string, serverVersion *version.Info, ownerRef *metav1.OwnerReference) { +func ValidateAndConfigureDrivers(context *clusterd.Context, namespace, rookImage, securityAccount string, serverVersion *version.Info, ownerInfo *k8sutil.OwnerInfo) { var v *CephCSIVersion var err error if !AllowUnsupported && CSIEnabled() { - if v, err = validateCSIVersion(context.Clientset, namespace, rookImage, securityAccount, ownerRef); err != nil { + if v, err = validateCSIVersion(context.Clientset, namespace, rookImage, securityAccount, ownerInfo); err != nil { logger.Errorf("invalid csi version. %+v", err) return } @@ -41,7 +40,7 @@ func ValidateAndConfigureDrivers(context *clusterd.Context, namespace, rookImage } if CSIEnabled() { - if err := startDrivers(context.Clientset, context.RookClientset, namespace, serverVersion, ownerRef, v); err != nil { + if err := startDrivers(context.Clientset, context.RookClientset, namespace, serverVersion, ownerInfo, v); err != nil { logger.Errorf("failed to start Ceph csi drivers. %v", err) return } diff --git a/pkg/operator/ceph/csi/secrets.go b/pkg/operator/ceph/csi/secrets.go index d43178f25..103560790 100644 --- a/pkg/operator/ceph/csi/secrets.go +++ b/pkg/operator/ceph/csi/secrets.go @@ -151,10 +151,13 @@ func createOrUpdateCSISecret(clusterInfo *client.ClusterInfo, csiRBDProvisionerS Data: secret, Type: k8sutil.RookType, } - k8sutil.SetOwnerRef(&s.ObjectMeta, &clusterInfo.OwnerRef) + err := clusterInfo.OwnerInfo.SetControllerReference(s) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference to keyring secret %q", secretName) + } // Create Kubernetes Secret - err := k.CreateSecret(s) + err = k.CreateSecret(s) if err != nil { return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", secret, clusterInfo.Namespace) } @@ -167,7 +170,7 @@ func createOrUpdateCSISecret(clusterInfo *client.ClusterInfo, csiRBDProvisionerS // CreateCSISecrets creates all the Kubernetes CSI Secrets func CreateCSISecrets(context *clusterd.Context, clusterInfo *client.ClusterInfo) error { - k := keyring.GetSecretStore(context, clusterInfo, &clusterInfo.OwnerRef) + k := keyring.GetSecretStore(context, clusterInfo, clusterInfo.OwnerInfo) // Create CSI RBD Provisioner Ceph key csiRBDProvisionerSecretKey, err := createCSIKeyringRBDProvisioner(k) diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index b036dd29b..8e67f45b9 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -206,7 +206,7 @@ func ValidateCSIParam() error { return nil } -func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Interface, namespace string, ver *version.Info, ownerRef *metav1.OwnerReference, v *CephCSIVersion) error { +func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Interface, namespace string, ver *version.Info, ownerInfo *k8sutil.OwnerInfo, v *CephCSIVersion) error { ctx := context.TODO() var ( err error @@ -382,6 +382,7 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter if err != nil { return errors.Wrap(err, "failed to load rbd plugin service template") } + rbdService.Namespace = namespace logger.Info("successfully started CSI Ceph RBD") } if EnableCephFS { @@ -399,6 +400,7 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter if err != nil { return errors.Wrap(err, "failed to load cephfs plugin service template") } + cephfsService.Namespace = namespace logger.Info("successfully started CSI CephFS driver") } @@ -412,7 +414,10 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter applyToPodSpec(&rbdPlugin.Spec.Template.Spec, pluginNodeAffinity, pluginTolerations) // apply resource request and limit to rbdplugin containers applyResourcesToContainers(clientset, rbdPluginResource, &rbdPlugin.Spec.Template.Spec) - k8sutil.SetOwnerRef(&rbdPlugin.ObjectMeta, ownerRef) + err = ownerInfo.SetControllerReference(rbdPlugin) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference to rbd plugin daemonset %q", rbdPlugin.Name) + } multusApplied, err := applyCephClusterNetworkConfig(ctx, &rbdPlugin.Spec.Template.ObjectMeta, rookclientset) if err != nil { return errors.Wrapf(err, "failed to apply network config to rbd plugin daemonset: %+v", rbdPlugin) @@ -431,7 +436,10 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter applyToPodSpec(&rbdProvisionerDeployment.Spec.Template.Spec, provisionerNodeAffinity, provisionerTolerations) // apply resource request and limit to rbd provisioner containers applyResourcesToContainers(clientset, rbdProvisionerResource, &rbdProvisionerDeployment.Spec.Template.Spec) - k8sutil.SetOwnerRef(&rbdProvisionerDeployment.ObjectMeta, ownerRef) + err = ownerInfo.SetControllerReference(rbdProvisionerDeployment) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference to rbd provisioner deployment %q", rbdProvisionerDeployment.Name) + } antiAffinity := GetPodAntiAffinity("app", csiRBDProvisioner) rbdProvisionerDeployment.Spec.Template.Spec.Affinity.PodAntiAffinity = &antiAffinity rbdProvisionerDeployment.Spec.Strategy = apps.DeploymentStrategy{ @@ -450,7 +458,11 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter } if rbdService != nil { - k8sutil.SetOwnerRef(&rbdService.ObjectMeta, ownerRef) + rbdService.Namespace = namespace + err = ownerInfo.SetControllerReference(rbdService) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference to rbd service %q", rbdService) + } _, err = k8sutil.CreateOrUpdateService(clientset, namespace, rbdService) if err != nil { return errors.Wrapf(err, "failed to create rbd service: %+v", rbdService) @@ -461,7 +473,10 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter applyToPodSpec(&cephfsPlugin.Spec.Template.Spec, pluginNodeAffinity, pluginTolerations) // apply resource request and limit to cephfs plugin containers applyResourcesToContainers(clientset, cephFSPluginResource, &cephfsPlugin.Spec.Template.Spec) - k8sutil.SetOwnerRef(&cephfsPlugin.ObjectMeta, ownerRef) + err = ownerInfo.SetControllerReference(cephfsPlugin) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference to cephfs plugin daemonset %q", cephfsPlugin.Name) + } multusApplied, err := applyCephClusterNetworkConfig(ctx, &cephfsPlugin.Spec.Template.ObjectMeta, rookclientset) if err != nil { return errors.Wrapf(err, "failed to apply network config to cephfs plugin daemonset: %+v", cephfsPlugin) @@ -481,7 +496,10 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter // get resource details for cephfs provisioner // apply resource request and limit to cephfs provisioner containers applyResourcesToContainers(clientset, cephFSProvisionerResource, &cephfsProvisionerDeployment.Spec.Template.Spec) - k8sutil.SetOwnerRef(&cephfsProvisionerDeployment.ObjectMeta, ownerRef) + err = ownerInfo.SetControllerReference(cephfsProvisionerDeployment) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference to cephfs provisioner deployment %q", cephfsProvisionerDeployment.Name) + } antiAffinity := GetPodAntiAffinity("app", csiCephFSProvisioner) cephfsProvisionerDeployment.Spec.Template.Spec.Affinity.PodAntiAffinity = &antiAffinity cephfsProvisionerDeployment.Spec.Strategy = apps.DeploymentStrategy{ @@ -499,10 +517,13 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter k8sutil.AddRookVersionLabelToDeployment(cephfsProvisionerDeployment) } if cephfsService != nil { - k8sutil.SetOwnerRef(&cephfsService.ObjectMeta, ownerRef) + err = ownerInfo.SetControllerReference(cephfsService) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference to cephfs service %q", cephfsService) + } _, err = k8sutil.CreateOrUpdateService(clientset, namespace, cephfsService) if err != nil { - return errors.Wrapf(err, "failed to create rbd service: %+v", cephfsService) + return errors.Wrapf(err, "failed to create cephfs service: %+v", cephfsService) } } @@ -678,14 +699,14 @@ func deleteCSIDriverInfo(ctx context.Context, clientset kubernetes.Interface, na } // ValidateCSIVersion checks if the configured ceph-csi image is supported -func validateCSIVersion(clientset kubernetes.Interface, namespace, rookImage, serviceAccountName string, ownerRef *metav1.OwnerReference) (*CephCSIVersion, error) { +func validateCSIVersion(clientset kubernetes.Interface, namespace, rookImage, serviceAccountName string, ownerInfo *k8sutil.OwnerInfo) (*CephCSIVersion, error) { timeout := 15 * time.Minute logger.Infof("detecting the ceph csi image version for image %q", CSIParam.CSIPluginImage) versionReporter, err := cmdreporter.New( clientset, - ownerRef, + ownerInfo, detectCSIVersionName, detectCSIVersionName, namespace, []string{"cephcsi"}, []string{"--version"}, rookImage, CSIParam.CSIPluginImage) diff --git a/pkg/operator/ceph/file/controller.go b/pkg/operator/ceph/file/controller.go index 825c325e6..fe3ae6532 100644 --- a/pkg/operator/ceph/file/controller.go +++ b/pkg/operator/ceph/file/controller.go @@ -276,13 +276,6 @@ func (r *ReconcileCephFilesystem) reconcileCreateFilesystem(cephFilesystem *ceph } } - // Create the controller owner ref - // It will be associated to all resources of the CephObjectStore - ref, err := opcontroller.GetControllerObjectOwnerReference(cephFilesystem, r.scheme) - if err != nil || ref == nil { - return reconcile.Result{}, errors.Wrapf(err, "failed to get controller %q owner reference", cephFilesystem.Name) - } - // preservePoolsOnDelete being set to true has data-loss concerns and is deprecated (see #6492). // If preservePoolsOnDelete is set to true, assume the user means preserveFilesystemOnDelete instead. if cephFilesystem.Spec.PreservePoolsOnDelete { @@ -292,7 +285,8 @@ func (r *ReconcileCephFilesystem) reconcileCreateFilesystem(cephFilesystem *ceph } } - err = createFilesystem(r.context, r.clusterInfo, *cephFilesystem, r.cephClusterSpec, *ref, r.cephClusterSpec.DataDirHostPath, r.scheme) + ownerInfo := k8sutil.NewOwnerInfo(cephFilesystem, r.scheme) + err := createFilesystem(r.context, r.clusterInfo, *cephFilesystem, r.cephClusterSpec, ownerInfo, r.cephClusterSpec.DataDirHostPath) if err != nil { return reconcile.Result{}, errors.Wrapf(err, "failed to create filesystem %q", cephFilesystem.Name) } @@ -312,13 +306,8 @@ func (r *ReconcileCephFilesystem) reconcileCreateFilesystem(cephFilesystem *ceph } func (r *ReconcileCephFilesystem) reconcileDeleteFilesystem(cephFilesystem *cephv1.CephFilesystem) error { - // Create the controller owner ref - ref, err := opcontroller.GetControllerObjectOwnerReference(cephFilesystem, r.scheme) - if err != nil || ref == nil { - return errors.Wrapf(err, "failed to get controller %q owner reference", cephFilesystem.Name) - } - - err = deleteFilesystem(r.context, r.clusterInfo, *cephFilesystem, r.cephClusterSpec, *ref, r.cephClusterSpec.DataDirHostPath, r.scheme) + ownerInfo := k8sutil.NewOwnerInfo(cephFilesystem, r.scheme) + err := deleteFilesystem(r.context, r.clusterInfo, *cephFilesystem, r.cephClusterSpec, ownerInfo, r.cephClusterSpec.DataDirHostPath) if err != nil { return err } diff --git a/pkg/operator/ceph/file/filesystem.go b/pkg/operator/ceph/file/filesystem.go index 87ff4883d..594c8c411 100644 --- a/pkg/operator/ceph/file/filesystem.go +++ b/pkg/operator/ceph/file/filesystem.go @@ -30,8 +30,6 @@ import ( cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/file/mds" "github.com/rook/rook/pkg/operator/ceph/pool" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" ) const ( @@ -51,9 +49,8 @@ func createFilesystem( clusterInfo *cephclient.ClusterInfo, fs cephv1.CephFilesystem, clusterSpec *cephv1.ClusterSpec, - ownerRefs metav1.OwnerReference, + ownerInfo *k8sutil.OwnerInfo, dataDirHostPath string, - scheme *runtime.Scheme, ) error { if len(fs.Spec.DataPools) != 0 { f := newFS(fs.Name, fs.Namespace) @@ -81,7 +78,7 @@ func createFilesystem( } logger.Infof("start running mdses for filesystem %q", fs.Name) - c := mds.NewCluster(clusterInfo, context, clusterSpec, fs, filesystem, ownerRefs, dataDirHostPath, scheme) + c := mds.NewCluster(clusterInfo, context, clusterSpec, fs, filesystem, ownerInfo, dataDirHostPath) if err := c.Start(); err != nil { return err } @@ -95,9 +92,8 @@ func deleteFilesystem( clusterInfo *cephclient.ClusterInfo, fs cephv1.CephFilesystem, clusterSpec *cephv1.ClusterSpec, - ownerRefs metav1.OwnerReference, + ownerInfo *k8sutil.OwnerInfo, dataDirHostPath string, - scheme *runtime.Scheme, ) error { filesystem, err := client.GetFilesystem(context, clusterInfo, fs.Name) if err != nil { @@ -107,7 +103,7 @@ func deleteFilesystem( } return errors.Wrapf(err, "failed to get filesystem %q", fs.Name) } - c := mds.NewCluster(clusterInfo, context, clusterSpec, fs, filesystem, ownerRefs, dataDirHostPath, scheme) + c := mds.NewCluster(clusterInfo, context, clusterSpec, fs, filesystem, ownerInfo, dataDirHostPath) // Delete mds CephX keys and configuration in centralized mon database replicas := fs.Spec.MetadataServer.ActiveCount * 2 diff --git a/pkg/operator/ceph/file/filesystem_test.go b/pkg/operator/ceph/file/filesystem_test.go index b10a72438..2ce81fa98 100644 --- a/pkg/operator/ceph/file/filesystem_test.go +++ b/pkg/operator/ceph/file/filesystem_test.go @@ -28,12 +28,13 @@ import ( "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - "github.com/rook/rook/pkg/client/clientset/versioned/scheme" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test" "github.com/rook/rook/pkg/operator/ceph/file/mds" "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/rook/rook/pkg/operator/k8sutil" testopk8s "github.com/rook/rook/pkg/operator/k8sutil/test" testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" @@ -218,14 +219,15 @@ func TestCreateFilesystem(t *testing.T) { clusterInfo := &client.ClusterInfo{FSID: "myfsid", CephVersion: version.Octopus} // start a basic cluster - err := createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", scheme.Scheme) + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + err := createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, ownerInfo, "/var/lib/rook/") assert.Nil(t, err) validateStart(ctx, t, context, fs) assert.ElementsMatch(t, []string{}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated)) testopk8s.ClearDeploymentsUpdated(deploymentsUpdated) // starting again should be a no-op - err = createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", scheme.Scheme) + err = createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, ownerInfo, "/var/lib/rook/") assert.Nil(t, err) validateStart(ctx, t, context, fs) assert.ElementsMatch(t, []string{fmt.Sprintf("rook-ceph-mds-%s-a", fsName), fmt.Sprintf("rook-ceph-mds-%s-b", fsName)}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated)) @@ -266,7 +268,7 @@ func TestCreateFilesystem(t *testing.T) { Clientset: clientset} fs.Spec.DataPools = append(fs.Spec.DataPools, cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}}) - err = createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", scheme.Scheme) + err = createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, ownerInfo, "/var/lib/rook/") assert.Nil(t, err) validateStart(ctx, t, context, fs) assert.ElementsMatch(t, []string{fmt.Sprintf("rook-ceph-mds-%s-a", fsName), fmt.Sprintf("rook-ceph-mds-%s-b", fsName)}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated)) @@ -292,7 +294,7 @@ func TestCreateFilesystem(t *testing.T) { } // Create another filesystem which should fail - err = createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", scheme.Scheme) + err = createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, &k8sutil.OwnerInfo{}, "/var/lib/rook/") assert.Error(t, err) assert.Equal(t, fmt.Sprintf("failed to create filesystem %q: multiple filesystems are only supported as of ceph pacific", fsName), err.Error()) @@ -307,7 +309,7 @@ func TestCreateFilesystem(t *testing.T) { ConfigDir: configDir, Clientset: clientset, } - err = createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", scheme.Scheme) + err = createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, ownerInfo, "/var/lib/rook/") assert.NoError(t, err) os.Unsetenv(client.MultiFsEnv) @@ -321,7 +323,7 @@ func TestCreateFilesystem(t *testing.T) { ConfigDir: configDir, Clientset: clientset, } - err = createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", scheme.Scheme) + err = createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, ownerInfo, "/var/lib/rook/") assert.NoError(t, err) } @@ -358,12 +360,13 @@ func TestCreateNopoolFilesystem(t *testing.T) { clusterInfo := &client.ClusterInfo{FSID: "myfsid"} // start a basic cluster - err := createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", scheme.Scheme) + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + err := createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, ownerInfo, "/var/lib/rook/") assert.Nil(t, err) validateStart(ctx, t, context, fs) // starting again should be a no-op - err = createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", scheme.Scheme) + err = createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, ownerInfo, "/var/lib/rook/") assert.Nil(t, err) validateStart(ctx, t, context, fs) } diff --git a/pkg/operator/ceph/file/mds/config.go b/pkg/operator/ceph/file/mds/config.go index d91ec4c68..618976c6d 100644 --- a/pkg/operator/ceph/file/mds/config.go +++ b/pkg/operator/ceph/file/mds/config.go @@ -44,7 +44,7 @@ func (c *Cluster) generateKeyring(m *mdsConfig) (string, error) { access := []string{"osd", "allow *", "mds", "allow", "mon", "allow profile mds"} // At present - s := keyring.GetSecretStore(c.context, c.clusterInfo, &c.ownerRef) + s := keyring.GetSecretStore(c.context, c.clusterInfo, c.ownerInfo) key, err := s.GenerateKey(user, access) if err != nil { diff --git a/pkg/operator/ceph/file/mds/mds.go b/pkg/operator/ceph/file/mds/mds.go index d3a651b19..1aa9e4e9d 100644 --- a/pkg/operator/ceph/file/mds/mds.go +++ b/pkg/operator/ceph/file/mds/mds.go @@ -37,8 +37,6 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mds") @@ -59,9 +57,8 @@ type Cluster struct { clusterSpec *cephv1.ClusterSpec fs cephv1.CephFilesystem fsID string - ownerRef metav1.OwnerReference + ownerInfo *k8sutil.OwnerInfo dataDirHostPath string - scheme *runtime.Scheme } type mdsConfig struct { @@ -77,9 +74,8 @@ func NewCluster( clusterSpec *cephv1.ClusterSpec, fs cephv1.CephFilesystem, fsdetails *client.CephFilesystemDetails, - ownerRef metav1.OwnerReference, + ownerInfo *k8sutil.OwnerInfo, dataDirHostPath string, - scheme *runtime.Scheme, ) *Cluster { return &Cluster{ clusterInfo: clusterInfo, @@ -87,9 +83,8 @@ func NewCluster( clusterSpec: clusterSpec, fs: fs, fsID: strconv.Itoa(fsdetails.ID), - ownerRef: ownerRef, + ownerInfo: ownerInfo, dataDirHostPath: dataDirHostPath, - scheme: scheme, } } @@ -157,14 +152,14 @@ func (c *Cluster) Start() error { } // start the deployment - d, err := c.makeDeployment(mdsConfig) + d, err := c.makeDeployment(mdsConfig, c.fs.Namespace) if err != nil { return errors.Wrapf(err, "failed to create deployment") } // Set owner ref to cephFilesystem object - err = controllerutil.SetControllerReference(&c.fs, d, c.scheme) + err = c.ownerInfo.SetControllerReference(d) if err != nil { - return errors.Wrapf(err, "failed to set owner reference for ceph filesystem %q secret", d.Name) + return errors.Wrapf(err, "failed to set owner reference for mds deployment %q", d.Name) } // Set the deployment hash as an annotation diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index 328ff9170..da47ab2eb 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -39,15 +39,16 @@ const ( mdsCacheMemoryLimitFactor = 0.5 ) -func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) (*apps.Deployment, error) { +func (c *Cluster) makeDeployment(mdsConfig *mdsConfig, namespace string) (*apps.Deployment, error) { mdsContainer := c.makeMdsDaemonContainer(mdsConfig) mdsContainer = config.ConfigureLivenessProbe(cephv1.KeyMds, mdsContainer, c.clusterSpec.HealthCheck) podSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Name: mdsConfig.ResourceName, - Labels: c.podLabels(mdsConfig, true), + Name: mdsConfig.ResourceName, + Namespace: namespace, + Labels: c.podLabels(mdsConfig, true), }, Spec: v1.PodSpec{ InitContainers: []v1.Container{ diff --git a/pkg/operator/ceph/file/mds/spec_test.go b/pkg/operator/ceph/file/mds/spec_test.go index f351aa0c5..675798a0f 100644 --- a/pkg/operator/ceph/file/mds/spec_test.go +++ b/pkg/operator/ceph/file/mds/spec_test.go @@ -19,9 +19,10 @@ package mds import ( "testing" - "github.com/rook/rook/pkg/client/clientset/versioned/scheme" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/ceph/test" + "github.com/rook/rook/pkg/operator/k8sutil" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" @@ -29,7 +30,6 @@ import ( cephclient "github.com/rook/rook/pkg/daemon/ceph/client" cephver "github.com/rook/rook/pkg/operator/ceph/version" - cephtest "github.com/rook/rook/pkg/operator/ceph/test" testop "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" apps "k8s.io/api/apps/v1" @@ -74,16 +74,15 @@ func testDeploymentObject(t *testing.T, network cephv1.NetworkSpec) (*apps.Deplo }, fs, &client.CephFilesystemDetails{ID: 15}, - metav1.OwnerReference{}, + &k8sutil.OwnerInfo{}, "/var/lib/rook/", - scheme.Scheme, ) mdsTestConfig := &mdsConfig{ DaemonID: "myfs-a", ResourceName: "rook-ceph-mds-myfs-a", DataPathMap: config.NewStatelessDaemonDataPathMap(config.MdsType, "myfs-a", "rook-ceph", "/var/lib/rook/"), } - return c.makeDeployment(mdsTestConfig) + return c.makeDeployment(mdsTestConfig, "ns") } func TestPodSpecs(t *testing.T) { @@ -94,10 +93,10 @@ func TestPodSpecs(t *testing.T) { assert.Equal(t, v1.RestartPolicyAlways, d.Spec.Template.Spec.RestartPolicy) // Deployment should have Ceph labels - cephtest.AssertLabelsContainCephRequirements(t, d.ObjectMeta.Labels, + test.AssertLabelsContainCephRequirements(t, d.ObjectMeta.Labels, config.MdsType, "myfs-a", "rook-ceph-mds", "ns") - podTemplate := cephtest.NewPodTemplateSpecTester(t, &d.Spec.Template) + podTemplate := test.NewPodTemplateSpecTester(t, &d.Spec.Template) podTemplate.RunFullSuite(config.MdsType, "myfs-a", "rook-ceph-mds", "ns", "ceph/ceph:testversion", "500", "250", "4337", "2169", /* resources */ "my-priority-class") diff --git a/pkg/operator/ceph/file/mirror/config.go b/pkg/operator/ceph/file/mirror/config.go index e522f0a7c..3439c809d 100644 --- a/pkg/operator/ceph/file/mirror/config.go +++ b/pkg/operator/ceph/file/mirror/config.go @@ -22,7 +22,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/rook/rook/pkg/operator/k8sutil" ) const ( @@ -43,7 +43,7 @@ const ( type daemonConfig struct { ResourceName string // the name rook gives to mirror resources in k8s metadata DataPathMap *config.DataPathMap // location to store data in container - ownerRef metav1.OwnerReference + ownerInfo *k8sutil.OwnerInfo } // PeerToken is the content of the peer token @@ -61,7 +61,7 @@ func (r *ReconcileFilesystemMirror) generateKeyring(clusterInfo *client.ClusterI "mds", "allow r", "osd", "allow rw tag cephfs metadata=*, allow r tag cephfs data=*", } - s := keyring.GetSecretStore(r.context, clusterInfo, &daemonConfig.ownerRef) + s := keyring.GetSecretStore(r.context, clusterInfo, daemonConfig.ownerInfo) key, err := s.GenerateKey(user, access) if err != nil { diff --git a/pkg/operator/ceph/file/mirror/mirror.go b/pkg/operator/ceph/file/mirror/mirror.go index 0b298ed71..a804e2e7e 100644 --- a/pkg/operator/ceph/file/mirror/mirror.go +++ b/pkg/operator/ceph/file/mirror/mirror.go @@ -26,7 +26,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/controller" - opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/k8sutil" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -50,17 +50,11 @@ func (r *ReconcileFilesystemMirror) start(filesystemMirror *cephv1.CephFilesyste return errors.Wrap(err, "error checking pod memory") } - // Create the controller owner ref - // It will be associated to all resources of the CephRBDMirror - ref, err := opcontroller.GetControllerObjectOwnerReference(filesystemMirror, r.scheme) - if err != nil || ref == nil { - return errors.Wrapf(err, "failed to get controller %q owner reference", filesystemMirror.Name) - } - + ownerInfo := k8sutil.NewOwnerInfo(filesystemMirror, r.scheme) daemonConf := &daemonConfig{ ResourceName: AppName, DataPathMap: config.NewDatalessDaemonDataPathMap(filesystemMirror.Namespace, r.cephClusterSpec.DataDirHostPath), - ownerRef: *ref, + ownerInfo: ownerInfo, } _, err = r.generateKeyring(r.clusterInfo, daemonConf) @@ -77,7 +71,7 @@ func (r *ReconcileFilesystemMirror) start(filesystemMirror *cephv1.CephFilesyste // Set owner ref to filesystemMirror object err = controllerutil.SetControllerReference(filesystemMirror, d, r.scheme) if err != nil { - return errors.Wrapf(err, "failed to set owner reference for ceph filesystem-mirror %q secret", d.Name) + return errors.Wrapf(err, "failed to set owner reference for ceph filesystem-mirror deployment %q", d.Name) } // Set the deployment hash as an annotation diff --git a/pkg/operator/ceph/file/mirror/spec.go b/pkg/operator/ceph/file/mirror/spec.go index 7da32d9fd..a73647994 100644 --- a/pkg/operator/ceph/file/mirror/spec.go +++ b/pkg/operator/ceph/file/mirror/spec.go @@ -31,8 +31,9 @@ import ( func (r *ReconcileFilesystemMirror) makeDeployment(daemonConfig *daemonConfig, fsMirror *cephv1.CephFilesystemMirror) (*apps.Deployment, error) { podSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Name: daemonConfig.ResourceName, - Labels: controller.CephDaemonAppLabels(AppName, fsMirror.Namespace, config.FilesystemMirrorType, userID, true), + Name: daemonConfig.ResourceName, + Namespace: fsMirror.Namespace, + Labels: controller.CephDaemonAppLabels(AppName, fsMirror.Namespace, config.FilesystemMirrorType, userID, true), }, Spec: v1.PodSpec{ InitContainers: []v1.Container{ diff --git a/pkg/operator/ceph/file/mirror/spec_test.go b/pkg/operator/ceph/file/mirror/spec_test.go index 765bd0803..5328d6be5 100644 --- a/pkg/operator/ceph/file/mirror/spec_test.go +++ b/pkg/operator/ceph/file/mirror/spec_test.go @@ -23,7 +23,7 @@ import ( "github.com/rook/rook/pkg/client/clientset/versioned/scheme" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/config" - cephtest "github.com/rook/rook/pkg/operator/ceph/test" + "github.com/rook/rook/pkg/operator/ceph/test" cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" @@ -89,10 +89,10 @@ func TestPodSpec(t *testing.T) { assert.Equal(t, 4, len(d.Spec.Template.Spec.Containers[0].VolumeMounts)) // Deployment should have Ceph labels - cephtest.AssertLabelsContainCephRequirements(t, d.ObjectMeta.Labels, + test.AssertLabelsContainCephRequirements(t, d.ObjectMeta.Labels, config.FilesystemMirrorType, userID, AppName, "ns") - podTemplate := cephtest.NewPodTemplateSpecTester(t, &d.Spec.Template) + podTemplate := test.NewPodTemplateSpecTester(t, &d.Spec.Template) podTemplate.RunFullSuite(config.FilesystemMirrorType, userID, AppName, "ns", "ceph/ceph:v16", "200", "100", "600", "300", /* resources */ "my-priority-class") diff --git a/pkg/operator/ceph/nfs/config.go b/pkg/operator/ceph/nfs/config.go index 769115a81..95b260392 100644 --- a/pkg/operator/ceph/nfs/config.go +++ b/pkg/operator/ceph/nfs/config.go @@ -25,6 +25,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config/keyring" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" cephver "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/rook/rook/pkg/operator/k8sutil" ) const ( @@ -85,7 +86,8 @@ func (r *ReconcileCephNFS) generateKeyring(n *cephv1.CephNFS, name string) error return errors.Wrapf(err, "failed to get controller %q owner reference", n.Name) } - s := keyring.GetSecretStore(r.context, r.clusterInfo, ref) + ownerInfo := k8sutil.NewOwnerInfo(n, r.scheme) + s := keyring.GetSecretStore(r.context, r.clusterInfo, ownerInfo) key, err := s.GenerateKey(user, caps) if err != nil { diff --git a/pkg/operator/ceph/nfs/nfs.go b/pkg/operator/ceph/nfs/nfs.go index 3078fc1e2..db67c34b5 100644 --- a/pkg/operator/ceph/nfs/nfs.go +++ b/pkg/operator/ceph/nfs/nfs.go @@ -87,7 +87,7 @@ func (r *ReconcileCephNFS) upCephNFS(n *cephv1.CephNFS) error { // Set owner ref to cephNFS object err = controllerutil.SetControllerReference(n, deployment, r.scheme) if err != nil { - return errors.Wrapf(err, "failed to set owner reference for ceph nfs %q secret", deployment.Name) + return errors.Wrapf(err, "failed to set owner reference for ceph nfs deployment %q", deployment.Name) } // Set the deployment hash as an annotation @@ -197,7 +197,7 @@ func (r *ReconcileCephNFS) createConfigMap(n *cephv1.CephNFS, name string) (stri // Set owner reference err := controllerutil.SetControllerReference(n, configMap, r.scheme) if err != nil { - return "", errors.Wrapf(err, "failed to set owner reference for ceph nfs %q config map", configMap.Name) + return "", errors.Wrapf(err, "failed to set owner reference for ceph ganesha configmap %q", configMap.Name) } if _, err := r.context.Clientset.CoreV1().ConfigMaps(n.Namespace).Create(ctx, configMap, metav1.CreateOptions{}); err != nil { diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go index 9555ff9b4..10b62627d 100644 --- a/pkg/operator/ceph/nfs/spec.go +++ b/pkg/operator/ceph/nfs/spec.go @@ -77,7 +77,7 @@ func (r *ReconcileCephNFS) createCephNFSService(nfs *cephv1.CephNFS, cfg daemonC // Set owner ref to the parent object err := controllerutil.SetControllerReference(nfs, s, r.scheme) if err != nil { - return errors.Wrap(err, "failed to set owner reference to ceph object store") + return errors.Wrapf(err, "failed to set owner reference to ceph nfs %q", s) } svc, err := r.context.Clientset.CoreV1().Services(nfs.Namespace).Create(ctx, s, metav1.CreateOptions{}) diff --git a/pkg/operator/ceph/object/config.go b/pkg/operator/ceph/object/config.go index db018980e..687be006c 100644 --- a/pkg/operator/ceph/object/config.go +++ b/pkg/operator/ceph/object/config.go @@ -80,7 +80,7 @@ func (c *clusterConfig) generateKeyring(rgwConfig *rgwConfig) (string, error) { user := generateCephXUser(rgwConfig.ResourceName) /* TODO: this says `osd allow rwx` while template says `osd allow *`; which is correct? */ access := []string{"osd", "allow rwx", "mon", "allow rw"} - s := keyring.GetSecretStore(c.context, c.clusterInfo, c.ownerRef) + s := keyring.GetSecretStore(c.context, c.clusterInfo, c.ownerInfo) key, err := s.GenerateKey(user, access) if err != nil { diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index f1e96bc4d..08bb3a858 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -34,6 +34,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/mon" opconfig "github.com/rook/rook/pkg/operator/ceph/config" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/exec" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -312,6 +313,7 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci } func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *cephv1.CephObjectStore, namespacedName types.NamespacedName, cluster cephv1.ClusterSpec) (reconcile.Result, error) { + ownerInfo := k8sutil.NewOwnerInfo(cephObjectStore, r.scheme) cfg := clusterConfig{ context: r.context, clusterInfo: r.clusterInfo, @@ -320,7 +322,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c clusterSpec: r.clusterSpec, DataPathMap: opconfig.NewStatelessDaemonDataPathMap(opconfig.RgwType, cephObjectStore.Name, cephObjectStore.Namespace, r.clusterSpec.DataDirHostPath), client: r.client, - scheme: r.scheme, + ownerInfo: ownerInfo, } objContext := NewContext(r.context, r.clusterInfo, cephObjectStore.Name) objContext.UID = string(cephObjectStore.UID) diff --git a/pkg/operator/ceph/object/mime.go b/pkg/operator/ceph/object/mime.go index e9f472fa0..e3fa79fbf 100644 --- a/pkg/operator/ceph/object/mime.go +++ b/pkg/operator/ceph/object/mime.go @@ -45,7 +45,7 @@ func mimeTypesMountPath() string { // store mime.types file in a config map func (c *clusterConfig) generateMimeTypes() error { - k := k8sutil.NewConfigMapKVStore(c.store.Namespace, c.context.Clientset, *c.ownerRef) + k := k8sutil.NewConfigMapKVStore(c.store.Namespace, c.context.Clientset, c.ownerInfo) if _, err := k.GetValue(c.mimeTypesConfigMapName(), mimeTypesFileName); err == nil || !kerrors.IsNotFound(err) { logger.Infof("config map for object pool %s already exists, not overwriting", c.store.Name) return nil diff --git a/pkg/operator/ceph/object/realm/controller.go b/pkg/operator/ceph/object/realm/controller.go index 3010c0f2a..3609ccef7 100644 --- a/pkg/operator/ceph/object/realm/controller.go +++ b/pkg/operator/ceph/object/realm/controller.go @@ -290,7 +290,7 @@ func (r *ReconcileObjectRealm) createRealmKeys(realm *cephv1.CephObjectRealm) (r } err = controllerutil.SetControllerReference(realm, secret, r.scheme) if err != nil { - return reconcile.Result{}, errors.Wrapf(err, "failed to set owner reference of secret %q", secret.Name) + return reconcile.Result{}, errors.Wrapf(err, "failed to set owner reference of rgw secret %q", secret.Name) } if _, err = r.context.Clientset.CoreV1().Secrets(realm.Namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { diff --git a/pkg/operator/ceph/object/rgw.go b/pkg/operator/ceph/object/rgw.go index 037037956..1249a8c8d 100644 --- a/pkg/operator/ceph/object/rgw.go +++ b/pkg/operator/ceph/object/rgw.go @@ -29,14 +29,11 @@ import ( cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/config" - opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/pool" "github.com/rook/rook/pkg/operator/k8sutil" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) type clusterConfig struct { @@ -45,10 +42,9 @@ type clusterConfig struct { store *cephv1.CephObjectStore rookVersion string clusterSpec *cephv1.ClusterSpec - ownerRef *metav1.OwnerReference + ownerInfo *k8sutil.OwnerInfo DataPathMap *config.DataPathMap client client.Client - scheme *runtime.Scheme } type rgwConfig struct { @@ -87,14 +83,6 @@ func (c *clusterConfig) startRGWPods(realmName, zoneGroupName, zoneName string) c.store.Spec.Gateway.Instances = 1 } - // Create the controller owner ref - // It will be associated to all resources of the CephObjectStore - ref, err := opcontroller.GetControllerObjectOwnerReference(c.store, c.scheme) - if err != nil || ref == nil { - return errors.Wrapf(err, "failed to get controller %q owner reference", c.store.Name) - } - c.ownerRef = ref - // start a new deployment and scale up desiredRgwInstances := int(c.store.Spec.Gateway.Instances) for i := 0; i < desiredRgwInstances; i++ { @@ -143,9 +131,9 @@ func (c *clusterConfig) startRGWPods(realmName, zoneGroupName, zoneName string) logger.Infof("object store %q deployment %q started", c.store.Name, deployment.Name) // Set owner ref to cephObjectStore object - err = controllerutil.SetControllerReference(c.store, deployment, c.scheme) + err = c.ownerInfo.SetControllerReference(deployment) if err != nil { - return errors.Wrapf(err, "failed to set owner reference for ceph object %q secret", deployment.Name) + return errors.Wrapf(err, "failed to set owner reference for rgw deployment %q", deployment.Name) } // Set the deployment hash as an annotation diff --git a/pkg/operator/ceph/object/rgw_test.go b/pkg/operator/ceph/object/rgw_test.go index 3eefb50f0..8c4f29d13 100644 --- a/pkg/operator/ceph/object/rgw_test.go +++ b/pkg/operator/ceph/object/rgw_test.go @@ -29,6 +29,7 @@ import ( cephclient "github.com/rook/rook/pkg/daemon/ceph/client" clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test" "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/k8sutil" testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" @@ -65,7 +66,8 @@ func TestStartRGW(t *testing.T) { r := &ReconcileCephObjectStore{client: cl, scheme: s} // start a basic cluster - c := &clusterConfig{context, info, store, version, &cephv1.ClusterSpec{}, &metav1.OwnerReference{}, data, r.client, s} + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + c := &clusterConfig{context, info, store, version, &cephv1.ClusterSpec{}, ownerInfo, data, r.client} err := c.startRGWPods(store.Name, store.Name, store.Name) assert.Nil(t, err) @@ -111,7 +113,8 @@ func TestCreateObjectStore(t *testing.T) { object := []runtime.Object{&cephv1.CephObjectStore{}} cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() r := &ReconcileCephObjectStore{client: cl, scheme: s} - c := &clusterConfig{context, info, store, "1.2.3.4", &cephv1.ClusterSpec{}, &metav1.OwnerReference{}, data, r.client, s} + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + c := &clusterConfig{context, info, store, "1.2.3.4", &cephv1.ClusterSpec{}, ownerInfo, data, r.client} err := c.createOrUpdateStore(store.Name, store.Name, store.Name) assert.Nil(t, err) } @@ -136,10 +139,9 @@ func TestGenerateSecretName(t *testing.T) { &cephv1.CephObjectStore{ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: "mycluster"}}, "v1.1.0", &cephv1.ClusterSpec{}, - &metav1.OwnerReference{}, + &k8sutil.OwnerInfo{}, &config.DataPathMap{}, - cl, - scheme.Scheme} + cl} secret := c.generateSecretName("a") assert.Equal(t, "rook-ceph-rgw-default-a-keyring", secret) } diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index e62faa528..0bf744e5f 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -33,7 +33,6 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) const ( @@ -369,9 +368,9 @@ func (c *clusterConfig) reconcileExternalEndpoint(cephObjectStore *cephv1.CephOb endpoint := c.generateEndpoint(cephObjectStore) // Set owner ref to the parent object - err := controllerutil.SetControllerReference(cephObjectStore, endpoint, c.scheme) + err := c.ownerInfo.SetControllerReference(endpoint) if err != nil { - return errors.Wrap(err, "failed to set owner reference to ceph object store endpoint") + return errors.Wrapf(err, "failed to set owner reference to ceph object store endpoint %q", endpoint.Name) } _, err = k8sutil.CreateOrUpdateEndpoint(c.context.Clientset, cephObjectStore.Namespace, endpoint) @@ -385,9 +384,9 @@ func (c *clusterConfig) reconcileExternalEndpoint(cephObjectStore *cephv1.CephOb func (c *clusterConfig) reconcileService(cephObjectStore *cephv1.CephObjectStore) (string, error) { service := c.generateService(cephObjectStore) // Set owner ref to the parent object - err := controllerutil.SetControllerReference(cephObjectStore, service, c.scheme) + err := c.ownerInfo.SetControllerReference(service) if err != nil { - return "", errors.Wrap(err, "failed to set owner reference to ceph object store service") + return "", errors.Wrapf(err, "failed to set owner reference to ceph object store service %q", service.Name) } svc, err := k8sutil.CreateOrUpdateService(c.context.Clientset, cephObjectStore.Namespace, service) diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index 08974c41c..fd46ea609 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -26,7 +26,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test" cephconfig "github.com/rook/rook/pkg/operator/ceph/config" - cephtest "github.com/rook/rook/pkg/operator/ceph/test" + "github.com/rook/rook/pkg/operator/ceph/test" cephver "github.com/rook/rook/pkg/operator/ceph/version" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" @@ -80,7 +80,7 @@ func TestPodSpecs(t *testing.T) { getLabels(c.store.Name, c.store.Namespace, false), s.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector.MatchLabels) - podTemplate := cephtest.NewPodTemplateSpecTester(t, &s) + podTemplate := test.NewPodTemplateSpecTester(t, &s) podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion", "200", "100", "1337", "500", /* resources */ "my-priority-class") @@ -125,7 +125,7 @@ func TestSSLPodSpec(t *testing.T) { s, err := c.makeRGWPodSpec(rgwConfig) assert.NoError(t, err) - podTemplate := cephtest.NewPodTemplateSpecTester(t, &s) + podTemplate := test.NewPodTemplateSpecTester(t, &s) podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion", "200", "100", "1337", "500", /* resources */ "my-priority-class") diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go index ade810fb6..e570a9cf0 100644 --- a/pkg/operator/ceph/object/user/controller.go +++ b/pkg/operator/ceph/object/user/controller.go @@ -379,7 +379,7 @@ func (r *ReconcileObjectStoreUser) reconcileCephUserSecret(cephObjectStoreUser * // Set owner ref to the object store user object err := controllerutil.SetControllerReference(cephObjectStoreUser, secret, r.scheme) if err != nil { - return reconcile.Result{}, errors.Wrapf(err, "failed to set owner reference for ceph object user %q secret", secret.Name) + return reconcile.Result{}, errors.Wrapf(err, "failed to set owner reference of ceph object user secret %q", secret.Name) } // Create Kubernetes Secret diff --git a/pkg/operator/ceph/operator.go b/pkg/operator/ceph/operator.go index 9d74b122b..92533d260 100644 --- a/pkg/operator/ceph/operator.go +++ b/pkg/operator/ceph/operator.go @@ -249,9 +249,10 @@ func (o *Operator) updateDrivers() error { ownerRef.BlockOwnerDeletion = &blockOwnerDeletion } + ownerInfo := k8sutil.NewOwnerInfoWithOwnerRef(ownerRef, o.operatorNamespace) // create an empty config map. config map will be filled with data // later when clusters have mons - err = csi.CreateCsiConfigMap(o.operatorNamespace, o.context.Clientset, ownerRef) + err = csi.CreateCsiConfigMap(o.operatorNamespace, o.context.Clientset, ownerInfo) if err != nil { return errors.Wrap(err, "failed creating csi config map") } @@ -260,7 +261,7 @@ func (o *Operator) updateDrivers() error { return errors.Wrap(err, "invalid csi params") } - go csi.ValidateAndConfigureDrivers(o.context, o.operatorNamespace, o.rookImage, o.securityAccount, serverVersion, ownerRef) + go csi.ValidateAndConfigureDrivers(o.context, o.operatorNamespace, o.rookImage, o.securityAccount, serverVersion, ownerInfo) return nil } diff --git a/pkg/operator/ceph/pool/config.go b/pkg/operator/ceph/pool/config.go index a359af6bb..65756e34b 100644 --- a/pkg/operator/ceph/pool/config.go +++ b/pkg/operator/ceph/pool/config.go @@ -55,7 +55,7 @@ func (r *ReconcileCephBlockPool) createBootstrapPeerSecret(cephBlockPool *cephv1 // set ownerref to the Secret err = controllerutil.SetControllerReference(cephBlockPool, s, r.scheme) if err != nil { - return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to set owner reference for rbd-mirror bootstrap peer %q secret", s.Name) + return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to set owner reference for rbd-mirror bootstrap peer secret %q", s.Name) } // Create Secret diff --git a/pkg/operator/ceph/test/ownerinfo/resources.go b/pkg/operator/ceph/test/ownerinfo/resources.go new file mode 100644 index 000000000..3d3b8b540 --- /dev/null +++ b/pkg/operator/ceph/test/ownerinfo/resources.go @@ -0,0 +1,39 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "testing" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func NewTestOwnerInfo(t *testing.T) *k8sutil.OwnerInfo { + cluster := &cephv1.CephCluster{} + scheme := runtime.NewScheme() + err := cephv1.AddToScheme(scheme) + assert.NoError(t, err) + return k8sutil.NewOwnerInfo(cluster, scheme) +} + +func NewTestOwnerInfoWithOwnerRef() *k8sutil.OwnerInfo { + return k8sutil.NewOwnerInfoWithOwnerRef(&metav1.OwnerReference{}, "") +} diff --git a/pkg/operator/k8sutil/cmdreporter/cmdreporter.go b/pkg/operator/k8sutil/cmdreporter/cmdreporter.go index 1e63d06f5..2053bee48 100644 --- a/pkg/operator/k8sutil/cmdreporter/cmdreporter.go +++ b/pkg/operator/k8sutil/cmdreporter/cmdreporter.go @@ -24,6 +24,7 @@ import ( "time" "github.com/coreos/pkg/capnslog" + "github.com/pkg/errors" "github.com/rook/rook/pkg/daemon/util" "github.com/rook/rook/pkg/operator/k8sutil" @@ -64,7 +65,7 @@ type CmdReporter struct { type cmdReporterCfg struct { clientset kubernetes.Interface - ownerRef *metav1.OwnerReference + ownerInfo *k8sutil.OwnerInfo appName string jobName string jobNamespace string @@ -88,14 +89,14 @@ type cmdReporterCfg struct { // then the command will run without the binaries being copied from the same Rook image. func New( clientset kubernetes.Interface, - ownerRef *metav1.OwnerReference, + ownerInfo *k8sutil.OwnerInfo, appName, jobName, jobNamespace string, cmd, args []string, rookImage, runImage string, ) (*CmdReporter, error) { cfg := &cmdReporterCfg{ clientset: clientset, - ownerRef: ownerRef, + ownerInfo: ownerInfo, appName: appName, jobName: jobName, jobNamespace: jobNamespace, @@ -107,8 +108,8 @@ func New( // Validate contents of config struct, not inputs to function to catch any developer errors // mis-assigning config items to the struct. - if cfg.clientset == nil || cfg.ownerRef == nil { - return nil, fmt.Errorf("clientset [%+v] and owner reference [%+v] must be specified", cfg.clientset, cfg.ownerRef) + if cfg.clientset == nil || cfg.ownerInfo == nil { + return nil, fmt.Errorf("clientset [%+v] and owner info [%+v] must be specified", cfg.clientset, cfg.ownerInfo) } if cfg.appName == "" || cfg.jobName == "" || cfg.jobNamespace == "" { return nil, fmt.Errorf("app name [%s], job name [%s], and job namespace [%s] must be specified", cfg.appName, cfg.jobName, cfg.jobNamespace) @@ -315,7 +316,10 @@ func (cr *cmdReporterCfg) initJobSpec() (*batch.Job, error) { }, } k8sutil.AddRookVersionLabelToJob(job) - k8sutil.SetOwnerRef(&job.ObjectMeta, cr.ownerRef) + err = cr.ownerInfo.SetControllerReference(job) + if err != nil { + return nil, errors.Wrapf(err, "failed to set owner reference to cmdreporter job %q", job.Name) + } return job, nil } diff --git a/pkg/operator/k8sutil/kvstore.go b/pkg/operator/k8sutil/kvstore.go index c28bb48d1..1b834b527 100644 --- a/pkg/operator/k8sutil/kvstore.go +++ b/pkg/operator/k8sutil/kvstore.go @@ -30,14 +30,14 @@ import ( type ConfigMapKVStore struct { namespace string clientset kubernetes.Interface - ownerRef metav1.OwnerReference + ownerInfo *OwnerInfo } -func NewConfigMapKVStore(namespace string, clientset kubernetes.Interface, ownerRef metav1.OwnerReference) *ConfigMapKVStore { +func NewConfigMapKVStore(namespace string, clientset kubernetes.Interface, ownerInfo *OwnerInfo) *ConfigMapKVStore { return &ConfigMapKVStore{ namespace: namespace, clientset: clientset, - ownerRef: ownerRef, + ownerInfo: ownerInfo, } } @@ -79,7 +79,10 @@ func (kv *ConfigMapKVStore) SetValueWithLabels(storeName, key, value string, lab if labels != nil { cm.Labels = labels } - SetOwnerRef(&cm.ObjectMeta, &kv.ownerRef) + err = kv.ownerInfo.SetControllerReference(cm) + if err != nil { + return err + } _, err = kv.clientset.CoreV1().ConfigMaps(kv.namespace).Create(ctx, cm, metav1.CreateOptions{}) return err diff --git a/pkg/operator/k8sutil/kvstore_test.go b/pkg/operator/k8sutil/kvstore_test.go index a915213fa..0af755842 100644 --- a/pkg/operator/k8sutil/kvstore_test.go +++ b/pkg/operator/k8sutil/kvstore_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -168,5 +168,6 @@ func newKVStore(stores ...*v1.ConfigMap) (*ConfigMapKVStore, string) { } clientset := fake.NewSimpleClientset(objects...) - return NewConfigMapKVStore(namespace, clientset, metav1.OwnerReference{}), storeName + ownerInfo := NewOwnerInfoWithOwnerRef(&metav1.OwnerReference{}, "") + return NewConfigMapKVStore(namespace, clientset, ownerInfo), storeName } diff --git a/pkg/operator/k8sutil/resources.go b/pkg/operator/k8sutil/resources.go index a578e1c98..54ec2ab53 100644 --- a/pkg/operator/k8sutil/resources.go +++ b/pkg/operator/k8sutil/resources.go @@ -20,12 +20,103 @@ package k8sutil // MergeResourceRequirements merges two resource requirements together (first overrides second values) import ( "encoding/json" + "fmt" "github.com/ghodss/yaml" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) +// OwnerInfo is to set owner references. Only one of owner and ownerRef must be valid. +type OwnerInfo struct { + owner metav1.Object + scheme *runtime.Scheme + ownerRef *metav1.OwnerReference + ownerRefNamespace string +} + +// NewOwnerInfo create a new ownerInfo to set ownerReference by controllerutil +func NewOwnerInfo(owner metav1.Object, scheme *runtime.Scheme) *OwnerInfo { + return &OwnerInfo{owner: owner, scheme: scheme} +} + +// NewOwnerInfoWithOwnerRef create a new ownerInfo to set ownerReference by rook itself +func NewOwnerInfoWithOwnerRef(ownerRef *metav1.OwnerReference, namespace string) *OwnerInfo { + return &OwnerInfo{ownerRef: ownerRef, ownerRefNamespace: namespace} +} + +func (info *OwnerInfo) validateOwner(object metav1.Object) error { + if info.ownerRefNamespace == "" { + return nil + } + objectNamespace := object.GetNamespace() + if objectNamespace == "" { + return fmt.Errorf("cluster-scoped resource %q must not have a namespaced resource %q in namespace %q", + object.GetName(), info.ownerRef.Name, info.ownerRefNamespace) + + } + if info.ownerRefNamespace != objectNamespace { + return fmt.Errorf("cross-namespaced owner references are disallowed. resource %q is in namespace %q, owner %q is in %q", + object.GetName(), object.GetNamespace(), info.ownerRef.Name, info.ownerRefNamespace) + } + return nil +} + +func (info *OwnerInfo) validateController(object metav1.Object) error { + existingController := metav1.GetControllerOf(object) + if existingController != nil && existingController.UID != info.ownerRef.UID { + return fmt.Errorf("%q already set its controller %q", object.GetName(), info.ownerRef.Name) + } + return nil +} + +// SetOwnerReference set the owner reference of object +func (info *OwnerInfo) SetOwnerReference(object metav1.Object) error { + if info.owner != nil { + return controllerutil.SetOwnerReference(info.owner, object, info.scheme) + } + err := info.validateOwner(object) + if err != nil { + return err + } + SetOwnerRef(object, info.ownerRef) + return nil +} + +// SetControllerReference set the controller reference of object +func (info *OwnerInfo) SetControllerReference(object metav1.Object) error { + if info.owner != nil { + return controllerutil.SetControllerReference(info.owner, object, info.scheme) + } + err := info.validateOwner(object) + if err != nil { + return err + } + err = info.validateController(object) + if err != nil { + return err + } + blockOwnerDeletion := true + controller := true + info.ownerRef.BlockOwnerDeletion = &blockOwnerDeletion + info.ownerRef.Controller = &controller + SetOwnerRef(object, info.ownerRef) + return nil +} + +// GetUID gets the UID of the owner +func (info *OwnerInfo) GetUID() types.UID { + return info.owner.GetUID() +} + +// GetUID gets the UID of the owner +func (info *OwnerInfo) GetOwnerRef() *metav1.OwnerReference { + return info.ownerRef +} + func MergeResourceRequirements(first, second v1.ResourceRequirements) v1.ResourceRequirements { // if the first has a value not set check if second has and set it in first if _, ok := first.Limits[v1.ResourceCPU]; !ok { @@ -63,14 +154,14 @@ func MergeResourceRequirements(first, second v1.ResourceRequirements) v1.Resourc return first } -func SetOwnerRef(object *metav1.ObjectMeta, ownerRef *metav1.OwnerReference) { +func SetOwnerRef(object metav1.Object, ownerRef *metav1.OwnerReference) { if ownerRef == nil { return } SetOwnerRefs(object, []metav1.OwnerReference{*ownerRef}) } -func SetOwnerRefsWithoutBlockOwner(object *metav1.ObjectMeta, ownerRefs []metav1.OwnerReference) { +func SetOwnerRefsWithoutBlockOwner(object metav1.Object, ownerRefs []metav1.OwnerReference) { if ownerRefs == nil { return } @@ -89,7 +180,7 @@ func SetOwnerRefsWithoutBlockOwner(object *metav1.ObjectMeta, ownerRefs []metav1 SetOwnerRefs(object, newOwners) } -func SetOwnerRefs(object *metav1.ObjectMeta, ownerRefs []metav1.OwnerReference) { +func SetOwnerRefs(object metav1.Object, ownerRefs []metav1.OwnerReference) { object.SetOwnerReferences(ownerRefs) } diff --git a/pkg/operator/k8sutil/resources_test.go b/pkg/operator/k8sutil/resources_test.go index b0f114def..1f4608f01 100644 --- a/pkg/operator/k8sutil/resources_test.go +++ b/pkg/operator/k8sutil/resources_test.go @@ -120,3 +120,43 @@ func TestYamlToContainerResource(t *testing.T) { assert.Len(t, res, 0) assert.Error(t, err) } + +func TestValidateOwner(t *testing.T) { + // global-scoped owner + ownerRef := &metav1.OwnerReference{} + ownerInfo := NewOwnerInfoWithOwnerRef(ownerRef, "") + object := &v1.ConfigMap{} + err := ownerInfo.validateOwner(object) + assert.NoError(t, err) + + // namespaced owner + ownerInfo = NewOwnerInfoWithOwnerRef(ownerRef, "test-ns") + object = &v1.ConfigMap{} + err = ownerInfo.validateOwner(object) + assert.Error(t, err) + object = &v1.ConfigMap{} + object.Namespace = "test-ns" + err = ownerInfo.validateOwner(object) + assert.NoError(t, err) + object.Namespace = "different-ns" + err = ownerInfo.validateOwner(object) + assert.Error(t, err) +} + +func TestValidateController(t *testing.T) { + controllerRef := &metav1.OwnerReference{UID: "test-id"} + ownerInfo := NewOwnerInfoWithOwnerRef(controllerRef, "") + object := &v1.ConfigMap{} + err := ownerInfo.validateController(object) + assert.NoError(t, err) + err = ownerInfo.SetControllerReference(object) + assert.NoError(t, err) + err = ownerInfo.validateController(object) + assert.NoError(t, err) + err = ownerInfo.SetControllerReference(object) + assert.NoError(t, err) + newControllerRef := &metav1.OwnerReference{UID: "different-id"} + newOwnerInfo := NewOwnerInfoWithOwnerRef(newControllerRef, "") + err = newOwnerInfo.validateController(object) + assert.Error(t, err) +} diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 9dabffe74..cbd786029 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -232,7 +232,8 @@ osd_pool_default_size = 1 `} customCM := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: "rook-config-override", + Name: "rook-config-override", + Namespace: namespace, }, Data: customSettings, }