diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 9bc324704..6fdc6c8fb 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -9,6 +9,7 @@ ### Cluster CRD - Removed the `versionTag` property. The container version to launch in all pods will be the same as the version of the operator container. +- Added the `cluster.rook.io` finalizer. When a cluster is deleted, the operator will cleanup resources and remove the finalizer, which then allows K8s to delete the CRD. ### Operator - Removed the `ROOK_REPO_PREFIX` env var. All containers will be launched with the same image as the operator diff --git a/cmd/rook/osd.go b/cmd/rook/osd.go index af1baab24..82d806615 100644 --- a/cmd/rook/osd.go +++ b/cmd/rook/osd.go @@ -23,6 +23,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/daemon/ceph/mon" "github.com/rook/rook/pkg/daemon/ceph/osd" + "github.com/rook/rook/pkg/operator/cluster" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/flags" "github.com/spf13/cobra" @@ -34,12 +35,13 @@ var osdCmd = &cobra.Command{ Hidden: true, } var ( - osdCluster mon.ClusterInfo osdDataDeviceFilter string + ownerRefID string ) func addOSDFlags(command *cobra.Command) { command.Flags().StringVar(&cfg.devices, "data-devices", "", "comma separated list of devices to use for storage") + command.Flags().StringVar(&ownerRefID, "cluster-id", "", "the UID of the cluster CRD that owns this cluster") command.Flags().StringVar(&osdDataDeviceFilter, "data-device-filter", "", "a regex filter for the device names to use, or \"all\"") command.Flags().StringVar(&cfg.directories, "data-directories", "", "comma separated list of directory paths to use for storage") command.Flags().StringVar(&cfg.metadataDevice, "metadata-device", "", "device to use for metadata (e.g. a high performance SSD/NVMe device)") @@ -64,7 +66,7 @@ func init() { } func startOSD(cmd *cobra.Command, args []string) error { - required := []string{"cluster-name", "mon-endpoints", "mon-secret", "admin-secret", "node-name", "public-ipv4", "private-ipv4"} + required := []string{"cluster-name", "cluster-id", "mon-endpoints", "mon-secret", "admin-secret", "node-name", "public-ipv4", "private-ipv4"} if err := flags.VerifyRequiredFlags(osdCmd, required); err != nil { return err } @@ -95,8 +97,6 @@ func startOSD(cmd *cobra.Command, args []string) error { context.Clientset = clientset context.RookClientset = rookClientset - kv := k8sutil.NewConfigMapKVStore(clusterInfo.Name, clientset) - locArgs, err := client.FormatLocation(cfg.location, cfg.nodeName) if err != nil { terminateFatal(fmt.Errorf("invalid location. %+v\n", err)) @@ -105,6 +105,8 @@ func startOSD(cmd *cobra.Command, args []string) error { forceFormat := false clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints) + ownerRef := cluster.ClusterOwnerRef(clusterInfo.Name, ownerRefID) + kv := k8sutil.NewConfigMapKVStore(clusterInfo.Name, clientset, ownerRef) agent := osd.NewAgent(context, dataDevices, usingDeviceFilter, cfg.metadataDevice, cfg.directories, forceFormat, crushLocation, cfg.storeConfig, &clusterInfo, cfg.nodeName, kv) diff --git a/pkg/daemon/agent/flexvolume/manager/ceph/manager_test.go b/pkg/daemon/agent/flexvolume/manager/ceph/manager_test.go index 532b8ad78..a3ef485ec 100644 --- a/pkg/daemon/agent/flexvolume/manager/ceph/manager_test.go +++ b/pkg/daemon/agent/flexvolume/manager/ceph/manager_test.go @@ -24,14 +24,14 @@ import ( "testing" "time" - "k8s.io/api/core/v1" - - "github.com/stretchr/testify/assert" - "github.com/rook/rook/pkg/clusterd" cephtest "github.com/rook/rook/pkg/daemon/ceph/test" + "github.com/rook/rook/pkg/operator/cluster/ceph/mon" "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type fakeDevicePathFinder struct { @@ -150,6 +150,8 @@ func TestAttach(t *testing.T) { called: 0, }, } + mon.CreateOrLoadClusterInfo(context, clusterName, &metav1.OwnerReference{}) + devicePath, err := vm.Attach("image1", "testpool", clusterName) assert.Equal(t, "/dev/rbd3", devicePath) assert.Nil(t, err) @@ -215,6 +217,7 @@ func TestDetach(t *testing.T) { called: 0, }, } + mon.CreateOrLoadClusterInfo(context, clusterName, &metav1.OwnerReference{}) err := vm.Detach("image1", "testpool", clusterName, false) assert.Nil(t, err) } diff --git a/pkg/daemon/api/object.go b/pkg/daemon/api/object.go index bce71f150..1181d1551 100644 --- a/pkg/daemon/api/object.go +++ b/pkg/daemon/api/object.go @@ -383,7 +383,7 @@ func enableObjectStore(c *Config, config model.ObjectStore) error { } store := oprgw.ModelToSpec(config, c.namespace) - err := oprgw.UpdateStore(c.context, *store, c.rookImage, c.hostNetwork) + err := oprgw.UpdateStore(c.context, *store, c.rookImage, c.hostNetwork, []metav1.OwnerReference{}) if err != nil { return fmt.Errorf("failed to start rgw. %+v", err) } diff --git a/pkg/daemon/ceph/mds/filesystem.go b/pkg/daemon/ceph/mds/filesystem.go index e66bc7c88..96281a82e 100644 --- a/pkg/daemon/ceph/mds/filesystem.go +++ b/pkg/daemon/ceph/mds/filesystem.go @@ -36,7 +36,7 @@ const ( appName = "cephfs" ) -// A castle file system (an instance of CephFS) +// A file system (an instance of CephFS) type Filesystem struct { Name string metadataPool *model.Pool diff --git a/pkg/daemon/ceph/osd/agent_test.go b/pkg/daemon/ceph/osd/agent_test.go index 4d40228e8..acdc6b4d2 100644 --- a/pkg/daemon/ceph/osd/agent_test.go +++ b/pkg/daemon/ceph/osd/agent_test.go @@ -35,6 +35,7 @@ import ( exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestOSDAgentWithDevicesFilestore(t *testing.T) { @@ -554,5 +555,5 @@ func mockDistributedPartitionScheme(t *testing.T, osdID int, metadataDevice, dev func mockKVStore() *k8sutil.ConfigMapKVStore { clientset := testop.New(1) - return k8sutil.NewConfigMapKVStore("myns", clientset) + return k8sutil.NewConfigMapKVStore("myns", clientset, metav1.OwnerReference{}) } diff --git a/pkg/operator/agent/agent.go b/pkg/operator/agent/agent.go index ee75d84ce..9a6f4f902 100644 --- a/pkg/operator/agent/agent.go +++ b/pkg/operator/agent/agent.go @@ -77,7 +77,7 @@ func New(clientset kubernetes.Interface) *Agent { // Start the agent func (a *Agent) Start(namespace, agentImage string) error { - err := k8sutil.MakeClusterRole(a.clientset, namespace, agentDaemonsetName, clusterAccessRules) + err := k8sutil.MakeClusterRole(a.clientset, namespace, agentDaemonsetName, clusterAccessRules, nil) if err != nil { return fmt.Errorf("failed to init RBAC for rook-agents. %+v", err) } diff --git a/pkg/operator/cluster/api/api.go b/pkg/operator/cluster/api/api.go index 7577112c8..d093a5951 100644 --- a/pkg/operator/cluster/api/api.go +++ b/pkg/operator/cluster/api/api.go @@ -77,10 +77,12 @@ type Cluster struct { Replicas int32 HostNetwork bool resources v1.ResourceRequirements + ownerRef metav1.OwnerReference } // New creates an instance -func New(context *clusterd.Context, namespace, version string, placement rookalpha.Placement, hostNetwork bool, resources v1.ResourceRequirements) *Cluster { +func New(context *clusterd.Context, namespace, version string, placement rookalpha.Placement, hostNetwork bool, + resources v1.ResourceRequirements, ownerRef metav1.OwnerReference) *Cluster { return &Cluster{ context: context, Namespace: namespace, @@ -89,6 +91,7 @@ func New(context *clusterd.Context, namespace, version string, placement rookalp Replicas: 1, HostNetwork: hostNetwork, resources: resources, + ownerRef: ownerRef, } } @@ -125,13 +128,10 @@ func (c *Cluster) Start() error { // make a cluster role func (c *Cluster) makeRole() error { - return k8sutil.MakeRole(c.context.Clientset, c.Namespace, deploymentName, clusterAccessRules) + return k8sutil.MakeRole(c.context.Clientset, c.Namespace, deploymentName, clusterAccessRules, c.ownerRef) } func (c *Cluster) makeDeployment() *extensions.Deployment { - deployment := &extensions.Deployment{} - deployment.Name = deploymentName - deployment.Namespace = c.Namespace podSpec := v1.PodSpec{ ServiceAccountName: deploymentName, @@ -156,9 +156,14 @@ func (c *Cluster) makeDeployment() *extensions.Deployment { Spec: podSpec, } - deployment.Spec = extensions.DeploymentSpec{Template: podTemplateSpec, Replicas: &c.Replicas} - - return deployment + return &extensions.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: deploymentName, + Namespace: c.Namespace, + OwnerReferences: []metav1.OwnerReference{c.ownerRef}, + }, + Spec: extensions.DeploymentSpec{Template: podTemplateSpec, Replicas: &c.Replicas}, + } } func (c *Cluster) apiContainer() v1.Container { @@ -192,9 +197,10 @@ func (c *Cluster) startService() error { labels := c.getLabels() s := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: deploymentName, - Namespace: c.Namespace, - Labels: labels, + Name: deploymentName, + Namespace: c.Namespace, + Labels: labels, + OwnerReferences: []metav1.OwnerReference{c.ownerRef}, }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ diff --git a/pkg/operator/cluster/api/api_test.go b/pkg/operator/cluster/api/api_test.go index 8cc0e34f2..6e6ebb0ce 100644 --- a/pkg/operator/cluster/api/api_test.go +++ b/pkg/operator/cluster/api/api_test.go @@ -31,7 +31,7 @@ import ( func TestStartAPI(t *testing.T) { clientset := testop.New(3) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "myversion", rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(&clusterd.Context{Clientset: clientset}, "ns", "myversion", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) // start a basic cluster err := c.Start() @@ -59,7 +59,7 @@ func validateStart(t *testing.T, c *Cluster) { func TestPodSpecs(t *testing.T) { clientset := testop.New(1) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "rook/rook:myversion", rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(&clusterd.Context{Clientset: clientset}, "ns", "rook/rook:myversion", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) d := c.makeDeployment() assert.NotNil(t, d) @@ -99,7 +99,7 @@ func TestPodSpecs(t *testing.T) { func TestClusterRole(t *testing.T) { clientset := testop.New(1) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "myversion", rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(&clusterd.Context{Clientset: clientset}, "ns", "myversion", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) // the role is create err := c.makeRole() diff --git a/pkg/operator/cluster/ceph/mgr/mgr.go b/pkg/operator/cluster/ceph/mgr/mgr.go index 61a751622..cf9b4c9b0 100644 --- a/pkg/operator/cluster/ceph/mgr/mgr.go +++ b/pkg/operator/cluster/ceph/mgr/mgr.go @@ -51,10 +51,12 @@ type Cluster struct { dataDir string HostNetwork bool resources v1.ResourceRequirements + ownerRef metav1.OwnerReference } // New creates an instance of the mgr -func New(context *clusterd.Context, namespace, version string, placement rookalpha.Placement, hostNetwork bool, resources v1.ResourceRequirements) *Cluster { +func New(context *clusterd.Context, namespace, version string, placement rookalpha.Placement, hostNetwork bool, + resources v1.ResourceRequirements, ownerRef metav1.OwnerReference) *Cluster { return &Cluster{ context: context, Namespace: namespace, @@ -64,6 +66,7 @@ func New(context *clusterd.Context, namespace, version string, placement rookalp dataDir: k8sutil.DataDir, HostNetwork: hostNetwork, resources: resources, + ownerRef: ownerRef, } } @@ -130,9 +133,6 @@ func (c *Cluster) makeService(name string) *v1.Service { } func (c *Cluster) makeDeployment(name string) *extensions.Deployment { - deployment := &extensions.Deployment{} - deployment.Name = name - deployment.Namespace = c.Namespace podSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ @@ -156,8 +156,14 @@ func (c *Cluster) makeDeployment(name string) *extensions.Deployment { c.placement.ApplyToPodSpec(&podSpec.Spec) replicas := int32(1) - deployment.Spec = extensions.DeploymentSpec{Template: podSpec, Replicas: &replicas} - return deployment + return &extensions.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: c.Namespace, + OwnerReferences: []metav1.OwnerReference{c.ownerRef}, + }, + Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &replicas}, + } } func (c *Cluster) mgrContainer(name string) v1.Container { @@ -228,7 +234,11 @@ func (c *Cluster) createKeyring(clusterName, name string) error { keyringName: keyring, } secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: c.Namespace}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: c.Namespace, + OwnerReferences: []metav1.OwnerReference{c.ownerRef}, + }, StringData: secrets, Type: k8sutil.RookType, } diff --git a/pkg/operator/cluster/ceph/mgr/mgr_test.go b/pkg/operator/cluster/ceph/mgr/mgr_test.go index d7b79d86b..555b984d5 100644 --- a/pkg/operator/cluster/ceph/mgr/mgr_test.go +++ b/pkg/operator/cluster/ceph/mgr/mgr_test.go @@ -44,7 +44,7 @@ func TestStartMGR(t *testing.T) { Executor: executor, ConfigDir: configDir, Clientset: testop.New(3)} - c := New(context, "ns", "myversion", rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(context, "ns", "myversion", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) defer os.RemoveAll(c.dataDir) // start a basic service @@ -76,7 +76,7 @@ func TestPodSpec(t *testing.T) { Requests: v1.ResourceList{ v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI), }, - }) + }, metav1.OwnerReference{}) d := c.makeDeployment("mgr1") assert.NotNil(t, d) @@ -103,7 +103,7 @@ func TestPodSpec(t *testing.T) { } func TestServiceSpec(t *testing.T) { - c := New(nil, "ns", "myversion", rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(nil, "ns", "myversion", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) s := c.makeService("rook-mgr") assert.NotNil(t, s) @@ -112,7 +112,7 @@ func TestServiceSpec(t *testing.T) { } func TestHostNetwork(t *testing.T) { - c := New(nil, "ns", "myversion", rookalpha.Placement{}, true, v1.ResourceRequirements{}) + c := New(nil, "ns", "myversion", rookalpha.Placement{}, true, v1.ResourceRequirements{}, metav1.OwnerReference{}) d := c.makeDeployment("mgr1") assert.NotNil(t, d) diff --git a/pkg/operator/cluster/ceph/mon/health_test.go b/pkg/operator/cluster/ceph/mon/health_test.go index 190998262..091330d1c 100644 --- a/pkg/operator/cluster/ceph/mon/health_test.go +++ b/pkg/operator/cluster/ceph/mon/health_test.go @@ -51,7 +51,7 @@ func TestCheckHealth(t *testing.T) { ConfigDir: configDir, Executor: executor, } - c := New(context, "ns", "", "myversion", 3, rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(context, "ns", "", "myversion", 3, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(1) c.waitForStart = false defer os.RemoveAll(c.context.ConfigDir) @@ -88,7 +88,7 @@ func TestCheckHealthNotFound(t *testing.T) { ConfigDir: configDir, Executor: executor, } - c := New(context, "ns", "", "myversion", 2, rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(context, "ns", "", "myversion", 2, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(2) c.waitForStart = false defer os.RemoveAll(c.context.ConfigDir) @@ -161,7 +161,7 @@ func TestCheckHealthTwoMonsOneNode(t *testing.T) { ConfigDir: configDir, Executor: executor, } - c := New(context, "ns", "", "myversion", 2, rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(context, "ns", "", "myversion", 2, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(2) c.waitForStart = false defer os.RemoveAll(c.context.ConfigDir) @@ -266,7 +266,7 @@ func TestCheckMonsValid(t *testing.T) { ConfigDir: configDir, Executor: executor, } - c := New(context, "ns", "", "myversion", 3, rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(context, "ns", "", "myversion", 3, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(1) c.waitForStart = false defer os.RemoveAll(c.context.ConfigDir) diff --git a/pkg/operator/cluster/ceph/mon/mon.go b/pkg/operator/cluster/ceph/mon/mon.go index ddaf3589a..23aa7e7c4 100644 --- a/pkg/operator/cluster/ceph/mon/mon.go +++ b/pkg/operator/cluster/ceph/mon/mon.go @@ -78,6 +78,7 @@ type Cluster struct { HostNetwork bool mapping *Mapping resources v1.ResourceRequirements + ownerRef metav1.OwnerReference } // monConfig for a single monitor @@ -100,7 +101,8 @@ type NodeInfo struct { } // New creates an instance of a mon cluster -func New(context *clusterd.Context, namespace, dataDirHostPath, version string, size int, placement rookalpha.Placement, hostNetwork bool, resources v1.ResourceRequirements) *Cluster { +func New(context *clusterd.Context, namespace, dataDirHostPath, version string, size int, placement rookalpha.Placement, hostNetwork bool, + resources v1.ResourceRequirements, ownerRef metav1.OwnerReference) *Cluster { return &Cluster{ context: context, placement: placement, @@ -119,6 +121,7 @@ func New(context *clusterd.Context, namespace, dataDirHostPath, version string, Port: map[string]int32{}, }, resources: resources, + ownerRef: ownerRef, } } @@ -184,7 +187,7 @@ func (c *Cluster) initClusterInfo() error { var err error // get the cluster info from secret - c.clusterInfo, c.maxMonID, c.mapping, err = LoadClusterInfo(c.context, c.Namespace) + c.clusterInfo, c.maxMonID, c.mapping, err = CreateOrLoadClusterInfo(c.context, c.Namespace, &c.ownerRef) if err != nil { return fmt.Errorf("failed to get cluster info. %+v", err) } @@ -240,8 +243,9 @@ func (c *Cluster) createService(mon *monConfig) (string, error) { labels := c.getLabels(mon.Name) s := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: mon.Name, - Labels: labels, + Name: mon.Name, + Labels: labels, + OwnerReferences: []metav1.OwnerReference{c.ownerRef}, }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ @@ -370,9 +374,10 @@ func (c *Cluster) waitForMonsToJoin(mons []*monConfig) error { func (c *Cluster) saveMonConfig() error { configMap := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: EndpointConfigMapName, - Namespace: c.Namespace, - Annotations: map[string]string{}, + Name: EndpointConfigMapName, + Namespace: c.Namespace, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{c.ownerRef}, }, } diff --git a/pkg/operator/cluster/ceph/mon/mon_test.go b/pkg/operator/cluster/ceph/mon/mon_test.go index ffd632073..18292d8bc 100644 --- a/pkg/operator/cluster/ceph/mon/mon_test.go +++ b/pkg/operator/cluster/ceph/mon/mon_test.go @@ -75,6 +75,7 @@ func newCluster(context *clusterd.Context, namespace string, hostNetwork bool, r Port: map[string]int32{}, }, resources: resources, + ownerRef: metav1.OwnerReference{}, } } @@ -217,7 +218,7 @@ func TestSaveMonEndpoints(t *testing.T) { clientset := test.New(1) configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", "", "myversion", 3, rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", "", "myversion", 3, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(1) // create the initial config map @@ -288,7 +289,7 @@ func TestMonID(t *testing.T) { func TestAvailableMonNodes(t *testing.T) { clientset := test.New(1) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", 3, rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", 3, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(0) nodes, err := c.getMonNodes() assert.Nil(t, err) @@ -305,7 +306,7 @@ func TestAvailableMonNodes(t *testing.T) { func TestAvailableNodesInUse(t *testing.T) { clientset := test.New(3) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", 3, rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", 3, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(0) // all three nodes are available by default @@ -336,7 +337,7 @@ func TestAvailableNodesInUse(t *testing.T) { func TestTaintedNodes(t *testing.T) { clientset := test.New(3) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", 3, rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", 3, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(0) nodes, err := c.getMonNodes() @@ -369,7 +370,7 @@ func TestTaintedNodes(t *testing.T) { func TestNodeAffinity(t *testing.T) { clientset := test.New(3) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", 3, rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", 3, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(0) nodes, err := c.getMonNodes() @@ -409,7 +410,7 @@ func TestNodeAffinity(t *testing.T) { func TestHostNetwork(t *testing.T) { clientset := test.New(3) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", 3, rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", 3, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(0) c.HostNetwork = true @@ -439,7 +440,7 @@ func TestGetNodeInfoFromNode(t *testing.T) { }, } - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", 3, rookalpha.Placement{}, true, v1.ResourceRequirements{}) + c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", 3, rookalpha.Placement{}, true, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(0) var info *NodeInfo @@ -468,7 +469,7 @@ func TestHostNetworkPortIncrease(t *testing.T) { c := New(&clusterd.Context{ Clientset: clientset, ConfigDir: configDir, - }, "ns", "", "myversion", 3, rookalpha.Placement{}, true, v1.ResourceRequirements{}) + }, "ns", "", "myversion", 3, rookalpha.Placement{}, true, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(0) mons := []*monConfig{ diff --git a/pkg/operator/cluster/ceph/mon/spec.go b/pkg/operator/cluster/ceph/mon/spec.go index 17ac64773..86f27fd85 100644 --- a/pkg/operator/cluster/ceph/mon/spec.go +++ b/pkg/operator/cluster/ceph/mon/spec.go @@ -64,9 +64,13 @@ func (c *Cluster) getLabels(name string) map[string]string { } func (c *Cluster) makeReplicaSet(config *monConfig, nodeName string) *extensions.ReplicaSet { - rs := &extensions.ReplicaSet{} - rs.Name = config.Name - rs.Namespace = c.Namespace + rs := &extensions.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: config.Name, + Namespace: c.Namespace, + OwnerReferences: []metav1.OwnerReference{c.ownerRef}, + }, + } pod := c.makeMonPod(config, nodeName) replicaCount := int32(1) diff --git a/pkg/operator/cluster/ceph/mon/spec_test.go b/pkg/operator/cluster/ceph/mon/spec_test.go index 77e61337c..30f16a8dd 100644 --- a/pkg/operator/cluster/ceph/mon/spec_test.go +++ b/pkg/operator/cluster/ceph/mon/spec_test.go @@ -23,10 +23,10 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" testop "github.com/rook/rook/pkg/operator/test" - "github.com/stretchr/testify/assert" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestPodSpecs(t *testing.T) { @@ -43,7 +43,7 @@ func testPodSpec(t *testing.T, dataDir string) { Requests: v1.ResourceList{ v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI), }, - }) + }, metav1.OwnerReference{}) c.clusterInfo = testop.CreateConfigDir(0) config := &monConfig{Name: "mon0", Port: 6790} diff --git a/pkg/operator/cluster/ceph/mon/util.go b/pkg/operator/cluster/ceph/mon/util.go index 2eba705b6..888899119 100644 --- a/pkg/operator/cluster/ceph/mon/util.go +++ b/pkg/operator/cluster/ceph/mon/util.go @@ -44,6 +44,11 @@ import ( // LoadClusterInfo constructs or loads a clusterinfo and returns it along with the maxMonID func LoadClusterInfo(context *clusterd.Context, namespace string) (*mon.ClusterInfo, int, *Mapping, error) { + return CreateOrLoadClusterInfo(context, namespace, nil) +} + +// CreateOrLoadClusterInfo constructs or loads a clusterinfo and returns it along with the maxMonID +func CreateOrLoadClusterInfo(context *clusterd.Context, namespace string, ownerRef *metav1.OwnerReference) (*mon.ClusterInfo, int, *Mapping, error) { var clusterInfo *mon.ClusterInfo maxMonID := -1 @@ -57,13 +62,16 @@ func LoadClusterInfo(context *clusterd.Context, namespace string) (*mon.ClusterI if !errors.IsNotFound(err) { return nil, maxMonID, monMapping, fmt.Errorf("failed to get mon secrets. %+v", err) } + if ownerRef == nil { + return nil, maxMonID, monMapping, fmt.Errorf("not expected to create new cluster info and did not find existing secret") + } clusterInfo, err = createNamedClusterInfo(context, namespace) if err != nil { return nil, maxMonID, monMapping, fmt.Errorf("failed to create mon secrets. %+v", err) } - err = createClusterAccessSecret(context.Clientset, namespace, clusterInfo) + err = createClusterAccessSecret(context.Clientset, namespace, clusterInfo, ownerRef) if err != nil { return nil, maxMonID, monMapping, err } @@ -157,7 +165,7 @@ func getMonID(name string) (int, error) { return id, nil } -func createClusterAccessSecret(clientset kubernetes.Interface, namespace string, clusterInfo *mon.ClusterInfo) error { +func createClusterAccessSecret(clientset kubernetes.Interface, namespace string, clusterInfo *mon.ClusterInfo, ownerRef *metav1.OwnerReference) error { logger.Infof("creating mon secrets for a new cluster") var err error @@ -169,10 +177,16 @@ func createClusterAccessSecret(clientset kubernetes.Interface, namespace string, adminSecretName: clusterInfo.AdminSecret, } secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: appName, Namespace: namespace}, + ObjectMeta: metav1.ObjectMeta{ + Name: appName, + Namespace: namespace, + }, StringData: secrets, Type: k8sutil.RookType, } + if ownerRef != nil { + secret.OwnerReferences = []metav1.OwnerReference{*ownerRef} + } if _, err = clientset.CoreV1().Secrets(namespace).Create(secret); err != nil { return fmt.Errorf("failed to save mon secrets. %+v", err) diff --git a/pkg/operator/cluster/ceph/osd/osd.go b/pkg/operator/cluster/ceph/osd/osd.go index b28f88dc5..1424d21da 100644 --- a/pkg/operator/cluster/ceph/osd/osd.go +++ b/pkg/operator/cluster/ceph/osd/osd.go @@ -63,10 +63,12 @@ type Cluster struct { dataDirHostPath string HostNetwork bool resources v1.ResourceRequirements + ownerRef metav1.OwnerReference } // New creates an instance of the OSD manager -func New(context *clusterd.Context, namespace, version string, storageSpec rookalpha.StorageSpec, dataDirHostPath string, placement rookalpha.Placement, hostNetwork bool, resources v1.ResourceRequirements) *Cluster { +func New(context *clusterd.Context, namespace, version string, storageSpec rookalpha.StorageSpec, dataDirHostPath string, placement rookalpha.Placement, hostNetwork bool, + resources v1.ResourceRequirements, ownerRef metav1.OwnerReference) *Cluster { return &Cluster{ context: context, Namespace: namespace, @@ -76,6 +78,7 @@ func New(context *clusterd.Context, namespace, version string, storageSpec rooka dataDirHostPath: dataDirHostPath, HostNetwork: hostNetwork, resources: resources, + ownerRef: ownerRef, } } @@ -83,8 +86,8 @@ func New(context *clusterd.Context, namespace, version string, storageSpec rooka func (c *Cluster) Start() error { logger.Infof("start running osds in namespace %s", c.Namespace) - // create the artifacts for the api service to work with RBAC enabled - err := k8sutil.MakeRole(c.context.Clientset, c.Namespace, appName, clusterAccessRules) + // create the artifacts for the osd to work with RBAC enabled + err := k8sutil.MakeRole(c.context.Clientset, c.Namespace, appName, clusterAccessRules, c.ownerRef) if err != nil { logger.Warningf("failed to init RBAC for OSDs. %+v", err) } @@ -130,34 +133,35 @@ func (c *Cluster) Start() error { } func (c *Cluster) makeDaemonSet(selection rookalpha.Selection, config rookalpha.Config) *extensions.DaemonSet { - ds := &extensions.DaemonSet{} - ds.Name = appName - ds.Namespace = c.Namespace - podSpec := c.podTemplateSpec(nil, selection, c.resources, config) - - ds.Spec = extensions.DaemonSetSpec{Template: podSpec} - return ds + return &extensions.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: appName, + Namespace: c.Namespace, + OwnerReferences: []metav1.OwnerReference{c.ownerRef}, + }, + Spec: extensions.DaemonSetSpec{Template: podSpec}, + } } func (c *Cluster) makeReplicaSet(nodeName string, devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements, config rookalpha.Config) *extensions.ReplicaSet { - rs := &extensions.ReplicaSet{} - rs.Name = fmt.Sprintf(appNameFmt, nodeName) - rs.Namespace = c.Namespace - podSpec := c.podTemplateSpec(devices, selection, resources, config) podSpec.Spec.NodeSelector = map[string]string{apis.LabelHostname: nodeName} - replicaCount := int32(1) - rs.Spec = extensions.ReplicaSetSpec{ - Template: podSpec, - Replicas: &replicaCount, + return &extensions.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(appNameFmt, nodeName), + Namespace: c.Namespace, + OwnerReferences: []metav1.OwnerReference{c.ownerRef}, + }, + Spec: extensions.ReplicaSetSpec{ + Template: podSpec, + Replicas: &replicaCount, + }, } - - return rs } func (c *Cluster) podTemplateSpec(devices []rookalpha.Device, selection rookalpha.Selection, @@ -220,6 +224,7 @@ func (c *Cluster) osdContainer(devices []rookalpha.Device, selection rookalpha.S envVars := []v1.EnvVar{ nodeNameEnvVar(), + {Name: "ROOK_CLUSTER_ID", Value: string(c.ownerRef.UID)}, k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), opmon.ClusterNameEnvVar(c.Namespace), diff --git a/pkg/operator/cluster/ceph/osd/osd_test.go b/pkg/operator/cluster/ceph/osd/osd_test.go index 59b8643ca..7dee3158b 100644 --- a/pkg/operator/cluster/ceph/osd/osd_test.go +++ b/pkg/operator/cluster/ceph/osd/osd_test.go @@ -25,13 +25,14 @@ import ( "github.com/stretchr/testify/assert" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" "k8s.io/kubernetes/pkg/kubelet/apis" ) func TestStartDaemonset(t *testing.T) { clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset}, "ns", "myversion", rookalpha.StorageSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(&clusterd.Context{Clientset: clientset}, "ns", "myversion", rookalpha.StorageSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) // Start the first time err := c.Start() @@ -66,7 +67,7 @@ func testPodDevices(t *testing.T, dataDir, deviceFilter string, allDevices bool) } clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset}, "ns", "rook/rook:myversion", storageSpec, dataDir, rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(&clusterd.Context{Clientset: clientset}, "ns", "rook/rook:myversion", storageSpec, dataDir, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) devMountNeeded := deviceFilter != "" || allDevices @@ -157,7 +158,7 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) { } clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset}, "ns", "rook/rook:myversion", storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(&clusterd.Context{Clientset: clientset}, "ns", "rook/rook:myversion", storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) n := c.Storage.ResolveNode(storageSpec.Nodes[0].Name) replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, n.Config) @@ -207,7 +208,7 @@ func TestStorageSpecConfig(t *testing.T) { } clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset}, "ns", "rook/rook:myversion", storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}) + c := New(&clusterd.Context{Clientset: clientset}, "ns", "rook/rook:myversion", storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) n := c.Storage.ResolveNode(storageSpec.Nodes[0].Name) replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Selection, c.Storage.Nodes[0].Resources, n.Config) @@ -245,7 +246,7 @@ func TestHostNetwork(t *testing.T) { } clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset}, "ns", "myversion", storageSpec, "", rookalpha.Placement{}, true, v1.ResourceRequirements{}) + c := New(&clusterd.Context{Clientset: clientset}, "ns", "myversion", storageSpec, "", rookalpha.Placement{}, true, v1.ResourceRequirements{}, metav1.OwnerReference{}) n := c.Storage.ResolveNode(storageSpec.Nodes[0].Name) r := c.makeReplicaSet(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, n.Config) diff --git a/pkg/operator/cluster/controller.go b/pkg/operator/cluster/controller.go index dd447e22d..142e2b942 100644 --- a/pkg/operator/cluster/controller.go +++ b/pkg/operator/cluster/controller.go @@ -41,6 +41,7 @@ import ( apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/cache" ) @@ -48,7 +49,7 @@ import ( const ( CustomResourceName = "cluster" CustomResourceNamePlural = "clusters" - crushConfigMapName = "crush-config" + crushConfigMapName = "rook-crush-config" crushmapCreatedKey = "initialCrushMapCreated" clusterCreateInterval = 6 * time.Second clusterCreateTimeout = 5 * time.Minute @@ -93,6 +94,7 @@ type cluster struct { osds *osd.Cluster apis *api.Cluster stopCh chan struct{} + ownerRef metav1.OwnerReference } // NewClusterController create controller for watching cluster custom resources created @@ -163,11 +165,11 @@ func (c *ClusterController) onAdd(obj interface{}) { poolController.StartWatch(cluster.Namespace, cluster.stopCh) // Start object store CRD watcher - objectStoreController := object.NewObjectStoreController(c.context, c.rookImage, cluster.Spec.HostNetwork) + objectStoreController := object.NewObjectStoreController(c.context, c.rookImage, cluster.Spec.HostNetwork, cluster.ownerRef) objectStoreController.StartWatch(cluster.Namespace, cluster.stopCh) // Start file system CRD watcher - fileController := file.NewFilesystemController(c.context, c.rookImage, cluster.Spec.HostNetwork) + fileController := file.NewFilesystemController(c.context, c.rookImage, cluster.Spec.HostNetwork, cluster.ownerRef) fileController.StartWatch(cluster.Namespace, cluster.stopCh) // Start mon health checker @@ -241,7 +243,18 @@ func (c *ClusterController) handleDelete(cluster *rookalpha.Cluster, retryInterv } func newCluster(c *rookalpha.Cluster, context *clusterd.Context) *cluster { - return &cluster{Namespace: c.Namespace, Spec: c.Spec, context: context} + return &cluster{Namespace: c.Namespace, Spec: c.Spec, context: context, ownerRef: ClusterOwnerRef(c.Namespace, string(c.UID))} +} + +func ClusterOwnerRef(namespace, clusterID string) metav1.OwnerReference { + blockOwner := true + return metav1.OwnerReference{ + APIVersion: ClusterResource.Version, + Kind: ClusterResource.Kind, + Name: namespace, + UID: types.UID(clusterID), + BlockOwnerDeletion: &blockOwner, + } } func (c *cluster) createInstance(rookImage string) error { @@ -251,15 +264,21 @@ func (c *cluster) createInstance(rookImage string) error { placeholderConfig := map[string]string{ k8sutil.ConfigOverrideVal: "", } - cm := &v1.ConfigMap{Data: placeholderConfig} - cm.Name = k8sutil.ConfigOverrideName + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: k8sutil.ConfigOverrideName, + OwnerReferences: []metav1.OwnerReference{c.ownerRef}, + }, + Data: placeholderConfig, + } + _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(cm) if err != nil && !errors.IsAlreadyExists(err) { return fmt.Errorf("failed to create override configmap %s. %+v", c.Namespace, err) } // Start the mon pods - c.mons = mon.New(c.context, c.Namespace, c.Spec.DataDirHostPath, rookImage, c.Spec.MonCount, c.Spec.Placement.GetMon(), c.Spec.HostNetwork, c.Spec.Resources.Mon) + c.mons = mon.New(c.context, c.Namespace, c.Spec.DataDirHostPath, rookImage, c.Spec.MonCount, c.Spec.Placement.GetMon(), c.Spec.HostNetwork, c.Spec.Resources.Mon, c.ownerRef) err = c.mons.Start() if err != nil { return fmt.Errorf("failed to start the mons. %+v", err) @@ -270,20 +289,20 @@ func (c *cluster) createInstance(rookImage string) error { return fmt.Errorf("failed to create initial crushmap: %+v", err) } - c.mgrs = mgr.New(c.context, c.Namespace, rookImage, c.Spec.Placement.GetMgr(), c.Spec.HostNetwork, c.Spec.Resources.Mgr) + c.mgrs = mgr.New(c.context, c.Namespace, rookImage, c.Spec.Placement.GetMgr(), c.Spec.HostNetwork, c.Spec.Resources.Mgr, c.ownerRef) err = c.mgrs.Start() if err != nil { return fmt.Errorf("failed to start the ceph mgr. %+v", err) } - c.apis = api.New(c.context, c.Namespace, rookImage, c.Spec.Placement.GetAPI(), c.Spec.HostNetwork, c.Spec.Resources.API) + c.apis = api.New(c.context, c.Namespace, rookImage, c.Spec.Placement.GetAPI(), c.Spec.HostNetwork, c.Spec.Resources.API, c.ownerRef) err = c.apis.Start() if err != nil { return fmt.Errorf("failed to start the REST api. %+v", err) } // Start the OSDs - c.osds = osd.New(c.context, c.Namespace, rookImage, c.Spec.Storage, c.Spec.DataDirHostPath, c.Spec.Placement.GetOSD(), c.Spec.HostNetwork, c.Spec.Resources.OSD) + c.osds = osd.New(c.context, c.Namespace, rookImage, c.Spec.Storage, c.Spec.DataDirHostPath, c.Spec.Placement.GetOSD(), c.Spec.HostNetwork, c.Spec.Resources.OSD, c.ownerRef) err = c.osds.Start() if err != nil { return fmt.Errorf("failed to start the osds. %+v", err) @@ -332,8 +351,9 @@ func (c *cluster) createInitialCrushMap() error { // save the fact that we've created the initial crushmap to a configmap configMap := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: crushConfigMapName, - Namespace: c.Namespace, + Name: crushConfigMapName, + Namespace: c.Namespace, + OwnerReferences: []metav1.OwnerReference{c.ownerRef}, }, Data: map[string]string{crushmapCreatedKey: "1"}, } diff --git a/pkg/operator/file/ceph/mds.go b/pkg/operator/file/ceph/mds.go index 9c55de25c..5030de3a1 100644 --- a/pkg/operator/file/ceph/mds.go +++ b/pkg/operator/file/ceph/mds.go @@ -59,11 +59,11 @@ func CreateFileSystem(context *clusterd.Context, clusterName string, f *model.Fi fs.Spec.DataPools = append(fs.Spec.DataPools, pool.ModelToSpec(p)) } - return CreateFilesystem(context, *fs, version, hostNetwork) + return CreateFilesystem(context, *fs, version, hostNetwork, []metav1.OwnerReference{}) } // Create the file system -func CreateFilesystem(context *clusterd.Context, fs rookalpha.Filesystem, version string, hostNetwork bool) error { +func CreateFilesystem(context *clusterd.Context, fs rookalpha.Filesystem, version string, hostNetwork bool, ownerRefs []metav1.OwnerReference) error { if err := validateFilesystem(context, fs); err != nil { return err } @@ -85,7 +85,7 @@ func CreateFilesystem(context *clusterd.Context, fs rookalpha.Filesystem, versio logger.Infof("start running mds for file system %s", fs.Name) // start the deployment - deployment := makeDeployment(fs, strconv.Itoa(filesystem.ID), version, hostNetwork) + deployment := makeDeployment(fs, strconv.Itoa(filesystem.ID), version, hostNetwork, ownerRefs) _, err = context.Clientset.ExtensionsV1beta1().Deployments(fs.Namespace).Create(deployment) if err != nil { if !errors.IsAlreadyExists(err) { @@ -123,10 +123,14 @@ func instanceName(fs rookalpha.Filesystem) string { return fmt.Sprintf("%s-%s", appName, fs.Name) } -func makeDeployment(fs rookalpha.Filesystem, filesystemID, version string, hostNetwork bool) *extensions.Deployment { - deployment := &extensions.Deployment{} - deployment.Name = instanceName(fs) - deployment.Namespace = fs.Namespace +func makeDeployment(fs rookalpha.Filesystem, filesystemID, version string, hostNetwork bool, ownerRefs []metav1.OwnerReference) *extensions.Deployment { + deployment := &extensions.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: instanceName(fs), + Namespace: fs.Namespace, + OwnerReferences: ownerRefs, + }, + } podSpec := v1.PodSpec{ Containers: []v1.Container{mdsContainer(fs, filesystemID, version)}, diff --git a/pkg/operator/file/ceph/mds_test.go b/pkg/operator/file/ceph/mds_test.go index 477cb9358..67c8d7948 100644 --- a/pkg/operator/file/ceph/mds_test.go +++ b/pkg/operator/file/ceph/mds_test.go @@ -75,12 +75,12 @@ func TestStartMDS(t *testing.T) { //defer os.RemoveAll(c.dataDir) // start a basic cluster - err := CreateFilesystem(context, fs, "v0.1", false) + err := CreateFilesystem(context, fs, "v0.1", false, []metav1.OwnerReference{}) assert.Nil(t, err) validateStart(t, context, fs) // starting again should be a no-op - err = CreateFilesystem(context, fs, "v0.1", false) + err = CreateFilesystem(context, fs, "v0.1", false, []metav1.OwnerReference{}) assert.Nil(t, err) validateStart(t, context, fs) } @@ -111,7 +111,7 @@ func TestPodSpecs(t *testing.T) { } mdsID := "mds1" - d := makeDeployment(fs, mdsID, "rook/rook:myversion", false) + d := makeDeployment(fs, mdsID, "rook/rook:myversion", false, []metav1.OwnerReference{}) assert.NotNil(t, d) assert.Equal(t, appName+"-myfs", d.Name) assert.Equal(t, v1.RestartPolicyAlways, d.Spec.Template.Spec.RestartPolicy) @@ -144,7 +144,7 @@ func TestHostNetwork(t *testing.T) { } mdsID := "mds1" - d := makeDeployment(fs, mdsID, "v0.1", true) + d := makeDeployment(fs, mdsID, "v0.1", true, []metav1.OwnerReference{}) assert.Equal(t, true, d.Spec.Template.Spec.HostNetwork) assert.Equal(t, v1.DNSClusterFirstWithHostNet, d.Spec.Template.Spec.DNSPolicy) diff --git a/pkg/operator/file/controller.go b/pkg/operator/file/controller.go index eb9f3cd14..d9324af26 100644 --- a/pkg/operator/file/controller.go +++ b/pkg/operator/file/controller.go @@ -26,6 +26,7 @@ import ( "github.com/rook/rook/pkg/clusterd" mds "github.com/rook/rook/pkg/operator/file/ceph" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" ) @@ -46,14 +47,16 @@ type FilesystemController struct { context *clusterd.Context rookImage string hostNetwork bool + ownerRef metav1.OwnerReference } // NewFilesystemController create controller for watching file system custom resources created -func NewFilesystemController(context *clusterd.Context, rookImage string, hostNetwork bool) *FilesystemController { +func NewFilesystemController(context *clusterd.Context, rookImage string, hostNetwork bool, ownerRef metav1.OwnerReference) *FilesystemController { return &FilesystemController{ context: context, rookImage: rookImage, hostNetwork: hostNetwork, + ownerRef: ownerRef, } } @@ -75,7 +78,7 @@ func (c *FilesystemController) StartWatch(namespace string, stopCh chan struct{} func (c *FilesystemController) onAdd(obj interface{}) { filesystem := obj.(*rookalpha.Filesystem).DeepCopy() - err := mds.CreateFilesystem(c.context, *filesystem, c.rookImage, c.hostNetwork) + err := mds.CreateFilesystem(c.context, *filesystem, c.rookImage, c.hostNetwork, c.filesystemOwners(filesystem)) if err != nil { logger.Errorf("failed to create file system %s. %+v", filesystem.Name, err) } @@ -91,7 +94,7 @@ func (c *FilesystemController) onUpdate(oldObj, newObj interface{}) { // if the file system is modified, allow the file system to be created if it wasn't already logger.Infof("updating filesystem %s", newFS) - err := mds.CreateFilesystem(c.context, *newFS, c.rookImage, c.hostNetwork) + err := mds.CreateFilesystem(c.context, *newFS, c.rookImage, c.hostNetwork, c.filesystemOwners(newFS)) if err != nil { logger.Errorf("failed to create (modify) file system %s. %+v", newFS.Name, err) } @@ -105,6 +108,15 @@ func (c *FilesystemController) onDelete(obj interface{}) { } } +func (c *FilesystemController) filesystemOwners(fs *rookalpha.Filesystem) []metav1.OwnerReference { + + // Only set the cluster crd as the owner of the filesystem resources. + // If the filesystem crd is deleted, the operator will explicitly remove the filesystem resources. + // If the filesystem crd still exists when the cluster crd is deleted, this will make sure the filesystem + // resources are cleaned up. + return []metav1.OwnerReference{c.ownerRef} +} + func filesystemChanged(oldFS, newFS rookalpha.FilesystemSpec) bool { if len(oldFS.DataPools) != len(newFS.DataPools) { logger.Infof("number of data pools changed from %d to %d", len(oldFS.DataPools), len(newFS.DataPools)) diff --git a/pkg/operator/k8sutil/kvstore.go b/pkg/operator/k8sutil/kvstore.go index 00729999c..1ca8dfbb4 100644 --- a/pkg/operator/k8sutil/kvstore.go +++ b/pkg/operator/k8sutil/kvstore.go @@ -28,12 +28,14 @@ import ( type ConfigMapKVStore struct { namespace string clientset kubernetes.Interface + ownerRef metav1.OwnerReference } -func NewConfigMapKVStore(namespace string, clientset kubernetes.Interface) *ConfigMapKVStore { +func NewConfigMapKVStore(namespace string, clientset kubernetes.Interface, ownerRef metav1.OwnerReference) *ConfigMapKVStore { return &ConfigMapKVStore{ namespace: namespace, clientset: clientset, + ownerRef: ownerRef, } } @@ -61,8 +63,9 @@ func (kv *ConfigMapKVStore) SetValue(storeName, key, value string) error { // the given config map doesn't exist yet, create it now with the given key/val cm = &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: storeName, - Namespace: kv.namespace, + Name: storeName, + Namespace: kv.namespace, + OwnerReferences: []metav1.OwnerReference{kv.ownerRef}, }, Data: map[string]string{key: value}, } diff --git a/pkg/operator/k8sutil/kvstore_test.go b/pkg/operator/k8sutil/kvstore_test.go index 3c438186d..a915213fa 100644 --- a/pkg/operator/k8sutil/kvstore_test.go +++ b/pkg/operator/k8sutil/kvstore_test.go @@ -24,6 +24,7 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" ) @@ -167,5 +168,5 @@ func newKVStore(stores ...*v1.ConfigMap) (*ConfigMapKVStore, string) { } clientset := fake.NewSimpleClientset(objects...) - return NewConfigMapKVStore(namespace, clientset), storeName + return NewConfigMapKVStore(namespace, clientset, metav1.OwnerReference{}), storeName } diff --git a/pkg/operator/k8sutil/rbac.go b/pkg/operator/k8sutil/rbac.go index fe6098c85..f2acd88b0 100644 --- a/pkg/operator/k8sutil/rbac.go +++ b/pkg/operator/k8sutil/rbac.go @@ -32,9 +32,9 @@ const ( enableRBACEnv = "RBAC_ENABLED" ) -func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1beta1.PolicyRule) error { +func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1beta1.PolicyRule, ownerRef metav1.OwnerReference) error { - err := makeServiceAccount(clientset, namespace, name) + err := makeServiceAccount(clientset, namespace, name, &ownerRef) if err != nil { return err } @@ -46,8 +46,13 @@ func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1 // Create the role if it doesn't yet exist. // If the role already exists we have to update it. Otherwise if the permissions change during an upgrade, // the create will fail with an error that we're changing the permissions. - role := &v1beta1.Role{Rules: rules} - role.Name = name + role := &v1beta1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + OwnerReferences: []metav1.OwnerReference{ownerRef}, + }, + Rules: rules, + } _, err = clientset.RbacV1beta1().Roles(namespace).Get(role.Name, metav1.GetOptions{}) if errors.IsNotFound(err) { logger.Infof("creating role %s in namespace %s", name, namespace) @@ -60,10 +65,20 @@ func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1 return fmt.Errorf("failed to create/update role %s in namespace %s. %+v", name, namespace, err) } - binding := &v1beta1.RoleBinding{} - binding.Name = name - binding.RoleRef = v1beta1.RoleRef{Name: name, Kind: "Role", APIGroup: "rbac.authorization.k8s.io"} - binding.Subjects = []v1beta1.Subject{{Kind: "ServiceAccount", Name: name, Namespace: namespace}} + binding := &v1beta1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + OwnerReferences: []metav1.OwnerReference{ownerRef}, + }, + RoleRef: v1beta1.RoleRef{ + Name: name, + Kind: "Role", + APIGroup: "rbac.authorization.k8s.io", + }, + Subjects: []v1beta1.Subject{ + {Kind: "ServiceAccount", Name: name, Namespace: namespace}, + }, + } _, err = clientset.RbacV1beta1().RoleBindings(namespace).Create(binding) if err != nil && !errors.IsAlreadyExists(err) { return fmt.Errorf("failed to create %s role binding in namespace %s. %+v", name, namespace, err) @@ -71,9 +86,9 @@ func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1 return nil } -func MakeClusterRole(clientset kubernetes.Interface, namespace, name string, rules []v1beta1.PolicyRule) error { +func MakeClusterRole(clientset kubernetes.Interface, namespace, name string, rules []v1beta1.PolicyRule, ownerRef *metav1.OwnerReference) error { - err := makeServiceAccount(clientset, namespace, name) + err := makeServiceAccount(clientset, namespace, name, ownerRef) if err != nil { return err } @@ -85,8 +100,16 @@ func MakeClusterRole(clientset kubernetes.Interface, namespace, name string, rul // Create the cluster scoped role if it doesn't yet exist. // If the role already exists we have to update it. Otherwise if the permissions change during an upgrade, // the create will fail with an error that we're changing the permissions. - role := &v1beta1.ClusterRole{Rules: rules} - role.Name = name + role := &v1beta1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Rules: rules, + } + if ownerRef != nil { + role.OwnerReferences = []metav1.OwnerReference{*ownerRef} + } + _, err = clientset.RbacV1beta1().ClusterRoles().Get(role.Name, metav1.GetOptions{}) if errors.IsNotFound(err) { logger.Infof("creating cluster role %s", name) @@ -99,10 +122,23 @@ func MakeClusterRole(clientset kubernetes.Interface, namespace, name string, rul return fmt.Errorf("failed to create/update cluster role %s. %+v", name, err) } - binding := &v1beta1.ClusterRoleBinding{} - binding.Name = name - binding.RoleRef = v1beta1.RoleRef{Name: name, Kind: "ClusterRole", APIGroup: "rbac.authorization.k8s.io"} - binding.Subjects = []v1beta1.Subject{{Kind: "ServiceAccount", Name: name, Namespace: namespace}} + binding := &v1beta1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + RoleRef: v1beta1.RoleRef{ + Name: name, + Kind: "ClusterRole", + APIGroup: "rbac.authorization.k8s.io", + }, + Subjects: []v1beta1.Subject{ + {Kind: "ServiceAccount", Name: name, Namespace: namespace}, + }, + } + if ownerRef != nil { + binding.OwnerReferences = []metav1.OwnerReference{*ownerRef} + } + _, err = clientset.RbacV1beta1().ClusterRoleBindings().Create(binding) if err != nil && !errors.IsAlreadyExists(err) { return fmt.Errorf("failed to create cluster role binding %s. %+v", name, err) @@ -110,10 +146,17 @@ func MakeClusterRole(clientset kubernetes.Interface, namespace, name string, rul return nil } -func makeServiceAccount(clientset kubernetes.Interface, namespace, name string) error { - account := &v1.ServiceAccount{} - account.Name = name - account.Namespace = namespace +func makeServiceAccount(clientset kubernetes.Interface, namespace, name string, ownerRef *metav1.OwnerReference) error { + account := &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + if ownerRef != nil { + account.OwnerReferences = []metav1.OwnerReference{*ownerRef} + } + _, err := clientset.CoreV1().ServiceAccounts(namespace).Create(account) if err != nil && !errors.IsAlreadyExists(err) { return fmt.Errorf("failed to create %s service account in namespace %s. %+v", name, namespace, err) diff --git a/pkg/operator/k8sutil/rbac_test.go b/pkg/operator/k8sutil/rbac_test.go index 9f5c1c089..5f3e10684 100644 --- a/pkg/operator/k8sutil/rbac_test.go +++ b/pkg/operator/k8sutil/rbac_test.go @@ -56,7 +56,7 @@ func TestMakeRole(t *testing.T) { }, } - err := MakeRole(clientset, namespace, name, rules) + err := MakeRole(clientset, namespace, name, rules, metav1.OwnerReference{}) role, err := clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{}) assert.Nil(t, err) @@ -82,7 +82,7 @@ func TestMakeRole(t *testing.T) { }, } - err = MakeRole(clientset, namespace, name, newRules) + err = MakeRole(clientset, namespace, name, newRules, metav1.OwnerReference{}) assert.Nil(t, err) role, err = clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{}) assert.Nil(t, err) @@ -110,7 +110,7 @@ func TestMakeClusterRole(t *testing.T) { }, } - err := MakeClusterRole(clientset, namespace, name, rules) + err := MakeClusterRole(clientset, namespace, name, rules, nil) role, err := clientset.RbacV1beta1().ClusterRoles().Get(name, metav1.GetOptions{}) assert.Nil(t, err) @@ -136,7 +136,7 @@ func TestMakeClusterRole(t *testing.T) { }, } - err = MakeClusterRole(clientset, namespace, name, newRules) + err = MakeClusterRole(clientset, namespace, name, newRules, nil) assert.Nil(t, err) role, err = clientset.RbacV1beta1().ClusterRoles().Get(name, metav1.GetOptions{}) assert.Nil(t, err) @@ -176,7 +176,7 @@ func TestMakeRoleRBACDisabled(t *testing.T) { }, } - err := MakeRole(clientset, namespace, name, rules) + err := MakeRole(clientset, namespace, name, rules, metav1.OwnerReference{}) _, err = clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{}) assert.NotNil(t, err) @@ -211,7 +211,7 @@ func TestMakeClusterRoleRBACDisabled(t *testing.T) { }, } - err := MakeClusterRole(clientset, namespace, name, rules) + err := MakeClusterRole(clientset, namespace, name, rules, nil) assert.Nil(t, err) _, err = clientset.RbacV1beta1().ClusterRoles().Get(name, metav1.GetOptions{}) assert.NotNil(t, err) diff --git a/pkg/operator/object/ceph/rgw.go b/pkg/operator/object/ceph/rgw.go index 16507c3b5..f7a2c188b 100644 --- a/pkg/operator/object/ceph/rgw.go +++ b/pkg/operator/object/ceph/rgw.go @@ -49,15 +49,15 @@ const ( ) // Start the rgw manager -func CreateStore(context *clusterd.Context, store rookalpha.ObjectStore, version string, hostNetwork bool) error { - return createOrUpdate(context, store, version, hostNetwork, false) +func CreateStore(context *clusterd.Context, store rookalpha.ObjectStore, version string, hostNetwork bool, ownerRefs []metav1.OwnerReference) error { + return createOrUpdate(context, store, version, hostNetwork, false, ownerRefs) } -func UpdateStore(context *clusterd.Context, store rookalpha.ObjectStore, version string, hostNetwork bool) error { - return createOrUpdate(context, store, version, hostNetwork, true) +func UpdateStore(context *clusterd.Context, store rookalpha.ObjectStore, version string, hostNetwork bool, ownerRefs []metav1.OwnerReference) error { + return createOrUpdate(context, store, version, hostNetwork, true, ownerRefs) } -func createOrUpdate(context *clusterd.Context, store rookalpha.ObjectStore, version string, hostNetwork, update bool) error { +func createOrUpdate(context *clusterd.Context, store rookalpha.ObjectStore, version string, hostNetwork, update bool, ownerRefs []metav1.OwnerReference) error { // validate the object store settings if err := validateStore(context, store); err != nil { return fmt.Errorf("invalid object store %s arguments. %+v", store.Name, err) @@ -74,13 +74,13 @@ func createOrUpdate(context *clusterd.Context, store rookalpha.ObjectStore, vers } logger.Infof("creating object store %s in namespace %s", store.Name, store.Namespace) - err = createKeyring(context, store) + err = createKeyring(context, store, ownerRefs) if err != nil { return fmt.Errorf("failed to create rgw keyring. %+v", err) } // start the service - serviceIP, err := startService(context, store, hostNetwork) + serviceIP, err := startService(context, store, hostNetwork, ownerRefs) if err != nil { return fmt.Errorf("failed to start rgw service. %+v", err) } @@ -92,7 +92,7 @@ func createOrUpdate(context *clusterd.Context, store rookalpha.ObjectStore, vers return fmt.Errorf("failed to create pools. %+v", err) } - if err := startRGWPods(context, store, version, hostNetwork, update); err != nil { + if err := startRGWPods(context, store, version, hostNetwork, update, ownerRefs); err != nil { return fmt.Errorf("failed to start pods. %+v", err) } @@ -100,7 +100,7 @@ func createOrUpdate(context *clusterd.Context, store rookalpha.ObjectStore, vers return nil } -func startRGWPods(context *clusterd.Context, store rookalpha.ObjectStore, version string, hostNetwork, update bool) error { +func startRGWPods(context *clusterd.Context, store rookalpha.ObjectStore, version string, hostNetwork, update bool, ownerRefs []metav1.OwnerReference) error { // if intended to update, remove the old pods so they can be created with the new spec settings if update { @@ -119,10 +119,10 @@ func startRGWPods(context *clusterd.Context, store rookalpha.ObjectStore, versio var err error if store.Spec.Gateway.AllNodes { rgwType = "daemonset" - err = startDaemonset(context, store, version, hostNetwork) + err = startDaemonset(context, store, version, hostNetwork, ownerRefs) } else { rgwType = "deployment" - err = startDeployment(context, store, version, store.Spec.Gateway.Instances, hostNetwork) + err = startDeployment(context, store, version, store.Spec.Gateway.Instances, hostNetwork, ownerRefs) } if err != nil { @@ -213,7 +213,7 @@ func storeExists(context *clusterd.Context, store rookalpha.ObjectStore) (bool, return false, nil } -func createKeyring(context *clusterd.Context, store rookalpha.ObjectStore) error { +func createKeyring(context *clusterd.Context, store rookalpha.ObjectStore, ownerRefs []metav1.OwnerReference) error { _, err := context.Clientset.CoreV1().Secrets(store.Namespace).Get(instanceName(store), metav1.GetOptions{}) if err == nil { logger.Infof("the rgw keyring was already generated") @@ -235,7 +235,11 @@ func createKeyring(context *clusterd.Context, store rookalpha.ObjectStore) error keyringName: keyring, } secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: instanceName(store), Namespace: store.Namespace}, + ObjectMeta: metav1.ObjectMeta{ + Name: instanceName(store), + Namespace: store.Namespace, + OwnerReferences: ownerRefs, + }, StringData: secrets, Type: k8sutil.RookType, } @@ -307,12 +311,13 @@ func makeRGWPodSpec(store rookalpha.ObjectStore, version string, hostNetwork boo } } -func startDeployment(context *clusterd.Context, store rookalpha.ObjectStore, version string, replicas int32, hostNetwork bool) error { +func startDeployment(context *clusterd.Context, store rookalpha.ObjectStore, version string, replicas int32, hostNetwork bool, ownerRefs []metav1.OwnerReference) error { deployment := &extensions.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: instanceName(store), - Namespace: store.Namespace, + Name: instanceName(store), + Namespace: store.Namespace, + OwnerReferences: ownerRefs, }, Spec: extensions.DeploymentSpec{Template: makeRGWPodSpec(store, version, hostNetwork), Replicas: &replicas}, } @@ -320,12 +325,13 @@ func startDeployment(context *clusterd.Context, store rookalpha.ObjectStore, ver return err } -func startDaemonset(context *clusterd.Context, store rookalpha.ObjectStore, version string, hostNetwork bool) error { +func startDaemonset(context *clusterd.Context, store rookalpha.ObjectStore, version string, hostNetwork bool, ownerRefs []metav1.OwnerReference) error { daemonset := &extensions.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ - Name: instanceName(store), - Namespace: store.Namespace, + Name: instanceName(store), + Namespace: store.Namespace, + OwnerReferences: ownerRefs, }, Spec: extensions.DaemonSetSpec{ Template: makeRGWPodSpec(store, version, hostNetwork), @@ -378,13 +384,14 @@ func rgwContainer(store rookalpha.ObjectStore, version string) v1.Container { return container } -func startService(context *clusterd.Context, store rookalpha.ObjectStore, hostNetwork bool) (string, error) { +func startService(context *clusterd.Context, store rookalpha.ObjectStore, hostNetwork bool, ownerRefs []metav1.OwnerReference) (string, error) { labels := getLabels(store) svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: instanceName(store), - Namespace: store.Namespace, - Labels: labels, + Name: instanceName(store), + Namespace: store.Namespace, + Labels: labels, + OwnerReferences: ownerRefs, }, Spec: v1.ServiceSpec{ Selector: labels, diff --git a/pkg/operator/object/ceph/rgw_test.go b/pkg/operator/object/ceph/rgw_test.go index 55abb1b64..735e53ac6 100644 --- a/pkg/operator/object/ceph/rgw_test.go +++ b/pkg/operator/object/ceph/rgw_test.go @@ -52,14 +52,14 @@ func TestStartRGW(t *testing.T) { version := "v1.1.0" // start a basic cluster - err := CreateStore(context, store, version, false) + err := CreateStore(context, store, version, false, []metav1.OwnerReference{}) assert.Nil(t, err) validateStart(t, store, clientset, false) // starting again should update the pods with the new settings store.Spec.Gateway.AllNodes = true - err = UpdateStore(context, store, version, false) + err = UpdateStore(context, store, version, false, []metav1.OwnerReference{}) assert.Nil(t, err) validateStart(t, store, clientset, true) @@ -180,7 +180,7 @@ func TestCreateObjectStore(t *testing.T) { context := &clusterd.Context{Executor: executor, Clientset: clientset} // create the pools - err := CreateStore(context, store, "1.2.3.4", false) + err := CreateStore(context, store, "1.2.3.4", false, []metav1.OwnerReference{}) assert.Nil(t, err) } diff --git a/pkg/operator/object/controller.go b/pkg/operator/object/controller.go index 82ad6af61..82eaa1e23 100644 --- a/pkg/operator/object/controller.go +++ b/pkg/operator/object/controller.go @@ -26,6 +26,7 @@ import ( "github.com/rook/rook/pkg/clusterd" rgw "github.com/rook/rook/pkg/operator/object/ceph" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" ) @@ -46,14 +47,16 @@ type ObjectStoreController struct { context *clusterd.Context rookImage string hostNetwork bool + ownerRef metav1.OwnerReference } // NewObjectStoreController create controller for watching object store custom resources created -func NewObjectStoreController(context *clusterd.Context, rookImage string, hostNetwork bool) *ObjectStoreController { +func NewObjectStoreController(context *clusterd.Context, rookImage string, hostNetwork bool, ownerRef metav1.OwnerReference) *ObjectStoreController { return &ObjectStoreController{ context: context, rookImage: rookImage, hostNetwork: hostNetwork, + ownerRef: ownerRef, } } @@ -74,7 +77,8 @@ func (c *ObjectStoreController) StartWatch(namespace string, stopCh chan struct{ func (c *ObjectStoreController) onAdd(obj interface{}) { store := obj.(*rookalpha.ObjectStore).DeepCopy() - err := rgw.CreateStore(c.context, *store, c.rookImage, c.hostNetwork) + + err := rgw.CreateStore(c.context, *store, c.rookImage, c.hostNetwork, c.storeOwners(store)) if err != nil { logger.Errorf("failed to create object store %s. %+v", store.Name, err) } @@ -90,7 +94,7 @@ func (c *ObjectStoreController) onUpdate(oldObj, newObj interface{}) { } logger.Infof("applying object store %s changes", newStore.Name) - err := rgw.UpdateStore(c.context, *newStore, c.rookImage, c.hostNetwork) + err := rgw.UpdateStore(c.context, *newStore, c.rookImage, c.hostNetwork, c.storeOwners(newStore)) if err != nil { logger.Errorf("failed to create (modify) object store %s. %+v", newStore.Name, err) } @@ -104,6 +108,14 @@ func (c *ObjectStoreController) onDelete(obj interface{}) { } } +func (c *ObjectStoreController) storeOwners(store *rookalpha.ObjectStore) []metav1.OwnerReference { + // Only set the cluster crd as the owner of the object store resources. + // If the object store crd is deleted, the operator will explicitly remove the object store resources. + // If the object store crd still exists when the cluster crd is deleted, this will make sure the object store + // resources are cleaned up. + return []metav1.OwnerReference{c.ownerRef} +} + func storeChanged(oldStore, newStore rookalpha.ObjectStoreSpec) bool { if oldStore.DataPool.Replicated.Size != newStore.DataPool.Replicated.Size { logger.Infof("data pool replication changed from %d to %d", oldStore.DataPool.Replicated.Size, newStore.DataPool.Replicated.Size)