Merge pull request #4235 from egafford/generate-keyrings-before-RGW-deployment

ceph: Generate RGW keyrings before creating dependent deployments

pkg/operator/ceph/object/config.go

@@ -109,17 +109,25 @@ func generateCephXUser(name string) string {
 	return "client" + strings.Replace(user, "-", ".", -1)
 }
 
-func (c *clusterConfig) generateKeyring(replicationControllerOwnerRef *metav1.OwnerReference) error {
-	user := generateCephXUser(replicationControllerOwnerRef.Name)
+func (c *clusterConfig) generateKeyring(rgwConfig *rgwConfig) (string, error) {
+	user := generateCephXUser(rgwConfig.ResourceName)
 	/* TODO: this says `osd allow rwx` while template says `osd allow *`; which is correct? */
 	access := []string{"osd", "allow rwx", "mon", "allow rw"}
-	s := keyring.GetSecretStore(c.context, c.store.Namespace, replicationControllerOwnerRef)
+	s := keyring.GetSecretStore(c.context, c.store.Namespace, &c.ownerRef)
 
 	key, err := s.GenerateKey(user, access)
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	keyring := fmt.Sprintf(keyringTemplate, user, key)
-	return s.CreateOrUpdate(replicationControllerOwnerRef.Name, keyring)
+	return keyring, s.CreateOrUpdate(rgwConfig.ResourceName, keyring)
+}
+
+func (c *clusterConfig) associateKeyring(existingKeyring string, ownerRef *metav1.OwnerReference) error {
+	resourceName := ownerRef.Name
+
+	s := keyring.GetSecretStore(c.context, c.store.Namespace, ownerRef)
+
+	return s.CreateOrUpdate(resourceName, existingKeyring)
 }

pkg/operator/ceph/object/controller.go

@@ -159,7 +159,7 @@ func (c *ObjectStoreController) createOrUpdateStore(objectstore *cephv1.CephObje
 		store:       *objectstore,
 		rookVersion: c.rookImage,
 		clusterSpec: c.clusterSpec,
-		ownerRefs:   c.storeOwners(objectstore),
+		ownerRef:    c.storeOwners(objectstore),
 		DataPathMap: cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, objectstore.Name, c.clusterInfo.Name, c.dataDirHostPath),
 		isUpgrade:   c.isUpgrade,
 	}
@@ -220,14 +220,14 @@ func (c *ObjectStoreController) ParentClusterChanged(cluster cephv1.ClusterSpec,
 	}
 }
 
-func (c *ObjectStoreController) storeOwners(store *cephv1.CephObjectStore) []metav1.OwnerReference {
+func (c *ObjectStoreController) storeOwners(store *cephv1.CephObjectStore) metav1.OwnerReference {
 	// Set the object store CR as the owner
-	return []metav1.OwnerReference{{
+	return metav1.OwnerReference{
 		APIVersion: fmt.Sprintf("%s/%s", ObjectStoreResource.Group, ObjectStoreResource.Version),
 		Kind:       ObjectStoreResource.Kind,
 		Name:       store.Name,
 		UID:        store.UID,
-	}}
+	}
 }
 
 func storeChanged(oldStore, newStore cephv1.ObjectStoreSpec) bool {

pkg/operator/ceph/object/rgw.go

@@ -40,7 +40,7 @@ type clusterConfig struct {
 	store             cephv1.CephObjectStore
 	rookVersion       string
 	clusterSpec       *cephv1.ClusterSpec
-	ownerRefs         []metav1.OwnerReference
+	ownerRef          metav1.OwnerReference
 	DataPathMap       *config.DataPathMap
 	isUpgrade         bool
 	skipUpgradeChecks bool
@@ -116,6 +116,13 @@ func (c *clusterConfig) startRGWPods() error {
 			DaemonID:     daemonName,
 		}
 
+		// Generate the keyring after starting the replication controller so that the keyring may use
+		// the controller as its owner reference; the keyring is deleted with the controller
+		keyring, err := c.generateKeyring(rgwConfig)
+		if err != nil {
+			return fmt.Errorf("failed to create rgw keyring. %+v", err)
+		}
+
 		deployment := c.createDeployment(rgwConfig)
 		logger.Infof("object store %s deployment %s started", c.store.Name, deployment.Name)
 		createdDeployment, createErr := c.context.Clientset.AppsV1().Deployments(c.store.Namespace).Create(deployment)
@@ -137,11 +144,9 @@ func (c *clusterConfig) startRGWPods() error {
 			Name:       rgwConfig.ResourceName,
 		}
 
-		// Generate the keyring after starting the replication controller so that the keyring may use
-		// the controller as its owner reference; the keyring is deleted with the controller
-		err = c.generateKeyring(resourceControllerOwnerRef)
+		err = c.associateKeyring(keyring, resourceControllerOwnerRef)
 		if err != nil {
-			return fmt.Errorf("failed to create rgw keyring. %+v", err)
+			logger.Warningf("failed to associate keyring with rgw deployment %q: %+v", createdDeployment.Name, err)
 		}
 
 		// Generate the mime.types file after the rep. controller as well for the same reason as keyring

pkg/operator/ceph/object/rgw_test.go

@@ -52,7 +52,7 @@ func TestStartRGW(t *testing.T) {
 	data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "my-fs", "rook-ceph", "/var/lib/rook/")
 
 	// start a basic cluster
-	c := &clusterConfig{info, context, store, version, &cephv1.ClusterSpec{}, []metav1.OwnerReference{}, data, false, false}
+	c := &clusterConfig{info, context, store, version, &cephv1.ClusterSpec{}, metav1.OwnerReference{}, data, false, false}
 	err := c.startRGWPods()
 	assert.Nil(t, err)
 
@@ -94,7 +94,7 @@ func TestCreateObjectStore(t *testing.T) {
 	data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "my-fs", "rook-ceph", "/var/lib/rook/")
 
 	// create the pools
-	c := &clusterConfig{info, context, store, "1.2.3.4", &cephv1.ClusterSpec{}, []metav1.OwnerReference{}, data, false, false}
+	c := &clusterConfig{info, context, store, "1.2.3.4", &cephv1.ClusterSpec{}, metav1.OwnerReference{}, data, false, false}
 	err := c.createOrUpdate()
 	assert.Nil(t, err)
 }

pkg/operator/ceph/object/spec.go

@@ -52,7 +52,7 @@ func (c *clusterConfig) createDeployment(rgwConfig *rgwConfig) *apps.Deployment
 	k8sutil.AddRookVersionLabelToDeployment(d)
 	c.store.Spec.Gateway.Annotations.ApplyToObjectMeta(&d.ObjectMeta)
 	opspec.AddCephVersionLabelToDeployment(c.clusterInfo.CephVersion, d)
-	k8sutil.SetOwnerRefs(&d.ObjectMeta, c.ownerRefs)
+	k8sutil.SetOwnerRef(&d.ObjectMeta, &c.ownerRef)
 
 	return d
 }
@@ -170,7 +170,7 @@ func (c *clusterConfig) startService() (string, error) {
 			Selector: labels,
 		},
 	}
-	k8sutil.SetOwnerRefs(&svc.ObjectMeta, c.ownerRefs)
+	k8sutil.SetOwnerRef(&svc.ObjectMeta, &c.ownerRef)
 	if c.clusterSpec.Network.IsHost() {
 		svc.Spec.ClusterIP = v1.ClusterIPNone
 	}