Merge pull request #16429 from rook/mergify/bp/release-1.18/pr-16421

osd: remove the osd bootstrap keyring (backport #16421)

pkg/daemon/ceph/client/keyring.go

@@ -56,16 +56,6 @@ func CephKeyring(cred CephCred) string {
 // CreateKeyring creates a keyring for access to the cluster with the desired set of privileges
 // and writes it to disk at the keyring path
 func CreateKeyring(context *clusterd.Context, clusterInfo *ClusterInfo, username, keyringPath string, access []string, generateContents func(string) string) error {
-	_, err := os.Stat(keyringPath)
-	if err == nil {
-		// no error, the file exists, bail out with no error
-		logger.Debugf("keyring already exists at %s", keyringPath)
-		return nil
-	} else if !os.IsNotExist(err) {
-		// some other error besides "does not exist", bail out with error
-		return errors.Wrapf(err, "failed to stat %s", keyringPath)
-	}
-
 	// get-or-create-key for the user account
 	key, err := AuthGetOrCreateKey(context, clusterInfo, username, access)
 	if err != nil {

pkg/daemon/ceph/osd/volume_test.go

@@ -576,11 +576,21 @@ func TestConfigureCVDevices(t *testing.T) {
 			}
 			return "", errors.Errorf("unknown command %s %s", command, args)
 		}
-
-		context := &clusterd.Context{Executor: executor, ConfigDir: cephConfigDir}
-		clusterInfo := &cephclient.ClusterInfo{
-			FSID: clusterFSID,
+		executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
+			logger.Infof("[MockExecuteCommandWithOutput] %s %v", command, args)
+			if args[0] == "auth" && args[1] == "get-or-create-key" {
+				return "{\"key\":\"mysecurekey\"}", nil
+			}
+			if args[1] == "ceph-volume" && args[2] == "--log-path" && args[3] == "/tmp/ceph-log" {
+				return `{}`, nil
+			}
+			return "", errors.Errorf("unknown command %s %s", command, args)
 		}
+		clusterInfo := &cephclient.ClusterInfo{
+			FSID:    clusterFSID,
+			Context: context.TODO(),
+		}
+		context := &clusterd.Context{Executor: executor, ConfigDir: cephConfigDir}
 		agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, storeConfig: config.StoreConfig{DeviceClass: "myclass", StoreType: "bluestore"}}
 		devices := &DeviceOsdMapping{
 			Entries: map[string]*DeviceOsdIDEntry{
@@ -611,6 +621,12 @@ func TestConfigureCVDevices(t *testing.T) {
 			if command == "sgdisk" {
 				return "Disk identifier (GUID): 18484D7E-5287-4CE9-AC73-D02FB69055CE", nil
 			}
+			if args[0] == "auth" && args[1] == "get-or-create-key" {
+				return "{\"key\":\"mysecurekey\"}", nil
+			}
+			if args[1] == "ceph-volume" && args[2] == "--log-path" && args[3] == "/tmp/ceph-log" {
+				return `{}`, nil
+			}
 			return "", errors.Errorf("unknown command %s %s", command, args)
 		}
 		deviceClassSet := false
@@ -624,10 +640,11 @@ func TestConfigureCVDevices(t *testing.T) {
 			return "", errors.Errorf("unknown command %s %s", command, args)
 		}
 
-		context := &clusterd.Context{Executor: executor, ConfigDir: cephConfigDir}
 		clusterInfo := &cephclient.ClusterInfo{
-			FSID: clusterFSID,
+			FSID:    clusterFSID,
+			Context: context.TODO(),
 		}
+		context := &clusterd.Context{Executor: executor, ConfigDir: cephConfigDir}
 		agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, storeConfig: config.StoreConfig{DeviceClass: "myclass", StoreType: "bluestore"}}
 		devices := &DeviceOsdMapping{
 			Entries: map[string]*DeviceOsdIDEntry{

pkg/operator/ceph/cluster/osd/osd.go

@@ -350,10 +350,22 @@ func (c *Cluster) Start() error {
 		return errors.Wrapf(err, "failed to update ceph storage status")
 	}
 
+	// remove the osdBootstrapKey as the osds are running
+	c.deleteOsdBootstrapKeyring()
+
 	logger.Infof("finished running OSDs in namespace %q", namespace)
 	return nil
 }
 
+func (c *Cluster) deleteOsdBootstrapKeyring() {
+	err := cephclient.AuthDelete(c.context, c.clusterInfo, "client.bootstrap-osd")
+	if err != nil {
+		logger.Debugf("failed to delete client.bootstrap-osd bootstrap key in the namespace %q. Error: %v", c.clusterInfo.Namespace, err)
+	} else {
+		logger.Debugf("successfully deleted client.bootstrap-osd bootstrap key in the namespace %q", c.clusterInfo.Namespace)
+	}
+}
+
 func (c *Cluster) startOSDMigration() (*migrationConfig, error) {
 	if !c.isMigrationRequested() {
 		logger.Debug("no OSD migration is requested")

pkg/operator/ceph/cluster/osd/osd_test.go

@@ -18,6 +18,7 @@ package osd
 
 import (
 	"context"
+	"strings"
 	"testing"
 
 	"github.com/pkg/errors"
@@ -343,7 +344,11 @@ func TestAddRemoveNode(t *testing.T) {
 				return `{}`, nil
 			}
 			if args[0] == "auth" && args[1] == "del" {
-				assert.Equal(t, "osd.1", args[2])
+				if strings.HasPrefix(args[2], "osd.") {
+					assert.Equal(t, "osd.1", args[2])
+				} else if strings.HasPrefix(args[2], "client.") {
+					assert.Equal(t, "client.bootstrap-osd", args[2])
+				}
 				return "", nil
 			}
 			return "", errors.Errorf("unexpected ceph command %q", args)