diff --git a/pkg/castled/osd.go b/pkg/castled/osd.go index fd5fddb4d..b0b55777d 100644 --- a/pkg/castled/osd.go +++ b/pkg/castled/osd.go @@ -233,6 +233,22 @@ func mountOSD(device string, mountPath string, executor util.Executor) error { return nil } +func registerOSDWithCluster(device string, bootstrapConn *cephd.Conn) (int, uuid.UUID, error) { + osdUUID, err := uuid.NewRandom() + if err != nil { + return 0, uuid.UUID{}, fmt.Errorf("failed to generate UUID for %s: %+v", device, err) + } + + // create the OSD instance via a mon_command, this assigns a cluster wide ID to the OSD + osdID, err := createOSD(bootstrapConn, osdUUID) + if err != nil { + return 0, uuid.UUID{}, err + } + + log.Printf("successfully created OSD %s with ID %d for %s", osdUUID.String(), osdID, device) + return osdID, osdUUID, nil +} + // looks for an existing OSD data path under the given root func findOSDDataPath(osdRoot, clusterName string) (string, error) { var osdDataPath string @@ -281,63 +297,49 @@ func getOSDInfo(osdDataPath string) (int, uuid.UUID, error) { return osdID, osdUUID, nil } -func initializeOSD(osdDataDir string, bootstrapConn *cephd.Conn, cluster *clusterInfo) (string, int, uuid.UUID, error) { - log.Printf("initializing the osd directory %s", osdDataDir) - osdUUID, err := uuid.NewRandom() - if err != nil { - return "", 0, uuid.UUID{}, fmt.Errorf("failed to generate UUID for %s: %+v", osdDataDir, err) - } - - // create the OSD instance via a mon_command, this assigns a cluster wide ID to the OSD - osdID, err := createOSD(bootstrapConn, osdUUID) - if err != nil { - return "", 0, uuid.UUID{}, err - } - - log.Printf("successfully created OSD %s with ID %d at %s", osdUUID.String(), osdID, osdDataDir) - +func initializeOSD(osdDataDir string, osdID int, osdUUID uuid.UUID, bootstrapConn *cephd.Conn, cluster *clusterInfo) (string, error) { // ensure that the OSD data directory is created osdDataPath := filepath.Join(osdDataDir, fmt.Sprintf("%s-%d", cluster.Name, osdID)) if err := os.MkdirAll(osdDataPath, 0777); err != nil { - return "", 0, uuid.UUID{}, fmt.Errorf("failed to create OSD data dir at %s, %+v", osdDataPath, err) + return "", fmt.Errorf("failed to create OSD data dir at %s, %+v", osdDataPath, err) } // write the OSD config file to disk keyringPath := getOSDKeyringPath(osdDataPath) - _, err = generateConfigFile(cluster, osdDataPath, fmt.Sprintf("osd.%d", osdID), keyringPath) + _, err := generateConfigFile(cluster, osdDataPath, fmt.Sprintf("osd.%d", osdID), keyringPath) if err != nil { - return "", 0, uuid.UUID{}, fmt.Errorf("failed to write OSD %d config file: %+v", osdID, err) + return "", fmt.Errorf("failed to write OSD %d config file: %+v", osdID, err) } // get the current monmap, it will be needed for creating the OSD file system monMapRaw, err := getMonMap(bootstrapConn) if err != nil { - return "", 0, uuid.UUID{}, fmt.Errorf("failed to get mon map: %+v", err) + return "", fmt.Errorf("failed to get mon map: %+v", err) } // create/initalize the OSD file system and journal if err := createOSDFileSystem(cluster.Name, osdID, osdUUID, osdDataPath, monMapRaw); err != nil { - return "", 0, uuid.UUID{}, err + return "", err } // add auth privileges for the OSD, the bootstrap-osd privileges were very limited if err := addOSDAuth(bootstrapConn, osdID, osdDataPath); err != nil { - return "", 0, uuid.UUID{}, err + return "", err } // open a connection to the cluster using the OSDs creds osdConn, err := connectToCluster(cluster, osdDataDir, fmt.Sprintf("osd.%d", osdID), keyringPath) if err != nil { - return "", 0, uuid.UUID{}, err + return "", err } defer osdConn.Shutdown() // add the new OSD to the cluster crush map if err := addOSDToCrushMap(osdConn, osdID, osdDataDir); err != nil { - return "", 0, uuid.UUID{}, err + return "", err } - return osdDataPath, osdID, osdUUID, nil + return osdDataPath, nil } // creates the OSD identity in the cluster via a mon_command diff --git a/pkg/castled/osdagent.go b/pkg/castled/osdagent.go index 83c1ffc78..b93e79a8a 100644 --- a/pkg/castled/osdagent.go +++ b/pkg/castled/osdagent.go @@ -80,7 +80,10 @@ func (a *osdAgent) createOSDs(adminConn *cephd.Conn, context *orchestrator.Clust defer bootstrapConn.Shutdown() // initialize all the desired OSD volumes - for i, device := range a.devices { + for _, device := range a.devices { + var osdID int + var osdUUID uuid.UUID + mountPoint, err := getDeviceMountPoint(device, context.Executor) if err != nil { return fmt.Errorf("unable to get mount point for device %s: %+v", device, err) @@ -93,7 +96,14 @@ func (a *osdAgent) createOSDs(adminConn *cephd.Conn, context *orchestrator.Clust return err } - mountPoint = fmt.Sprintf("/tmp/osd%d", i) + // register the OSD with the cluster now to get an official ID + osdID, osdUUID, err = registerOSDWithCluster(device, bootstrapConn) + if err != nil { + + } + + // mount the device using a mount point that reflects the OSD's ID + mountPoint = fmt.Sprintf("/tmp/osd%d", osdID) if err := mountOSD(device, mountPoint, context.Executor); err != nil { return err } @@ -107,12 +117,9 @@ func (a *osdAgent) createOSDs(adminConn *cephd.Conn, context *orchestrator.Clust return err } - var osdID int - var osdUUID uuid.UUID - if _, err := os.Stat(filepath.Join(osdDataPath, "whoami")); os.IsNotExist(err) { // osd_data_dir/whoami does not exist yet, create/initialize the OSD - osdDataPath, osdID, osdUUID, err = initializeOSD(osdDataDir, bootstrapConn, a.cluster) + osdDataPath, err = initializeOSD(osdDataDir, osdID, osdUUID, bootstrapConn, a.cluster) if err != nil { return fmt.Errorf("failed to initialize OSD at %s: %+v", osdDataDir, err) }