CAS-1262 bootstrap OSDs and get them running

This commit is contained in:
Jared Watts committed 2016-08-13 23:04:48 -07:00
1 parent 61a1d7390a
commit ff8af3aed6
5 files changed
+427 -133

No files matched your search

Generated
+4 -2
View File
@@ -1,5 +1,5 @@
hash: d8e1942658abe6166ea9f306c3caa1a79a4420b9ea68c8f9acdc3f96476bd0f1
updated: 2016-08-09T17:19:26.234852718-07:00
hash: 694ee1a5d914d2e03ac46bec40a76e3900cc18ef6a9a71c3a46906c966fda86f
updated: 2016-08-11T20:36:48.20533737-07:00
imports:
- name: github.com/coreos/etcd
version: d53923c636e0e4ab7f00cb75681b97a8f11f5a9d
@@ -10,6 +10,8 @@ imports:
- pkg/types
- name: github.com/inconshreveable/mousetrap
version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
- name: github.com/nu7hatch/gouuid
version: 179d4d0c4d8d407a32af483c2354df1d2c91e6c3
- name: github.com/spf13/cobra
version: dbb7c2d02e284aa24423e3e6e105969dffe6e4d6
subpackages:
+1
View File
@@ -8,3 +8,4 @@ import:
- client
- error
- package: github.com/spf13/pflag
- package: github.com/nu7hatch/gouuid
+414 -125
View File
@@ -8,16 +8,21 @@ import (
"log"
"os"
"os/exec"
"os/user"
"path"
"path/filepath"
"regexp"
"strconv"
"syscall"
"time"
etcd "github.com/coreos/etcd/client"
"github.com/nu7hatch/gouuid"
"golang.org/x/net/context"
"github.com/quantum/castle/pkg/cephd"
"github.com/quantum/castle/pkg/kvstore"
"github.com/quantum/castle/pkg/proc"
etcd "github.com/coreos/etcd/client"
"golang.org/x/net/context"
)
const (
@@ -48,7 +53,6 @@ const (
[client.bootstrap-osd]
keyring=%s
`
monitorConfigTemplate = `
[mon.%s]
name = %s
@@ -58,6 +62,10 @@ const (
[client.bootstrap-osd]
key = %s
caps mon = "allow profile bootstrap-osd"
`
osdClientConfigTemplate = `
[osd.%d]
keyring=%s
`
)
@@ -145,6 +153,7 @@ func Bootstrap(cfg Config, executor proc.Executor) ([]*exec.Cmd, error) {
return procs, nil
}
// attempt to load any previously created and saved cluster info
func loadClusterInfo(etcdClient etcd.KeysAPI) (clusterInfo, error) {
resp, err := etcdClient.Get(context.Background(), path.Join(cephKey, "fsid"), nil)
if err != nil {
@@ -184,6 +193,7 @@ func isClusterInfoSet(c clusterInfo) bool {
return c.FSID != "" && c.MonitorSecret != "" && c.AdminSecret != ""
}
// create new cluster info (FSID, shared keys)
func createClusterInfo() (clusterInfo, error) {
fsid, err := cephd.NewFsid()
if err != nil {
@@ -207,6 +217,7 @@ func createClusterInfo() (clusterInfo, error) {
}, nil
}
// save the given cluster info to the key value store
func saveClusterInfo(c clusterInfo, etcdClient etcd.KeysAPI) error {
_, err := etcdClient.Set(context.Background(), path.Join(cephKey, "fsid"), c.FSID, nil)
if err != nil {
@@ -228,6 +239,7 @@ func saveClusterInfo(c clusterInfo, etcdClient etcd.KeysAPI) error {
return nil
}
// opens a connection to the cluster that can be used for management operations
func connectToCluster(clusterName, user, confFilePath string) (*cephd.Conn, error) {
log.Printf("connecting to ceph cluster %s with user %s", clusterName, user)
@@ -247,6 +259,7 @@ func connectToCluster(clusterName, user, confFilePath string) (*cephd.Conn, erro
return conn, nil
}
// register the names and endpoints of all monitors on this machine
func registerMonitors(cfg Config, etcdClient etcd.KeysAPI) error {
port := 6790
for i, monName := range cfg.MonNames {
@@ -268,6 +281,7 @@ func getMonitorEndpointKey(name string) string {
return fmt.Sprintf(path.Join(cephKey, "mons/%s/endpoint"), name)
}
// wait for all expected initial monitors to register (report their names/endpoints)
func waitForMonitorRegistration(cfg Config, etcdClient etcd.KeysAPI) error {
for i := range cfg.InitialMonitors {
monName := cfg.InitialMonitors[i].Name
@@ -301,6 +315,8 @@ func waitForMonitorRegistration(cfg Config, etcdClient etcd.KeysAPI) error {
return nil
}
// represents the response from a mon_status mon_command (subset of all available fields, only
// marshal ones we care about)
type MonStatusResponse struct {
Quorum []int `json:"quorum"`
MonMap struct {
@@ -308,12 +324,14 @@ type MonStatusResponse struct {
} `json:"monmap"`
}
// represents an entry in the monitor map
type MonMapEntry struct {
Name string `json:"name"`
Rank int `json:"rank"`
Address string `json:"addr"`
}
// waits for all expected initial monitors to establish and join quorum
func waitForMonitorQuorum(adminConn *cephd.Conn, cfg Config) error {
retryCount := 0
retryMax := 20
@@ -357,6 +375,7 @@ func waitForMonitorQuorum(adminConn *cephd.Conn, cfg Config) error {
}
// using the current initial monitor's mon map entry, check to see if it's in the quorum list
// (a list of monitor rank values)
inQuorumList := false
for _, q := range monStatusResp.Quorum {
if monMapEntry.Rank == q {
@@ -382,6 +401,7 @@ func waitForMonitorQuorum(adminConn *cephd.Conn, cfg Config) error {
return nil
}
// calls mon_status mon_command
func getMonStatus(adminConn *cephd.Conn) (MonStatusResponse, error) {
monCommand := "mon_status"
command, err := json.Marshal(map[string]string{"prefix": monCommand, "format": "json"})
@@ -401,35 +421,34 @@ func getMonStatus(adminConn *cephd.Conn) (MonStatusResponse, error) {
return resp, nil
}
// creates and initializes the given monitors file systems
func makeMonitorFileSystems(cfg Config, c clusterInfo) error {
for _, monName := range cfg.MonNames {
// write the keyring to disk
keyringPath, err := writeMonitorKeyring(monName, c)
if err != nil {
if err := writeMonitorKeyring(monName, c); err != nil {
return err
}
// write the config file to disk
monConfFilePath, err := writeMonitorConfigFile(monName, cfg, c, keyringPath)
if err != nil {
if err := writeMonitorConfigFile(monName, cfg, c, getMonKeyringPath(monName)); err != nil {
return err
}
// create monitor data dir
monDataDir := fmt.Sprintf("/tmp/%s/mon.%s", monName, monName)
monDataDir := getMonDataDirPath(monName)
if err := os.MkdirAll(filepath.Dir(monDataDir), 0744); err != nil {
fmt.Printf("failed to create monitor data directory at %s: %+v", monDataDir, err)
}
// call mon --mkfs in a child process
err = proc.RunChildProcess(
err := proc.RunChildProcess(
"mon",
"--mkfs",
fmt.Sprintf("--cluster=%s", cfg.ClusterName),
fmt.Sprintf("--name=mon.%s", monName),
fmt.Sprintf("--mon-data=%s", monDataDir),
fmt.Sprintf("--conf=%s", monConfFilePath),
fmt.Sprintf("--keyring=%s", keyringPath))
fmt.Sprintf("--conf=%s", getMonConfFilePath(monName)),
fmt.Sprintf("--keyring=%s", getMonKeyringPath(monName)))
if err != nil {
return fmt.Errorf("failed mon %s --mkfs: %+v", monName, err)
}
@@ -438,51 +457,51 @@ func makeMonitorFileSystems(cfg Config, c clusterInfo) error {
return nil
}
func writeMonitorKeyring(monName string, c clusterInfo) (string, error) {
// writes the monitor keyring to disk
func writeMonitorKeyring(monName string, c clusterInfo) error {
keyring := fmt.Sprintf(monitorKeyringTemplate, c.MonitorSecret, c.AdminSecret)
keyringPath := fmt.Sprintf("/tmp/%s/tmp_keyring", monName)
keyringPath := getMonKeyringPath(monName)
if err := os.MkdirAll(filepath.Dir(keyringPath), 0744); err != nil {
return "", fmt.Errorf("failed to create keyring directory for %s: %+v", keyringPath, err)
return fmt.Errorf("failed to create keyring directory for %s: %+v", keyringPath, err)
}
if err := ioutil.WriteFile(keyringPath, []byte(keyring), 0644); err != nil {
return "", fmt.Errorf("failed to write monitor keyring to %s: %+v", keyringPath, err)
return fmt.Errorf("failed to write monitor keyring to %s: %+v", keyringPath, err)
}
return keyringPath, nil
return nil
}
func writeMonitorConfigFile(monName string, cfg Config, c clusterInfo, adminKeyringPath string) (string, error) {
// generates and writes the monitor config file to disk
func writeMonitorConfigFile(monName string, cfg Config, c clusterInfo, adminKeyringPath string) error {
var contentBuffer bytes.Buffer
if err := writeGlobalConfigFileSection(&contentBuffer, cfg, c, getMonRunDirPath(monName)); err != nil {
return "", fmt.Errorf("failed to write mon %s global config section, %+v", monName, err)
return fmt.Errorf("failed to write mon %s global config section, %+v", monName, err)
}
_, err := contentBuffer.WriteString(fmt.Sprintf(adminClientConfigTemplate, adminKeyringPath))
if err != nil {
return "", fmt.Errorf("failed to write mon %s admin client config section, %+v", monName, err)
return fmt.Errorf("failed to write mon %s admin client config section, %+v", monName, err)
}
if err := writeInitialMonitorsConfigFileSections(&contentBuffer, cfg); err != nil {
return "", fmt.Errorf("failed to write mon %s initial monitor config sections, %+v", monName, err)
return fmt.Errorf("failed to write mon %s initial monitor config sections, %+v", monName, err)
}
// write the entire config to disk
monConfFilePath := getMonConfFilePath(monName)
if err := writeFile(monConfFilePath, contentBuffer); err != nil {
return "", err
if err := writeFile(getMonConfFilePath(monName), contentBuffer); err != nil {
return err
}
return monConfFilePath, nil
return nil
}
// runs all the given monitors in child processes
func runMonitors(cfg Config) ([]*exec.Cmd, error) {
procs := make([]*exec.Cmd, len(cfg.MonNames))
for i, monName := range cfg.MonNames {
monConfFilePath := getMonConfFilePath(monName)
monDataDir := fmt.Sprintf("/tmp/%s/mon.%s", monName, monName)
// find the current monitor's endpoint
var monEndpoint string
for i := range cfg.InitialMonitors {
if cfg.InitialMonitors[i].Name == monName {
@@ -501,8 +520,8 @@ func runMonitors(cfg Config) ([]*exec.Cmd, error) {
"--foreground",
fmt.Sprintf("--cluster=%v", cfg.ClusterName),
fmt.Sprintf("--name=mon.%v", monName),
fmt.Sprintf("--mon-data=%s", monDataDir),
fmt.Sprintf("--conf=%s", monConfFilePath),
fmt.Sprintf("--mon-data=%s", getMonDataDirPath(monName)),
fmt.Sprintf("--conf=%s", getMonConfFilePath(monName)),
fmt.Sprintf("--public-addr=%v", monEndpoint))
if err != nil {
return nil, fmt.Errorf("failed to start monitor %s: %+v", monName, err)
@@ -514,127 +533,141 @@ func runMonitors(cfg Config) ([]*exec.Cmd, error) {
return procs, nil
}
// get the path of a given monitor's run dir
func getMonRunDirPath(monName string) string {
return fmt.Sprintf("/tmp/%s", monName)
}
// get the path of a given monitor's config file
func getMonConfFilePath(monName string) string {
return filepath.Join(getMonRunDirPath(monName), "tmp_config")
return filepath.Join(getMonRunDirPath(monName), "config")
}
// get the path of a given monitor's keyring
func getMonKeyringPath(monName string) string {
return filepath.Join(getMonRunDirPath(monName), "keyring")
}
// get the path of a given monitor's data dir
func getMonDataDirPath(monName string) string {
return filepath.Join(getMonRunDirPath(monName), fmt.Sprintf("mon.%s", monName))
}
// create and initalize OSDs for all the devices specified in the given config
func createOSDs(adminConn *cephd.Conn, cfg Config, c clusterInfo, executor proc.Executor) ([]*exec.Cmd, error) {
// generate and write the OSD bootstrap keyring
if err := createOSDBootstrapKeyring(adminConn, cfg.ClusterName, executor); err != nil {
return nil, err
}
bootstrapOSDConfFilePath, err := writeBootstrapOSDConfFile(cfg, c, getBootstrapOSDKeyringPath(cfg.ClusterName))
if err != nil {
// write the bootstrap OSD config file to disk
bootstrapOSDConfFilePath := getBootstrapOSDConfFilePath()
if err := writeBootstrapOSDConfFile(cfg, c, getBootstrapOSDKeyringPath(cfg.ClusterName)); err != nil {
return nil, fmt.Errorf("failed to write bootstrap-osd conf file at %s: %+v", bootstrapOSDConfFilePath, err)
}
// connect to the cluster using the bootstrap-osd creds, this connection will be used for config operations
bootstrapConn, err := connectToCluster(cfg.ClusterName, "client.bootstrap-osd", bootstrapOSDConfFilePath)
if err != nil {
return nil, err
}
defer bootstrapConn.Shutdown()
osdProcs := []*exec.Cmd{}
// initialize all the desired OSD volumes
for i, device := range cfg.Devices {
done, err := formatOSD(device, cfg.ForceFormat, executor)
if err != nil {
// attempting to format the volume failed, bail out with error
return nil, err
return osdProcs, err
} else if !done {
// the formatting was not done, probably because the drive already has a filesystem.
// just move on to the next OSD.
continue
}
// TODO: this OSD data dir isn't consistent across multiple runs
osdDataDir := fmt.Sprintf("/tmp/osd%d", i)
if err := mountOSD(device, osdDataDir, executor); err != nil {
return nil, err
return osdProcs, err
}
/*
// find the OSD data dir
osdDataPath, err := findOSDDataPath(osdDataDir, cfg.ClusterName)
if err != nil {
return osdProcs, err
}
// find the OSD data dir
osdDataPath, err := findOSDDataPath(osd.DataDir, ceph.ClusterName)
if _, err := os.Stat(filepath.Join(osdDataPath, "whoami")); os.IsNotExist(err) {
// osd_data_dir/whoami does not exist yet, create/initialize the OSD
log.Printf("initializing the osd directory %s", osdDataDir)
osdUUID, err := uuid.NewV4()
if err != nil {
return err
return osdProcs, fmt.Errorf("failed to generate UUID for %s: %+v", osdDataDir, err)
}
if _, err := os.Stat(filepath.Join(osdDataPath, "whoami")); os.IsNotExist(err) {
// osd_data_dir/whoami does not exist yet
log.Printf("initializing the osd directory %s", osd.DataDir)
osdUUID, err := uuid.NewV4()
if err != nil {
return fmt.Errorf("failed to generate UUID for %s: %+v", osd.DataDir, err)
}
osdID, err := createOSD(bootstrapConn, osdUUID)
if err != nil {
return err
}
log.Printf("successfully created OSD %s with ID %d at %s", osdUUID.String(), osdID, osd.DataDir)
if err := addOSDKeyringToConf(ceph.ClusterName, osdID); err != nil {
return err
}
osdDataPath = filepath.Join(osd.DataDir, fmt.Sprintf("%s-%d", ceph.ClusterName, osdID))
if err := os.MkdirAll(osdDataPath, 0777); err != nil {
return fmt.Errorf("failed to create OSD data dir at %s, %+v", osdDataPath, err)
}
if err := chownCephPath(osdDataPath, executor); err != nil {
return err
}
// create a link from the default location of the OSD directory to our mounted volume
osdDefaultPath := filepath.Join(string(filepath.Separator), "var", "lib", "ceph", "osd", fmt.Sprintf("%s-%d", ceph.ClusterName, osdID))
if err := os.Symlink(osdDataPath, osdDefaultPath); err != nil {
return fmt.Errorf("failed to link OSD default path %s to %s: %+v", osdDefaultPath, osdDataPath, err)
}
monMapRaw, err := getMonMap(bootstrapConn)
if err != nil {
return fmt.Errorf("failed to get mon map: %+v", err)
}
if err := createOSDFileSystem(ceph, osdID, osdUUID, osdDefaultPath, monMapRaw, executor); err != nil {
return err
}
if err := addOSDAuth(bootstrapConn, osdID, osdDefaultPath); err != nil {
return err
}
// open a connection using the OSDs creds
osdConn, err := connectToCluster(ceph.ClusterName, fmt.Sprintf("osd.%d", osdID))
if err != nil {
return err
}
defer osdConn.Shutdown()
if err := addOSDToCrushMap(osdConn, osdID, osd.DataDir); err != nil {
return err
}
if err := chownCephPath(osdDefaultPath+string(filepath.Separator), executor); err != nil {
return err
}
// everything should be configured for the OSD, process its units now
osdUnits := OSDUnits(ceph, osdID)
if err := ProcessUnits(osdUnits, um); err != nil {
return err
}
// create the OSD instance via a mon_command, this assigns a cluster wide ID to the OSD
osdID, err := createOSD(bootstrapConn, osdUUID)
if err != nil {
return osdProcs, err
}
*/
log.Printf("successfully created OSD %s with ID %d at %s", osdUUID.String(), osdID, osdDataDir)
// ensure that the OSD data directory is created
osdDataPath = filepath.Join(osdDataDir, fmt.Sprintf("%s-%d", cfg.ClusterName, osdID))
if err := os.MkdirAll(osdDataPath, 0777); err != nil {
return osdProcs, fmt.Errorf("failed to create OSD data dir at %s, %+v", osdDataPath, err)
}
// write the OSD config file to disk
if err := writeOSDConfFile(cfg, c, osdDataPath, osdID); err != nil {
return osdProcs, fmt.Errorf("failed to write OSD %d config file: %+v", osdID, err)
}
// get the current monmap, it will be needed for creating the OSD file system
monMapRaw, err := getMonMap(bootstrapConn)
if err != nil {
return osdProcs, fmt.Errorf("failed to get mon map: %+v", err)
}
// create/initalize the OSD file system and journal
if err := createOSDFileSystem(cfg.ClusterName, osdID, osdUUID, osdDataPath, monMapRaw); err != nil {
return osdProcs, err
}
// add auth privileges for the OSD, the bootstrap-osd privileges were very limited
if err := addOSDAuth(bootstrapConn, osdID, osdDataPath); err != nil {
return osdProcs, err
}
// open a connection to the cluster using the OSDs creds
osdConn, err := connectToCluster(cfg.ClusterName, fmt.Sprintf("osd.%d", osdID), getOSDConfFilePath(osdDataPath))
if err != nil {
return osdProcs, err
}
defer osdConn.Shutdown()
// add the new OSD to the cluster crush map
if err := addOSDToCrushMap(osdConn, osdID, osdDataDir); err != nil {
return osdProcs, err
}
// run the OSD in a child process now that it is fully initialized and ready to go
osdProc, err := runOSD(cfg.ClusterName, osdID, osdUUID, osdDataPath)
if err != nil {
return osdProcs, fmt.Errorf("failed to run osd %d: %+v", osdID, err)
}
osdProcs = append(osdProcs, osdProc)
}
}
return nil, nil
return osdProcs, nil
}
// create a keyring for the bootstrap-osd client, it gets a limited set of privileges
func createOSDBootstrapKeyring(conn *cephd.Conn, clusterName string, executor proc.Executor) error {
bootstrapOSDKeyringPath := getBootstrapOSDKeyringPath(clusterName)
if _, err := os.Stat(bootstrapOSDKeyringPath); os.IsNotExist(err) {
@@ -661,7 +694,7 @@ func createOSDBootstrapKeyring(conn *cephd.Conn, clusterName string, executor pr
bootstrapOSDKey := resp["key"].(string)
log.Printf("succeeded %s command, bootstrapOSDKey: %s", cmd, bootstrapOSDKey)
// create the bootstrap-osd keyring
// write the bootstrap-osd keyring to disk
bootstrapOSDKeyringDir := filepath.Dir(bootstrapOSDKeyringPath)
if err := os.MkdirAll(bootstrapOSDKeyringDir, 0744); err != nil {
fmt.Printf("failed to create bootstrap OSD keyring dir at %s: %+v", bootstrapOSDKeyringDir, err)
@@ -675,43 +708,33 @@ func createOSDBootstrapKeyring(conn *cephd.Conn, clusterName string, executor pr
return nil
}
func getBootstrapOSDDir() string {
return "/tmp/bootstrap-osd"
}
func getBootstrapOSDKeyringPath(clusterName string) string {
return fmt.Sprintf("%s/%s", getBootstrapOSDDir(), fmt.Sprintf("%s.keyring", clusterName))
}
func getBootstrapOSDConfFilePath() string {
return fmt.Sprintf("%s/tmp_config", getBootstrapOSDDir())
}
func writeBootstrapOSDConfFile(cfg Config, c clusterInfo, bootstrapOSDKeyringPath string) (string, error) {
// write the bootstrap-osd config file to disk
func writeBootstrapOSDConfFile(cfg Config, c clusterInfo, bootstrapOSDKeyringPath string) error {
var contentBuffer bytes.Buffer
bootstrapOSDConfFilePath := getBootstrapOSDConfFilePath()
if err := writeGlobalConfigFileSection(&contentBuffer, cfg, c, getBootstrapOSDDir()); err != nil {
return bootstrapOSDConfFilePath, fmt.Errorf("failed to write bootstrap-osd global config section, %+v", err)
return fmt.Errorf("failed to write bootstrap-osd global config section, %+v", err)
}
_, err := contentBuffer.WriteString(fmt.Sprintf(bootstrapOSDClientConfigTemplate, bootstrapOSDKeyringPath))
if err != nil {
return bootstrapOSDConfFilePath, fmt.Errorf("failed to write bootstrap-osd client config section, %+v", err)
return fmt.Errorf("failed to write bootstrap-osd client config section, %+v", err)
}
if err := writeInitialMonitorsConfigFileSections(&contentBuffer, cfg); err != nil {
return bootstrapOSDConfFilePath, fmt.Errorf("failed to write bootstrap-osd initial monitor config sections, %+v", err)
return fmt.Errorf("failed to write bootstrap-osd initial monitor config sections, %+v", err)
}
// write the entire config to disk
if err := writeFile(bootstrapOSDConfFilePath, contentBuffer); err != nil {
return bootstrapOSDConfFilePath, err
return err
}
return bootstrapOSDConfFilePath, nil
return nil
}
// format the given device for usage by an OSD
func formatOSD(device string, forceFormat bool, executor proc.Executor) (bool, error) {
// format the current volume
cmd := fmt.Sprintf("blkid %s", device)
@@ -722,16 +745,19 @@ func formatOSD(device string, forceFormat bool, executor proc.Executor) (bool, e
return false, fmt.Errorf("command %s failed: %+v", cmd, err)
}
if devFS != "" && forceFormat {
// there's a filesystem on the device, but the user has specified to force a format. give a warning about that.
log.Printf("WARNING: device %s already formatted with %s, but forcing a format!!!", device, devFS)
}
if devFS == "" || forceFormat {
// execute the format operation
cmd = fmt.Sprintf("format %s", device)
err = executor.ExecuteCommand(cmd, "sudo", "/usr/sbin/mkfs.btrfs", "-f", "-m", "single", "-n", "32768", fmt.Sprintf("/dev/%s", device))
if err != nil {
return false, fmt.Errorf("command %s failed: %+v", cmd, err)
}
} else {
// disk is already formatted and the user doesn't want to force it, return no error, but also specify that no format was done
log.Printf("device %s already formatted with %s, cannot use for OSD", device, devFS)
return false, nil
}
@@ -739,6 +765,7 @@ func formatOSD(device string, forceFormat bool, executor proc.Executor) (bool, e
return true, nil
}
// mount the OSD data directory onto the given device
func mountOSD(device string, mountPath string, executor proc.Executor) error {
cmd := fmt.Sprintf("lsblk %s", device)
var diskUUID string
@@ -747,6 +774,8 @@ func mountOSD(device string, mountPath string, executor proc.Executor) error {
retryMax := 10
sleepTime := 2
for {
// there is lag in between when a filesytem is created and its UUID is available. retry as needed
// until we have a usable UUID for the newly formatted filesystem.
var err error
diskUUID, err = executor.ExecuteCommandWithOutput(cmd, "lsblk", fmt.Sprintf("/dev/%s", device), "-d", "-n", "-r", "-o", "UUID")
if err != nil {
@@ -775,5 +804,265 @@ func mountOSD(device string, mountPath string, executor proc.Executor) error {
return fmt.Errorf("command %s failed: %+v", cmd, err)
}
// chown for the current user since we had to format and mount with sudo
currentUser, err := user.Current()
if err != nil {
log.Printf("unable to find current user: %+v", err)
} else {
cmd = fmt.Sprintf("chown %s", mountPath)
if err := executor.ExecuteCommand(cmd, "sudo", "chown", "-R",
fmt.Sprintf("%s:%s", currentUser.Username, currentUser.Username), mountPath); err != nil {
log.Printf("command %s failed: %+v", cmd, err)
}
}
return nil
}
// looks for an existing OSD data path under the given root
func findOSDDataPath(osdRoot, clusterName string) (string, error) {
var osdDataPath string
fl, err := ioutil.ReadDir(osdRoot)
if err != nil {
return "", fmt.Errorf("failed to read dir %s: %+v", osdRoot, err)
}
p := fmt.Sprintf(`%s-[A-Za-z0-9._-]+`, clusterName)
for _, f := range fl {
if f.IsDir() {
matched, err := regexp.MatchString(p, f.Name())
if err == nil && matched {
osdDataPath = filepath.Join(osdRoot, f.Name())
break
}
}
}
return osdDataPath, nil
}
// creates the OSD identity in the cluster via a mon_command
func createOSD(bootstrapConn *cephd.Conn, osdUUID *uuid.UUID) (int, error) {
cmd := "osd create"
command, err := json.Marshal(map[string]interface{}{
"prefix": cmd,
"format": "json",
"entity": "client.bootstrap-osd",
"uuid": osdUUID.String(),
})
if err != nil {
return 0, fmt.Errorf("command %s marshall failed: %+v", cmd, err)
}
buf, _, err := bootstrapConn.MonCommand(command)
if err != nil {
return 0, fmt.Errorf("mon_command %s failed: %+v", cmd, err)
}
var resp map[string]interface{}
err = json.Unmarshal(buf, &resp)
if err != nil {
return 0, fmt.Errorf("failed to unmarshall %s response: %+v. raw response: '%s'", cmd, err, string(buf[:]))
}
return int(resp["osdid"].(float64)), nil
}
// writes a config file to disk for the given OSD and config
func writeOSDConfFile(cfg Config, c clusterInfo, osdDataPath string, osdID int) error {
var contentBuffer bytes.Buffer
osdConfFilePath := getOSDConfFilePath(osdDataPath)
if err := writeGlobalConfigFileSection(&contentBuffer, cfg, c, osdDataPath); err != nil {
return fmt.Errorf("failed to write osd %d global config section, %+v", osdID, err)
}
_, err := contentBuffer.WriteString(fmt.Sprintf(osdClientConfigTemplate, osdID, getOSDKeyringPath(osdDataPath)))
if err != nil {
return fmt.Errorf("failed to write osd %d config section, %+v", osdID, err)
}
if err := writeInitialMonitorsConfigFileSections(&contentBuffer, cfg); err != nil {
return fmt.Errorf("failed to write osd %d initial monitor config sections, %+v", osdID, err)
}
// write the entire config to disk
if err := writeFile(osdConfFilePath, contentBuffer); err != nil {
return err
}
return nil
}
// gets the current mon map for the cluster
func getMonMap(bootstrapConn *cephd.Conn) ([]byte, error) {
cmd := "mon getmap"
command, err := json.Marshal(map[string]interface{}{
"prefix": cmd,
"format": "json",
"entity": "client.bootstrap-osd",
})
if err != nil {
return nil, fmt.Errorf("command %s marshall failed: %+v", cmd, err)
}
buf, _, err := bootstrapConn.MonCommand(command)
if err != nil {
return nil, fmt.Errorf("mon_command %s failed: %+v", cmd, err)
}
return buf, nil
}
// creates/initalizes the OSD filesystem and journal via a child process
func createOSDFileSystem(clusterName string, osdID int, osdUUID *uuid.UUID, osdDataPath string, monMap []byte) error {
log.Printf("Initializing OSD %d file system at %s...", osdID, osdDataPath)
// the current monmap is needed to create the OSD, save it to a temp location so it is accessible
monMapTmpPath := getOSDTempMonMapPath(osdDataPath)
monMapTmpDir := filepath.Dir(monMapTmpPath)
if err := os.MkdirAll(monMapTmpDir, 0744); err != nil {
return fmt.Errorf("failed to create monmap tmp file directory at %s: %+v", monMapTmpDir, err)
}
if err := ioutil.WriteFile(monMapTmpPath, monMap, 0644); err != nil {
return fmt.Errorf("failed to write mon map to tmp file %s, %+v", monMapTmpPath, err)
}
// create the OSD file system and journal
err := proc.RunChildProcess(
"osd",
"--mkfs",
"--mkkey",
fmt.Sprintf("--id=%s", strconv.Itoa(osdID)),
fmt.Sprintf("--cluster=%s", clusterName),
fmt.Sprintf("--osd-data=%s", osdDataPath),
fmt.Sprintf("--osd-journal=%s", getOSDJournalPath(osdDataPath)),
fmt.Sprintf("--conf=%s", getOSDConfFilePath(osdDataPath)),
fmt.Sprintf("--keyring=%s", getOSDKeyringPath(osdDataPath)),
fmt.Sprintf("--osd-uuid=%s", osdUUID.String()),
fmt.Sprintf("--monmap=%s", monMapTmpPath))
if err != nil {
return fmt.Errorf("failed osd mkfs for OSD ID %d, UUID %s, dataDir %s: %+v",
osdID, osdUUID.String(), osdDataPath, err)
}
return nil
}
// add OSD auth privileges for the given OSD ID. the bootstrap-osd privileges are limited and a real OSD needs more.
func addOSDAuth(bootstrapConn *cephd.Conn, osdID int, osdDataPath string) error {
// create a new auth for this OSD
osdKeyringPath := getOSDKeyringPath(osdDataPath)
keyringBuffer, err := ioutil.ReadFile(osdKeyringPath)
if err != nil {
return fmt.Errorf("failed to read OSD keyring at %s, %+v", osdKeyringPath, err)
}
cmd := "auth add"
osdEntity := fmt.Sprintf("osd.%d", osdID)
command, err := json.Marshal(map[string]interface{}{
"prefix": cmd,
"format": "json",
"entity": osdEntity,
"caps": []string{"osd", "allow *", "mon", "allow profile osd"},
})
if err != nil {
return fmt.Errorf("command %s marshall failed: %+v", cmd, err)
}
_, info, err := bootstrapConn.MonCommandWithInputBuffer(command, keyringBuffer)
if err != nil {
return fmt.Errorf("mon_command %s failed: %+v", cmd, err)
}
log.Printf("succeeded %s command for %s. info: %s", cmd, osdEntity, info)
return nil
}
// adds the given OSD to the crush map
func addOSDToCrushMap(osdConn *cephd.Conn, osdID int, osdDataPath string) error {
// get the size of the volume containing the OSD data dir
s := syscall.Statfs_t{}
if err := syscall.Statfs(osdDataPath, &s); err != nil {
return fmt.Errorf("failed to statfs on %s, %+v", osdDataPath, err)
}
all := s.Blocks * uint64(s.Bsize)
// weight is ratio of (size in KB) / (1 GB)
weight := float64(all/1024) / 1073741824.0
weight, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", weight), 64)
osdEntity := fmt.Sprintf("osd.%d", osdID)
log.Printf("OSD %s at %s, bytes: %d, weight: %.2f", osdEntity, osdDataPath, all, weight)
hostName, err := os.Hostname()
if err != nil {
return fmt.Errorf("failed to get hostname, %+v", err)
}
cmd := "osd crush create-or-move"
command, err := json.Marshal(map[string]interface{}{
"prefix": cmd,
"format": "json",
"id": osdID,
"weight": weight,
"args": []string{fmt.Sprintf("host=%s", hostName), "root=default"},
})
if err != nil {
return fmt.Errorf("command %s marshall failed: %+v", cmd, err)
}
log.Printf("%s command: '%s'", cmd, string(command))
_, info, err := osdConn.MonCommand(command)
if err != nil {
return fmt.Errorf("mon_command %s failed: %+v", cmd, err)
}
log.Printf("succeeded adding %s to crush map. info: %s", osdEntity, info)
return nil
}
// runs an OSD with the given config in a child process
func runOSD(clusterName string, osdID int, osdUUID *uuid.UUID, osdDataPath string) (*exec.Cmd, error) {
// start the OSD daemon in the foreground with the given config
log.Printf("starting osd %d at %s", osdID, osdDataPath)
cmd, err := proc.StartChildProcess(
"osd",
"--foreground",
fmt.Sprintf("--id=%s", strconv.Itoa(osdID)),
fmt.Sprintf("--cluster=%s", clusterName),
fmt.Sprintf("--osd-data=%s", osdDataPath),
fmt.Sprintf("--osd-journal=%s", getOSDJournalPath(osdDataPath)),
fmt.Sprintf("--conf=%s", getOSDConfFilePath(osdDataPath)),
fmt.Sprintf("--keyring=%s", getOSDKeyringPath(osdDataPath)),
fmt.Sprintf("--osd-uuid=%s", osdUUID.String()))
if err != nil {
return nil, fmt.Errorf("failed to start osd %d: %+v", osdID, err)
}
return cmd, nil
}
func getOSDConfFilePath(osdDataPath string) string {
return filepath.Join(osdDataPath, "tmp_config")
}
func getOSDKeyringPath(osdDataPath string) string {
return filepath.Join(osdDataPath, "keyring")
}
func getOSDJournalPath(osdDataPath string) string {
return filepath.Join(osdDataPath, "journal")
}
func getOSDTempMonMapPath(osdDataPath string) string {
return filepath.Join(osdDataPath, "tmp", "activate.monmap")
}
func getBootstrapOSDDir() string {
return "/tmp/bootstrap-osd"
}
func getBootstrapOSDKeyringPath(clusterName string) string {
return fmt.Sprintf("%s/%s", getBootstrapOSDDir(), fmt.Sprintf("%s.keyring", clusterName))
}
func getBootstrapOSDConfFilePath() string {
return fmt.Sprintf("%s/tmp_config", getBootstrapOSDDir())
}
+7 -5
View File
@@ -84,11 +84,13 @@ func bootstrap(cmd *cobra.Command, args []string) error {
<-ch
fmt.Println("terminating due to ctrl-c interrupt...")
for i := range procs {
fmt.Println("stopping child process")
if err := proc.StopChildProcess(procs[i]); err != nil {
fmt.Printf("failed to stop child process: %+v", err)
} else {
fmt.Println("child process stopped successfully")
if procs[i] != nil {
fmt.Printf("stopping child process %d\n", i)
if err := proc.StopChildProcess(procs[i]); err != nil {
fmt.Printf("failed to stop child process %d: %+v\n", i, err)
} else {
fmt.Printf("child process %d stopped successfully\n", i)
}
}
}
+1 -1
View File
@@ -55,7 +55,7 @@ func splitList(list string) []string {
func writeFile(filePath string, contentBuffer bytes.Buffer) error {
dir := filepath.Dir(filePath)
if err := os.MkdirAll(dir, 0744); err != nil {
return fmt.Errorf("failed to create config file directory for %s: %+v", filepath.Dir(filePath), err)
return fmt.Errorf("failed to create config file directory at %s: %+v", dir, err)
}
if err := ioutil.WriteFile(filePath, contentBuffer.Bytes(), 0644); err != nil {
return fmt.Errorf("failed to write config file to %s: %+v", filePath, err)