712 lines
22 KiB
Go
712 lines
22 KiB
Go
/*
|
|
Copyright 2016 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
package cephmgr
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"log"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
|
|
etcd "github.com/coreos/etcd/client"
|
|
ctx "golang.org/x/net/context"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/rook/rook/pkg/cephmgr/client"
|
|
"github.com/rook/rook/pkg/cephmgr/partition"
|
|
"github.com/rook/rook/pkg/clusterd"
|
|
"github.com/rook/rook/pkg/clusterd/inventory"
|
|
"github.com/rook/rook/pkg/util"
|
|
"github.com/rook/rook/pkg/util/exec"
|
|
"github.com/rook/rook/pkg/util/sys"
|
|
)
|
|
|
|
const (
|
|
DevicesValue = "devices"
|
|
ForceFormatValue = "forceFormat"
|
|
sgdisk = "sgdisk"
|
|
cephOsdKey = "/rook/services/ceph/osd"
|
|
desiredOsdRootKey = cephOsdKey + "/" + desiredKey + "/%s"
|
|
deviceDesiredKey = desiredOsdRootKey + "/device"
|
|
dirDesiredKey = desiredOsdRootKey + "/dir"
|
|
bootstrapOSDKeyringTemplate = `
|
|
[client.bootstrap-osd]
|
|
key = %s
|
|
caps mon = "allow profile bootstrap-osd"
|
|
`
|
|
)
|
|
|
|
type Device struct {
|
|
Name string `json:"name"`
|
|
NodeID string `json:"nodeId"`
|
|
Dir bool `json:"bool"`
|
|
}
|
|
|
|
func loadDesiredDevices(etcdClient etcd.KeysAPI, nodeID string) (map[string]int, error) {
|
|
devices := map[string]int{}
|
|
key := path.Join(fmt.Sprintf(deviceDesiredKey, nodeID))
|
|
devKeys, err := etcdClient.Get(ctx.Background(), key, &etcd.GetOptions{Recursive: true})
|
|
if err != nil {
|
|
if util.IsEtcdKeyNotFound(err) {
|
|
return devices, nil
|
|
}
|
|
return nil, err
|
|
}
|
|
|
|
// parse the dirs from etcd
|
|
for _, dev := range devKeys.Node.Nodes {
|
|
device := util.GetLeafKeyPath(dev.Key)
|
|
osdID := unassignedOSDID
|
|
|
|
for _, setting := range dev.Nodes {
|
|
if strings.HasSuffix(setting.Key, "/osd-id") {
|
|
id, err := strconv.Atoi(setting.Value)
|
|
if err == nil {
|
|
osdID = id
|
|
}
|
|
}
|
|
}
|
|
|
|
devices[device] = osdID
|
|
}
|
|
|
|
return devices, nil
|
|
}
|
|
|
|
func setOSDOnDevice(etcdClient etcd.KeysAPI, nodeID, name string, id int, dir bool) error {
|
|
var key string
|
|
if dir {
|
|
key = path.Join(fmt.Sprintf(dirDesiredKey, nodeID), getPseudoDir(name))
|
|
} else {
|
|
key = path.Join(fmt.Sprintf(deviceDesiredKey, nodeID), name)
|
|
}
|
|
|
|
_, err := etcdClient.Set(ctx.Background(), path.Join(key, "osd-id"), fmt.Sprintf("%d", id), nil)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to associate osd %d with %s", id, name)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// add a device to the desired state
|
|
func AddDesiredDevice(etcdClient etcd.KeysAPI, device, nodeID string) error {
|
|
key := path.Join(fmt.Sprintf(deviceDesiredKey, nodeID), device)
|
|
err := util.CreateEtcdDir(etcdClient, key)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to add device %s on node %s to desired. %v", device, nodeID, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func loadDesiredDirs(etcdClient etcd.KeysAPI, nodeID string) (map[string]int, error) {
|
|
dirs := map[string]int{}
|
|
key := path.Join(fmt.Sprintf(dirDesiredKey, nodeID))
|
|
dirKeys, err := etcdClient.Get(ctx.Background(), key, &etcd.GetOptions{Recursive: true})
|
|
if err != nil {
|
|
if util.IsEtcdKeyNotFound(err) {
|
|
return dirs, nil
|
|
}
|
|
return nil, err
|
|
}
|
|
|
|
// parse the dirs from etcd
|
|
for _, dev := range dirKeys.Node.Nodes {
|
|
id := unassignedOSDID
|
|
var path string
|
|
for _, setting := range dev.Nodes {
|
|
if strings.HasSuffix(setting.Key, "/path") {
|
|
path = setting.Value
|
|
} else if strings.HasSuffix(setting.Key, "/osd-id") {
|
|
osdID, err := strconv.Atoi(setting.Value)
|
|
if err == nil {
|
|
id = osdID
|
|
}
|
|
}
|
|
}
|
|
|
|
if path != "" {
|
|
dirs[path] = id
|
|
}
|
|
}
|
|
|
|
return dirs, nil
|
|
}
|
|
|
|
// add a device to the desired state
|
|
func AddDesiredDir(etcdClient etcd.KeysAPI, dir, nodeID string) error {
|
|
key := path.Join(fmt.Sprintf(dirDesiredKey, nodeID), getPseudoDir(dir), "path")
|
|
_, err := etcdClient.Set(ctx.Background(), key, dir, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to add desired dir %s on node %s. %v", dir, nodeID, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// remove a device from the desired state
|
|
func RemoveDesiredDevice(etcdClient etcd.KeysAPI, device, nodeID string) error {
|
|
|
|
key := path.Join(fmt.Sprintf(deviceDesiredKey, nodeID), device)
|
|
_, err := etcdClient.Delete(ctx.Background(), key, &etcd.DeleteOptions{Dir: true})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to remove device %s on node %s from desired. %v", device, nodeID, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// get the bootstrap OSD root dir
|
|
func getBootstrapOSDDir(configDir string) string {
|
|
return path.Join(configDir, "bootstrap-osd")
|
|
}
|
|
|
|
func getOSDRootDir(root string, osdID int) string {
|
|
return fmt.Sprintf("%s/osd%d", root, osdID)
|
|
}
|
|
|
|
// get the full path to the bootstrap OSD keyring
|
|
func getBootstrapOSDKeyringPath(configDir, clusterName string) string {
|
|
return fmt.Sprintf("%s/%s.keyring", getBootstrapOSDDir(configDir), clusterName)
|
|
}
|
|
|
|
// get the full path to the given OSD's config file
|
|
func getOSDConfFilePath(osdDataPath, clusterName string) string {
|
|
return fmt.Sprintf("%s/%s.config", osdDataPath, clusterName)
|
|
}
|
|
|
|
// get the full path to the given OSD's keyring
|
|
func getOSDKeyringPath(osdDataPath string) string {
|
|
return filepath.Join(osdDataPath, "keyring")
|
|
}
|
|
|
|
// get the full path to the given OSD's journal
|
|
func getOSDJournalPath(osdDataPath string) string {
|
|
return filepath.Join(osdDataPath, "journal")
|
|
}
|
|
|
|
// get the full path to the given OSD's temporary mon map
|
|
func getOSDTempMonMapPath(osdDataPath string) string {
|
|
return filepath.Join(osdDataPath, "tmp", "activate.monmap")
|
|
}
|
|
|
|
// create a keyring for the bootstrap-osd client, it gets a limited set of privileges
|
|
func createOSDBootstrapKeyring(conn client.Connection, configDir, clusterName string) error {
|
|
bootstrapOSDKeyringPath := getBootstrapOSDKeyringPath(configDir, clusterName)
|
|
_, err := os.Stat(bootstrapOSDKeyringPath)
|
|
if err == nil {
|
|
// no error, the file exists, bail out with no error
|
|
log.Printf("bootstrap OSD keyring already exists at %s", bootstrapOSDKeyringPath)
|
|
return nil
|
|
} else if !os.IsNotExist(err) {
|
|
// some other error besides "does not exist", bail out with error
|
|
return fmt.Errorf("failed to stat %s: %+v", bootstrapOSDKeyringPath, err)
|
|
}
|
|
|
|
// get-or-create-key for client.bootstrap-osd
|
|
bootstrapOSDKey, err := client.AuthGetOrCreateKey(conn, "client.bootstrap-osd", []string{"mon", "allow profile bootstrap-osd"})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get or create osd auth key %s. %+v", bootstrapOSDKeyringPath, err)
|
|
}
|
|
|
|
log.Printf("succeeded bootstrap OSD get/create key, bootstrapOSDKey: %s", bootstrapOSDKey)
|
|
|
|
// write the bootstrap-osd keyring to disk
|
|
bootstrapOSDKeyringDir := filepath.Dir(bootstrapOSDKeyringPath)
|
|
if err := os.MkdirAll(bootstrapOSDKeyringDir, 0744); err != nil {
|
|
return fmt.Errorf("failed to create bootstrap OSD keyring dir at %s: %+v", bootstrapOSDKeyringDir, err)
|
|
}
|
|
|
|
bootstrapOSDKeyring := fmt.Sprintf(bootstrapOSDKeyringTemplate, bootstrapOSDKey)
|
|
log.Printf("Writing osd keyring to: %s", bootstrapOSDKeyring)
|
|
if err := ioutil.WriteFile(bootstrapOSDKeyringPath, []byte(bootstrapOSDKeyring), 0644); err != nil {
|
|
return fmt.Errorf("failed to write bootstrap-osd keyring to %s: %+v", bootstrapOSDKeyringPath, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// format the given device for usage by an OSD
|
|
func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool) error {
|
|
// format the current volume
|
|
devFS, err := sys.GetDeviceFilesystems(config.deviceName, context.Executor)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get device %s filesystem: %+v", config.deviceName, err)
|
|
}
|
|
|
|
if devFS != "" && forceFormat {
|
|
// there's a filesystem on the device, but the user has specified to force a format. give a warning about that.
|
|
log.Printf("WARNING: device %s already formatted with %s, but forcing a format!!!", config.deviceName, devFS)
|
|
}
|
|
|
|
if devFS == "" || forceFormat {
|
|
log.Printf("Partitioning device %s for bluestore", config.deviceName)
|
|
|
|
err := partitionBluestoreDevice(context, config)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to partion device %s. %v", config.deviceName, err)
|
|
}
|
|
|
|
} else {
|
|
// disk is already formatted and the user doesn't want to force it, but we require partitioning
|
|
return fmt.Errorf("device %s already formatted with %s", config.deviceName, devFS)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Partitions a device for use by a bluestore osd.
|
|
// If there are any partitions or formatting already on the device, it will be wiped.
|
|
func partitionBluestoreDevice(context *clusterd.Context, config *osdConfig) error {
|
|
size, err := inventory.GetDeviceSize(config.deviceName, context.NodeID, context.EtcdClient)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get device %s size. %+v", config.deviceName, err)
|
|
}
|
|
|
|
cmd := fmt.Sprintf("zap %s", config.deviceName)
|
|
err = context.Executor.ExecuteCommand(cmd, sgdisk, "--zap-all", "/dev/"+config.deviceName)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to zap partitions on /dev/%s: %+v", config.deviceName, err)
|
|
}
|
|
|
|
scheme, err := partition.GetSimpleScheme(int(size / 1024 / 1024))
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get simple scheme. %+v", err)
|
|
}
|
|
config.diskUUID = scheme.DiskUUID
|
|
|
|
// get args for creating partitions
|
|
args := scheme.GetArgs(config.deviceName)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get partition args. %+v", err)
|
|
}
|
|
|
|
// execute the partition command
|
|
cmd = fmt.Sprintf("partition %s", config.deviceName)
|
|
err = context.Executor.ExecuteCommand(cmd, sgdisk, args...)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to partition /dev/%s. %+v", config.deviceName, err)
|
|
}
|
|
|
|
err = inventory.SetDeviceUUID(context.NodeID, config.deviceName, scheme.DiskUUID, context.EtcdClient)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to set uuid %s. %+v", scheme.DiskUUID, err)
|
|
}
|
|
|
|
// save the scheme
|
|
err = scheme.Save(config.rootPath)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to save partition scheme. %+v", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func registerOSD(bootstrapConn client.Connection, config *osdConfig) error {
|
|
var err error
|
|
config.uuid, err = uuid.NewRandom()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to generate UUID for osd: %+v", err)
|
|
}
|
|
|
|
// create the OSD instance via a mon_command, this assigns a cluster wide ID to the OSD
|
|
config.id, err = createOSD(bootstrapConn, config.uuid)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Printf("successfully created OSD %s with ID %d", config.uuid.String(), config.id)
|
|
return nil
|
|
}
|
|
|
|
func isOSDDataNotExist(osdDataPath string) bool {
|
|
_, err := os.Stat(filepath.Join(osdDataPath, "whoami"))
|
|
return os.IsNotExist(err)
|
|
}
|
|
|
|
func loadOSDInfo(config *osdConfig) error {
|
|
idFile := filepath.Join(config.rootPath, "whoami")
|
|
idContent, err := ioutil.ReadFile(idFile)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to read OSD ID from %s: %+v", idFile, err)
|
|
}
|
|
|
|
osdID, err := strconv.Atoi(strings.TrimSpace(string(idContent[:])))
|
|
if err != nil {
|
|
return fmt.Errorf("failed to parse OSD ID from %s with content %s: %+v", idFile, idContent, err)
|
|
}
|
|
|
|
uuidFile := filepath.Join(config.rootPath, "fsid")
|
|
fsidContent, err := ioutil.ReadFile(uuidFile)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to read UUID from %s: %+v", uuidFile, err)
|
|
}
|
|
|
|
osdUUID, err := uuid.Parse(strings.TrimSpace(string(fsidContent[:])))
|
|
if err != nil {
|
|
return fmt.Errorf("failed to parse UUID from %s with content %s: %+v", uuidFile, string(fsidContent[:]), err)
|
|
}
|
|
|
|
config.id = osdID
|
|
config.uuid = osdUUID
|
|
return nil
|
|
}
|
|
|
|
func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context *clusterd.Context,
|
|
bootstrapConn client.Connection, cluster *ClusterInfo, location string, debug bool, executor exec.Executor) error {
|
|
|
|
cephConfig := createDefaultCephConfig(cluster, config.rootPath, debug, config.bluestore)
|
|
if !config.bluestore {
|
|
// using the local file system requires some config overrides
|
|
// http://docs.ceph.com/docs/jewel/rados/configuration/filesystem-recommendations/#not-recommended
|
|
cephConfig.cephGlobalConfig.OsdMaxObjectNameLen = 256
|
|
cephConfig.cephGlobalConfig.OsdMaxObjectNamespaceLen = 64
|
|
}
|
|
|
|
// bluestore has some extra settings
|
|
settings, err := getStoreSettings(context, config)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to read store settings. %+v", err)
|
|
}
|
|
|
|
// write the OSD config file to disk
|
|
keyringPath := getOSDKeyringPath(config.rootPath)
|
|
_, err = generateConfigFile(context, cluster, config.rootPath, fmt.Sprintf("osd.%d", config.id),
|
|
keyringPath, debug, config.bluestore, cephConfig, settings)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to write OSD %d config file: %+v", config.id, err)
|
|
}
|
|
|
|
// get the current monmap, it will be needed for creating the OSD file system
|
|
monMapRaw, err := getMonMap(bootstrapConn)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get mon map: %+v", err)
|
|
}
|
|
|
|
// create/initalize the OSD file system and journal
|
|
if err := createOSDFileSystem(context, cluster.Name, config, monMapRaw); err != nil {
|
|
return err
|
|
}
|
|
|
|
// add auth privileges for the OSD, the bootstrap-osd privileges were very limited
|
|
if err := addOSDAuth(bootstrapConn, config.id, config.rootPath); err != nil {
|
|
return err
|
|
}
|
|
|
|
// open a connection to the cluster using the OSDs creds
|
|
osdConn, err := connectToCluster(context, factory, cluster, path.Join(config.rootPath, "tmp"),
|
|
fmt.Sprintf("osd.%d", config.id), keyringPath, debug)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer osdConn.Shutdown()
|
|
|
|
// add the new OSD to the cluster crush map
|
|
if err := addOSDToCrushMap(osdConn, context, config.id, config.rootPath, location); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func getStoreSettings(context *clusterd.Context, config *osdConfig) (map[string]string, error) {
|
|
settings := map[string]string{}
|
|
if !config.bluestore {
|
|
return settings, nil
|
|
}
|
|
|
|
scheme, err := partition.LoadScheme(config.rootPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to load partition config. %+v", err)
|
|
}
|
|
|
|
walPartition, ok := scheme.PartitionUUIDs[partition.WalPartitionName]
|
|
if !ok {
|
|
return nil, fmt.Errorf("failed to find wal partition for osd %d", config.id)
|
|
}
|
|
dbPartition, ok := scheme.PartitionUUIDs[partition.DatabasePartitionName]
|
|
if !ok {
|
|
return nil, fmt.Errorf("failed to find db partition for osd %d", config.id)
|
|
}
|
|
blockPartition, ok := scheme.PartitionUUIDs[partition.BlockPartitionName]
|
|
if !ok {
|
|
return nil, fmt.Errorf("failed to find block partition for osd %d", config.id)
|
|
}
|
|
|
|
prefix := "/dev/disk/by-partuuid"
|
|
settings["bluestore block wal path"] = path.Join(prefix, walPartition)
|
|
settings["bluestore block db path"] = path.Join(prefix, dbPartition)
|
|
settings["bluestore block path"] = path.Join(prefix, blockPartition)
|
|
|
|
return settings, nil
|
|
}
|
|
|
|
// creates the OSD identity in the cluster via a mon_command
|
|
func createOSD(bootstrapConn client.Connection, osdUUID uuid.UUID) (int, error) {
|
|
cmd := map[string]interface{}{
|
|
"prefix": "osd create",
|
|
"entity": "client.bootstrap-osd",
|
|
"uuid": osdUUID.String(),
|
|
}
|
|
buf, err := client.ExecuteMonCommand(bootstrapConn, cmd, fmt.Sprintf("create osd %s", osdUUID))
|
|
if err != nil {
|
|
return 0, fmt.Errorf("failed to create osd %s: %+v", osdUUID, err)
|
|
}
|
|
|
|
var resp map[string]interface{}
|
|
err = json.Unmarshal(buf, &resp)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("failed to unmarshall %s response: %+v. raw response: '%s'", cmd, err, string(buf[:]))
|
|
}
|
|
|
|
return int(resp["osdid"].(float64)), nil
|
|
}
|
|
|
|
// gets the current mon map for the cluster
|
|
func getMonMap(bootstrapConn client.Connection) ([]byte, error) {
|
|
cmd := map[string]interface{}{
|
|
"prefix": "mon getmap",
|
|
"entity": "client.bootstrap-osd",
|
|
}
|
|
|
|
buf, err := client.ExecuteMonCommand(bootstrapConn, cmd, "get mon map")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get mon map: %+v", err)
|
|
}
|
|
return buf, nil
|
|
}
|
|
|
|
// creates/initalizes the OSD filesystem and journal via a child process
|
|
func createOSDFileSystem(context *clusterd.Context, clusterName string, config *osdConfig, monMap []byte) error {
|
|
log.Printf("Initializing OSD %d file system at %s...", config.id, config.rootPath)
|
|
|
|
// the current monmap is needed to create the OSD, save it to a temp location so it is accessible
|
|
monMapTmpPath := getOSDTempMonMapPath(config.rootPath)
|
|
monMapTmpDir := filepath.Dir(monMapTmpPath)
|
|
if err := os.MkdirAll(monMapTmpDir, 0744); err != nil {
|
|
return fmt.Errorf("failed to create monmap tmp file directory at %s: %+v", monMapTmpDir, err)
|
|
}
|
|
if err := ioutil.WriteFile(monMapTmpPath, monMap, 0644); err != nil {
|
|
return fmt.Errorf("failed to write mon map to tmp file %s, %+v", monMapTmpPath, err)
|
|
}
|
|
|
|
options := []string{
|
|
"--mkfs",
|
|
"--mkkey",
|
|
fmt.Sprintf("--id=%d", config.id),
|
|
fmt.Sprintf("--cluster=%s", clusterName),
|
|
fmt.Sprintf("--conf=%s", getConfFilePath(config.rootPath, clusterName)),
|
|
fmt.Sprintf("--osd-data=%s", config.rootPath),
|
|
fmt.Sprintf("--osd-uuid=%s", config.uuid.String()),
|
|
fmt.Sprintf("--monmap=%s", monMapTmpPath),
|
|
}
|
|
|
|
if !config.bluestore {
|
|
options = append(options, fmt.Sprintf("--osd-journal=%s", getOSDJournalPath(config.rootPath)))
|
|
options = append(options, fmt.Sprintf("--keyring=%s", getOSDKeyringPath(config.rootPath)))
|
|
}
|
|
|
|
// create the OSD file system and journal
|
|
err := context.ProcMan.Run(
|
|
"osd",
|
|
options...)
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("failed osd mkfs for OSD ID %d, UUID %s, dataDir %s: %+v",
|
|
config.id, config.uuid.String(), config.rootPath, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// add OSD auth privileges for the given OSD ID. the bootstrap-osd privileges are limited and a real OSD needs more.
|
|
func addOSDAuth(bootstrapConn client.Connection, osdID int, osdDataPath string) error {
|
|
// create a new auth for this OSD
|
|
osdKeyringPath := getOSDKeyringPath(osdDataPath)
|
|
keyringBuffer, err := ioutil.ReadFile(osdKeyringPath)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to read OSD keyring at %s, %+v", osdKeyringPath, err)
|
|
}
|
|
|
|
cmd := "auth add"
|
|
osdEntity := fmt.Sprintf("osd.%d", osdID)
|
|
command, err := json.Marshal(map[string]interface{}{
|
|
"prefix": cmd,
|
|
"format": "json",
|
|
"entity": osdEntity,
|
|
"caps": []string{"osd", "allow *", "mon", "allow profile osd"},
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("command %s marshall failed: %+v", cmd, err)
|
|
}
|
|
_, info, err := bootstrapConn.MonCommandWithInputBuffer(command, keyringBuffer)
|
|
if err != nil {
|
|
return fmt.Errorf("mon_command %s failed: %+v", cmd, err)
|
|
}
|
|
|
|
log.Printf("succeeded %s command for %s. info: %s", cmd, osdEntity, info)
|
|
return nil
|
|
}
|
|
|
|
// adds the given OSD to the crush map
|
|
func addOSDToCrushMap(osdConn client.Connection, context *clusterd.Context, osdID int, osdDataPath, location string) error {
|
|
// get the size of the volume containing the OSD data dir
|
|
s := syscall.Statfs_t{}
|
|
if err := syscall.Statfs(osdDataPath, &s); err != nil {
|
|
return fmt.Errorf("failed to statfs on %s, %+v", osdDataPath, err)
|
|
}
|
|
all := s.Blocks * uint64(s.Bsize)
|
|
|
|
// weight is ratio of (size in KB) / (1 GB)
|
|
weight := float64(all/1024) / 1073741824.0
|
|
weight, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", weight), 64)
|
|
|
|
osdEntity := fmt.Sprintf("osd.%d", osdID)
|
|
log.Printf("OSD %s at %s, bytes: %d, weight: %.4f", osdEntity, osdDataPath, all, weight)
|
|
|
|
locArgs, err := formatLocation(location)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
cmd := map[string]interface{}{
|
|
"prefix": "osd crush create-or-move",
|
|
"id": osdID,
|
|
"weight": weight,
|
|
"args": locArgs,
|
|
}
|
|
_, err = client.ExecuteMonCommand(osdConn, cmd, fmt.Sprintf("adding %s to crush map", osdEntity))
|
|
if err != nil {
|
|
return fmt.Errorf("failed adding %s to crush map: %+v", osdEntity, err)
|
|
}
|
|
|
|
if err := inventory.SetLocation(context.EtcdClient, context.NodeID, strings.Join(locArgs, ",")); err != nil {
|
|
return fmt.Errorf("failed to save CRUSH location for OSD %s: %+v", osdEntity, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func markOSDOut(connection client.Connection, id int) error {
|
|
command := map[string]interface{}{
|
|
"prefix": "osd out",
|
|
"ids": []int{id},
|
|
}
|
|
_, err := client.ExecuteMonCommand(connection, command, fmt.Sprintf("mark osd %d out", id))
|
|
return err
|
|
}
|
|
|
|
func purgeOSD(connection client.Connection, id int) error {
|
|
// ceph osd crush remove <name>
|
|
command := map[string]interface{}{
|
|
"prefix": "osd crush remove",
|
|
"name": fmt.Sprintf("osd.%d", id),
|
|
}
|
|
_, err := client.ExecuteMonCommand(connection, command, fmt.Sprintf("remove osd %d from crush map", id))
|
|
if err != nil {
|
|
return fmt.Errorf("failed to remove osd %d from crush map. %v", id, err)
|
|
}
|
|
|
|
// ceph auth del osd.$osd_num
|
|
err = client.AuthDelete(connection, fmt.Sprintf("osd.%d", id))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// ceph osd rm $osd_num
|
|
command = map[string]interface{}{
|
|
"prefix": "osd rm",
|
|
"ids": []int{id},
|
|
}
|
|
_, err = client.ExecuteMonCommand(connection, command, fmt.Sprintf("rm osd %d", id))
|
|
if err != nil {
|
|
return fmt.Errorf("failed to rm osd %d. %v", id, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// calls osd getcrushmap
|
|
func GetCrushMap(adminConn client.Connection) (string, error) {
|
|
cmd := map[string]interface{}{"prefix": "osd crush dump"}
|
|
buf, err := client.ExecuteMonCommand(adminConn, cmd, fmt.Sprintf("retrieving crush map"))
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to get crush map. %v", err)
|
|
}
|
|
|
|
return string(buf), nil
|
|
}
|
|
|
|
func formatLocation(location string) ([]string, error) {
|
|
var pairs []string
|
|
if location == "" {
|
|
pairs = []string{}
|
|
} else {
|
|
pairs = strings.Split(location, ",")
|
|
}
|
|
|
|
for _, p := range pairs {
|
|
if !isValidCrushFieldFormat(p) {
|
|
return nil, fmt.Errorf("CRUSH location field '%s' is not in a valid format", p)
|
|
}
|
|
}
|
|
|
|
if !isCrushFieldSet("hostName", pairs) {
|
|
// host name isn't set yet, attempt to set a default
|
|
hostName, err := os.Hostname()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get hostname, %+v", err)
|
|
}
|
|
|
|
pairs = append(pairs, formatProperty("hostName", hostName))
|
|
}
|
|
|
|
// set a default root if it's not already set
|
|
if !isCrushFieldSet("root", pairs) {
|
|
pairs = append(pairs, formatProperty("root", "default"))
|
|
}
|
|
|
|
return pairs, nil
|
|
}
|
|
|
|
func isValidCrushFieldFormat(pair string) bool {
|
|
matched, err := regexp.MatchString("^.+=.+$", pair)
|
|
return matched && err == nil
|
|
}
|
|
|
|
func isCrushFieldSet(fieldName string, pairs []string) bool {
|
|
for _, p := range pairs {
|
|
kv := strings.Split(p, "=")
|
|
if len(kv) == 2 && kv[0] == fieldName && kv[1] != "" {
|
|
return true
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
func formatProperty(name, value string) string {
|
|
return fmt.Sprintf("%s=%s", name, value)
|
|
}
|