Files
my-rook-config/pkg/cephmgr/osd/device.go
T

438 lines
14 KiB
Go

/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package osd
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"syscall"
"github.com/google/uuid"
"github.com/rook/rook/pkg/cephmgr/client"
"github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/cephmgr/osd/partition"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/clusterd/inventory"
"github.com/rook/rook/pkg/util/exec"
"github.com/rook/rook/pkg/util/sys"
)
const (
DevicesValue = "devices"
ForceFormatValue = "forceFormat"
sgdisk = "sgdisk"
cephOsdKey = mon.CephKey + "/osd"
desiredOsdRootKey = cephOsdKey + "/" + clusterd.DesiredKey + "/%s"
deviceDesiredKey = desiredOsdRootKey + "/device"
dirDesiredKey = desiredOsdRootKey + "/dir"
bootstrapOSDKeyringTemplate = `
[client.bootstrap-osd]
key = %s
caps mon = "allow profile bootstrap-osd"
`
)
type osdConfig struct {
deviceName string
rootPath string
id int
uuid uuid.UUID
diskUUID string
bluestore bool
}
type Device struct {
Name string `json:"name"`
NodeID string `json:"nodeId"`
Dir bool `json:"bool"`
}
// format the given device for usage by an OSD
func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool) error {
// format the current volume
devFS, err := sys.GetDeviceFilesystems(config.deviceName, context.Executor)
if err != nil {
return fmt.Errorf("failed to get device %s filesystem: %+v", config.deviceName, err)
}
if devFS != "" && forceFormat {
// there's a filesystem on the device, but the user has specified to force a format. give a warning about that.
logger.Warningf("device %s already formatted with %s, but forcing a format!!!", config.deviceName, devFS)
}
if devFS == "" || forceFormat {
logger.Infof("Partitioning device %s for bluestore", config.deviceName)
err := partitionBluestoreDevice(context, config)
if err != nil {
return fmt.Errorf("failed to partion device %s. %v", config.deviceName, err)
}
} else {
// disk is already formatted and the user doesn't want to force it, but we require partitioning
return fmt.Errorf("device %s already formatted with %s", config.deviceName, devFS)
}
return nil
}
// Partitions a device for use by a bluestore osd.
// If there are any partitions or formatting already on the device, it will be wiped.
func partitionBluestoreDevice(context *clusterd.Context, config *osdConfig) error {
size, err := inventory.GetDeviceSize(config.deviceName, context.NodeID, context.EtcdClient)
if err != nil {
return fmt.Errorf("failed to get device %s size. %+v", config.deviceName, err)
}
cmd := fmt.Sprintf("zap %s", config.deviceName)
err = context.Executor.ExecuteCommand(cmd, sgdisk, "--zap-all", "/dev/"+config.deviceName)
if err != nil {
return fmt.Errorf("failed to zap partitions on /dev/%s: %+v", config.deviceName, err)
}
scheme, err := partition.GetSimpleScheme(int(size / 1024 / 1024))
if err != nil {
return fmt.Errorf("failed to get simple scheme. %+v", err)
}
config.diskUUID = scheme.DiskUUID
// get args for creating partitions
args := scheme.GetArgs(config.deviceName)
if err != nil {
return fmt.Errorf("failed to get partition args. %+v", err)
}
// execute the partition command
cmd = fmt.Sprintf("partition %s", config.deviceName)
err = context.Executor.ExecuteCommand(cmd, sgdisk, args...)
if err != nil {
return fmt.Errorf("failed to partition /dev/%s. %+v", config.deviceName, err)
}
err = inventory.SetDeviceUUID(context.NodeID, config.deviceName, scheme.DiskUUID, context.EtcdClient)
if err != nil {
return fmt.Errorf("failed to set uuid %s. %+v", scheme.DiskUUID, err)
}
// save the scheme
err = scheme.Save(config.rootPath)
if err != nil {
return fmt.Errorf("failed to save partition scheme. %+v", err)
}
return nil
}
func registerOSD(bootstrapConn client.Connection, config *osdConfig) error {
var err error
config.uuid, err = uuid.NewRandom()
if err != nil {
return fmt.Errorf("failed to generate UUID for osd: %+v", err)
}
// create the OSD instance via a mon_command, this assigns a cluster wide ID to the OSD
config.id, err = createOSD(bootstrapConn, config.uuid)
if err != nil {
return err
}
logger.Infof("successfully created OSD %s with ID %d", config.uuid.String(), config.id)
return nil
}
func getStoreSettings(context *clusterd.Context, config *osdConfig) (map[string]string, error) {
settings := map[string]string{}
if !config.bluestore {
return settings, nil
}
scheme, err := partition.LoadScheme(config.rootPath)
if err != nil {
return nil, fmt.Errorf("failed to load partition config. %+v", err)
}
walPartition, ok := scheme.PartitionUUIDs[partition.WalPartitionName]
if !ok {
return nil, fmt.Errorf("failed to find wal partition for osd %d", config.id)
}
dbPartition, ok := scheme.PartitionUUIDs[partition.DatabasePartitionName]
if !ok {
return nil, fmt.Errorf("failed to find db partition for osd %d", config.id)
}
blockPartition, ok := scheme.PartitionUUIDs[partition.BlockPartitionName]
if !ok {
return nil, fmt.Errorf("failed to find block partition for osd %d", config.id)
}
prefix := "/dev/disk/by-partuuid"
settings["bluestore block wal path"] = path.Join(prefix, walPartition)
settings["bluestore block db path"] = path.Join(prefix, dbPartition)
settings["bluestore block path"] = path.Join(prefix, blockPartition)
return settings, nil
}
func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context *clusterd.Context,
bootstrapConn client.Connection, cluster *mon.ClusterInfo, location string, executor exec.Executor) error {
cephConfig := mon.CreateDefaultCephConfig(cluster, config.rootPath, context.LogLevel, config.bluestore)
if !config.bluestore {
// using the local file system requires some config overrides
// http://docs.ceph.com/docs/jewel/rados/configuration/filesystem-recommendations/#not-recommended
cephConfig.GlobalConfig.OsdMaxObjectNameLen = 256
cephConfig.GlobalConfig.OsdMaxObjectNamespaceLen = 64
}
// bluestore has some extra settings
settings, err := getStoreSettings(context, config)
if err != nil {
return fmt.Errorf("failed to read store settings. %+v", err)
}
// write the OSD config file to disk
keyringPath := getOSDKeyringPath(config.rootPath)
_, err = mon.GenerateConfigFile(context, cluster, config.rootPath, fmt.Sprintf("osd.%d", config.id),
keyringPath, config.bluestore, cephConfig, settings)
if err != nil {
return fmt.Errorf("failed to write OSD %d config file: %+v", config.id, err)
}
// get the current monmap, it will be needed for creating the OSD file system
monMapRaw, err := getMonMap(bootstrapConn)
if err != nil {
return fmt.Errorf("failed to get mon map: %+v", err)
}
// create/initalize the OSD file system and journal
if err := createOSDFileSystem(context, cluster.Name, config, monMapRaw); err != nil {
return err
}
// add auth privileges for the OSD, the bootstrap-osd privileges were very limited
if err := addOSDAuth(bootstrapConn, config.id, config.rootPath); err != nil {
return err
}
// open a connection to the cluster using the OSDs creds
osdConn, err := mon.ConnectToCluster(context, factory, cluster, path.Join(config.rootPath, "tmp"),
fmt.Sprintf("osd.%d", config.id), keyringPath)
if err != nil {
return err
}
defer osdConn.Shutdown()
// add the new OSD to the cluster crush map
if err := addOSDToCrushMap(osdConn, context, config.id, config.rootPath, location); err != nil {
return err
}
return nil
}
// creates the OSD identity in the cluster via a mon_command
func createOSD(bootstrapConn client.Connection, osdUUID uuid.UUID) (int, error) {
cmd := map[string]interface{}{
"prefix": "osd create",
"entity": "client.bootstrap-osd",
"uuid": osdUUID.String(),
}
buf, err := client.ExecuteMonCommand(bootstrapConn, cmd, fmt.Sprintf("create osd %s", osdUUID))
if err != nil {
return 0, fmt.Errorf("failed to create osd %s: %+v", osdUUID, err)
}
var resp map[string]interface{}
err = json.Unmarshal(buf, &resp)
if err != nil {
return 0, fmt.Errorf("failed to unmarshall %s response: %+v. raw response: '%s'", cmd, err, string(buf[:]))
}
return int(resp["osdid"].(float64)), nil
}
// gets the current mon map for the cluster
func getMonMap(bootstrapConn client.Connection) ([]byte, error) {
cmd := map[string]interface{}{
"prefix": "mon getmap",
"entity": "client.bootstrap-osd",
}
buf, err := client.ExecuteMonCommand(bootstrapConn, cmd, "get mon map")
if err != nil {
return nil, fmt.Errorf("failed to get mon map: %+v", err)
}
return buf, nil
}
// creates/initalizes the OSD filesystem and journal via a child process
func createOSDFileSystem(context *clusterd.Context, clusterName string, config *osdConfig, monMap []byte) error {
logger.Infof("Initializing OSD %d file system at %s...", config.id, config.rootPath)
// the current monmap is needed to create the OSD, save it to a temp location so it is accessible
monMapTmpPath := getOSDTempMonMapPath(config.rootPath)
monMapTmpDir := filepath.Dir(monMapTmpPath)
if err := os.MkdirAll(monMapTmpDir, 0744); err != nil {
return fmt.Errorf("failed to create monmap tmp file directory at %s: %+v", monMapTmpDir, err)
}
if err := ioutil.WriteFile(monMapTmpPath, monMap, 0644); err != nil {
return fmt.Errorf("failed to write mon map to tmp file %s, %+v", monMapTmpPath, err)
}
options := []string{
"--mkfs",
"--mkkey",
fmt.Sprintf("--id=%d", config.id),
fmt.Sprintf("--cluster=%s", clusterName),
fmt.Sprintf("--conf=%s", mon.GetConfFilePath(config.rootPath, clusterName)),
fmt.Sprintf("--osd-data=%s", config.rootPath),
fmt.Sprintf("--osd-uuid=%s", config.uuid.String()),
fmt.Sprintf("--monmap=%s", monMapTmpPath),
}
if !config.bluestore {
options = append(options, fmt.Sprintf("--osd-journal=%s", getOSDJournalPath(config.rootPath)))
options = append(options, fmt.Sprintf("--keyring=%s", getOSDKeyringPath(config.rootPath)))
}
// create the OSD file system and journal
err := context.ProcMan.Run(
fmt.Sprintf("mkfs-osd%d", config.id),
"osd",
options...)
if err != nil {
return fmt.Errorf("failed osd mkfs for OSD ID %d, UUID %s, dataDir %s: %+v",
config.id, config.uuid.String(), config.rootPath, err)
}
return nil
}
// add OSD auth privileges for the given OSD ID. the bootstrap-osd privileges are limited and a real OSD needs more.
func addOSDAuth(bootstrapConn client.Connection, osdID int, osdDataPath string) error {
// create a new auth for this OSD
osdKeyringPath := getOSDKeyringPath(osdDataPath)
keyringBuffer, err := ioutil.ReadFile(osdKeyringPath)
if err != nil {
return fmt.Errorf("failed to read OSD keyring at %s, %+v", osdKeyringPath, err)
}
cmd := "auth add"
osdEntity := fmt.Sprintf("osd.%d", osdID)
command, err := json.Marshal(map[string]interface{}{
"prefix": cmd,
"format": "json",
"entity": osdEntity,
"caps": []string{"osd", "allow *", "mon", "allow profile osd"},
})
if err != nil {
return fmt.Errorf("command %s marshall failed: %+v", cmd, err)
}
_, info, err := bootstrapConn.MonCommandWithInputBuffer(command, keyringBuffer)
if err != nil {
return fmt.Errorf("mon_command %s failed: %+v", cmd, err)
}
logger.Debugf("succeeded %s command for %s. info: %s", cmd, osdEntity, info)
return nil
}
// adds the given OSD to the crush map
func addOSDToCrushMap(osdConn client.Connection, context *clusterd.Context, osdID int, osdDataPath, location string) error {
// get the size of the volume containing the OSD data dir
s := syscall.Statfs_t{}
if err := syscall.Statfs(osdDataPath, &s); err != nil {
return fmt.Errorf("failed to statfs on %s, %+v", osdDataPath, err)
}
all := s.Blocks * uint64(s.Bsize)
// weight is ratio of (size in KB) / (1 GB)
weight := float64(all/1024) / 1073741824.0
weight, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", weight), 64)
osdEntity := fmt.Sprintf("osd.%d", osdID)
logger.Infof("OSD %s at %s, bytes: %d, weight: %.4f", osdEntity, osdDataPath, all, weight)
locArgs, err := formatLocation(location)
if err != nil {
return err
}
cmd := map[string]interface{}{
"prefix": "osd crush create-or-move",
"id": osdID,
"weight": weight,
"args": locArgs,
}
_, err = client.ExecuteMonCommand(osdConn, cmd, fmt.Sprintf("adding %s to crush map", osdEntity))
if err != nil {
return fmt.Errorf("failed adding %s to crush map: %+v", osdEntity, err)
}
if err := inventory.SetLocation(context.EtcdClient, context.NodeID, strings.Join(locArgs, ",")); err != nil {
return fmt.Errorf("failed to save CRUSH location for OSD %s: %+v", osdEntity, err)
}
return nil
}
func markOSDOut(connection client.Connection, id int) error {
command := map[string]interface{}{
"prefix": "osd out",
"ids": []int{id},
}
_, err := client.ExecuteMonCommand(connection, command, fmt.Sprintf("mark osd %d out", id))
return err
}
func purgeOSD(connection client.Connection, id int) error {
// ceph osd crush remove <name>
command := map[string]interface{}{
"prefix": "osd crush remove",
"name": fmt.Sprintf("osd.%d", id),
}
_, err := client.ExecuteMonCommand(connection, command, fmt.Sprintf("remove osd %d from crush map", id))
if err != nil {
return fmt.Errorf("failed to remove osd %d from crush map. %v", id, err)
}
// ceph auth del osd.$osd_num
err = client.AuthDelete(connection, fmt.Sprintf("osd.%d", id))
if err != nil {
return err
}
// ceph osd rm $osd_num
command = map[string]interface{}{
"prefix": "osd rm",
"ids": []int{id},
}
_, err = client.ExecuteMonCommand(connection, command, fmt.Sprintf("rm osd %d", id))
if err != nil {
return fmt.Errorf("failed to rm osd %d. %v", id, err)
}
return nil
}