my-rook-config/pkg/daemon/ceph/osd/scheme.go

/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package osd

import (
	"encoding/json"
	"fmt"

	"github.com/google/uuid"
	rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha1"
	"github.com/rook/rook/pkg/operator/k8sutil"
	"k8s.io/apimachinery/pkg/api/errors"
)

const (
	Filestore             = "filestore"
	Bluestore             = "bluestore"
	DefaultStore          = Bluestore
	UseRemainingSpace     = -1
	schemeKeyName         = "partition-scheme"
	WalDefaultSizeMB      = 576
	DBDefaultSizeMB       = 20480
	JournalDefaultSizeMB  = 5120
	bluestoreDirBlockName = "bluestore-block"
	bluestoreDirWalName   = "bluestore-wal"
	bluestoreDirDBName    = "bluestore-db"
)

type PartitionType int

const (
	WalPartitionType PartitionType = iota
	DatabasePartitionType
	BlockPartitionType
	FilestoreDataPartitionType
	FilestoreJournalPartitionType
)

// top level representation of an overall performance oriented partition scheme, with a dedicated metadata device
// and entries for all OSDs that define where their partitions live
type PerfScheme struct {
	Metadata *MetadataDeviceInfo `json:"metadata"`
	Entries  []*PerfSchemeEntry  `json:"entries"`
}

// represents an OSD and details about all of its partitions
type PerfSchemeEntry struct {
	ID         int                                           `json:"id"`
	OsdUUID    uuid.UUID                                     `json:"osdUuid"`
	Partitions map[PartitionType]*PerfSchemePartitionDetails `json:"partitions"` // mapping of partition name to its details
	StoreType  string                                        `json:"storeType,omitempty"`
	FSCreated  bool                                          `json:"fsCreated"`
}

// details for 1 OSD partition
type PerfSchemePartitionDetails struct {
	Device        string `json:"device"`
	DiskUUID      string `json:"diskUuid"`
	PartitionUUID string `json:"partitionUuid"`
	SizeMB        int    `json:"sizeMB"`
	OffsetMB      int    `json:"offsetMB"`
}

// represents a dedicated metadata device and all of the partitions stored on it
type MetadataDeviceInfo struct {
	Device     string                     `json:"device"`
	DiskUUID   string                     `json:"diskUuid"`
	Partitions []*MetadataDevicePartition `json:"partitions"`
}

// representsa specific partition on a metadata device, including details about which OSD it belongs to
type MetadataDevicePartition struct {
	ID            int           `json:"id"`
	OsdUUID       uuid.UUID     `json:"osdUuid"`
	Type          PartitionType `json:"type"`
	PartitionUUID string        `json:"partitionUuid"`
	SizeMB        int           `json:"sizeMB"`
	OffsetMB      int           `json:"offsetMB"`
}

func NewPerfScheme() *PerfScheme {
	return &PerfScheme{
		Entries: []*PerfSchemeEntry{},
	}
}

func NewPerfSchemeEntry(storeType string) *PerfSchemeEntry {
	return &PerfSchemeEntry{
		Partitions: map[PartitionType]*PerfSchemePartitionDetails{}, // mapping of partition name (e.g. WAL) to it's details
		StoreType:  storeType,
	}
}

func NewMetadataDeviceInfo(device string) *MetadataDeviceInfo {
	return &MetadataDeviceInfo{Device: device, Partitions: []*MetadataDevicePartition{}}
}

// Load the persistent partition info from the config directory.
func LoadScheme(kv *k8sutil.ConfigMapKVStore, storeName string) (*PerfScheme, error) {
	schemeRaw, err := kv.GetValue(storeName, schemeKeyName)
	if err != nil {
		if errors.IsNotFound(err) {
			// the scheme key doesn't exist yet, just return a new empty scheme with no error
			return NewPerfScheme(), nil
		}

		return nil, err
	}

	var scheme PerfScheme
	err = json.Unmarshal([]byte(schemeRaw), &scheme)
	if err != nil {
		return nil, err
	}

	return &scheme, nil
}

// Save the partition scheme to the config dir
func (s *PerfScheme) SaveScheme(kv *k8sutil.ConfigMapKVStore, storeName string) error {
	b, err := json.Marshal(s)
	if err != nil {
		return err
	}

	err = kv.SetValue(storeName, schemeKeyName, string(b))
	if err != nil {
		return err
	}

	return nil
}

func (s *PerfScheme) updateSchemeEntry(entry *PerfSchemeEntry) error {
	if entry == nil {
		return fmt.Errorf("cannot update a nil entry")
	}

	foundEntry := false
	for i := range s.Entries {
		if s.Entries[i].ID == entry.ID {
			// found the matching existing entry, update it to the given entry
			s.Entries[i] = entry
			foundEntry = true
			break
		}
	}

	if !foundEntry {
		return fmt.Errorf("failed to find entry %+v", entry)
	}

	return nil
}

// populates a partition scheme entry for an OSD where all its partitions are collocated on a single device
func PopulateCollocatedPerfSchemeEntry(entry *PerfSchemeEntry, device string, storeConfig rookalpha.StoreConfig) error {

	if storeConfig.StoreType == Filestore {
		diskUUID, dataUUID, _, err := createFilestoreUUIDs()
		if err != nil {
			return err
		}

		// the filestore data partition will take up the entire given device (and we do not create a separate partition/entry
		// for the journal)
		entry.Partitions[FilestoreDataPartitionType] = &PerfSchemePartitionDetails{
			Device:        device,
			DiskUUID:      diskUUID.String(),
			PartitionUUID: dataUUID.String(),
			SizeMB:        UseRemainingSpace,
			OffsetMB:      1,
		}
	} else {
		diskUUID, walUUID, dbUUID, blockUUID, err := createBluestoreUUIDs()
		if err != nil {
			return err
		}

		walSize := WalDefaultSizeMB
		if storeConfig.WalSizeMB > 0 {
			walSize = storeConfig.WalSizeMB
		}
		dbSize := DBDefaultSizeMB
		if storeConfig.DatabaseSizeMB > 0 {
			dbSize = storeConfig.DatabaseSizeMB
		}

		offset := 1

		// layout the partitions for WAL, DB, and Block
		entry.Partitions[WalPartitionType] = &PerfSchemePartitionDetails{
			Device:        device,
			DiskUUID:      diskUUID.String(),
			PartitionUUID: walUUID.String(),
			SizeMB:        walSize,
			OffsetMB:      offset,
		}
		offset += entry.Partitions[WalPartitionType].SizeMB

		entry.Partitions[DatabasePartitionType] = &PerfSchemePartitionDetails{
			Device:        device,
			DiskUUID:      diskUUID.String(),
			PartitionUUID: dbUUID.String(),
			SizeMB:        dbSize,
			OffsetMB:      offset,
		}
		offset += entry.Partitions[DatabasePartitionType].SizeMB

		entry.Partitions[BlockPartitionType] = &PerfSchemePartitionDetails{
			Device:        device,
			DiskUUID:      diskUUID.String(),
			PartitionUUID: blockUUID.String(),
			SizeMB:        UseRemainingSpace,
			OffsetMB:      offset,
		}
	}

	return nil
}

// populates a partition scheme entry for an OSD that will have distributed partitions: its metadata will live on a
// dedicated metadata device and its block data will live on a dedicated device
func PopulateDistributedPerfSchemeEntry(entry *PerfSchemeEntry, device string, metadataInfo *MetadataDeviceInfo,
	storeConfig rookalpha.StoreConfig) error {

	if storeConfig.StoreType == Filestore {
		// TODO: support separate metadata device for filestore
		return fmt.Errorf("filestore not yet supported for distributed partition scheme")
	}

	diskUUID, walUUID, dbUUID, blockUUID, err := createBluestoreUUIDs()
	if err != nil {
		return err
	}

	// the block partition will take up the entire given device
	entry.Partitions[BlockPartitionType] = &PerfSchemePartitionDetails{
		Device:        device,
		DiskUUID:      diskUUID.String(),
		PartitionUUID: blockUUID.String(),
		SizeMB:        UseRemainingSpace,
		OffsetMB:      1,
	}

	// the WAL and DB will be on a separate metadata device
	offset := 1
	numMetadataParts := len(metadataInfo.Partitions)
	if numMetadataParts == 0 {
		// the metadata device hasn't been used yet, create a disk UUID for it
		u, err := uuid.NewRandom()
		if err != nil {
			return fmt.Errorf("failed to get metadata disk uuid. %+v", err)
		}
		metadataInfo.DiskUUID = u.String()
	} else {
		lastEntry := metadataInfo.Partitions[numMetadataParts-1]
		offset = lastEntry.OffsetMB + lastEntry.SizeMB
	}

	walSize := WalDefaultSizeMB
	if storeConfig.WalSizeMB > 0 {
		walSize = storeConfig.WalSizeMB
	}
	dbSize := DBDefaultSizeMB
	if storeConfig.DatabaseSizeMB > 0 {
		dbSize = storeConfig.DatabaseSizeMB
	}

	// record information about the WAL partition
	entry.Partitions[WalPartitionType] = &PerfSchemePartitionDetails{
		Device:        metadataInfo.Device,
		DiskUUID:      metadataInfo.DiskUUID,
		PartitionUUID: walUUID.String(),
		SizeMB:        walSize,
		OffsetMB:      offset,
	}
	walPartitionInfo := &MetadataDevicePartition{
		ID:            entry.ID,
		OsdUUID:       entry.OsdUUID,
		Type:          WalPartitionType,
		PartitionUUID: walUUID.String(),
		SizeMB:        walSize,
		OffsetMB:      offset,
	}
	metadataInfo.Partitions = append(metadataInfo.Partitions, walPartitionInfo)
	offset += entry.Partitions[WalPartitionType].SizeMB

	// record information about the DB partition
	entry.Partitions[DatabasePartitionType] = &PerfSchemePartitionDetails{
		Device:        metadataInfo.Device,
		DiskUUID:      metadataInfo.DiskUUID,
		PartitionUUID: dbUUID.String(),
		SizeMB:        dbSize,
		OffsetMB:      offset,
	}
	dbPartitionInfo := &MetadataDevicePartition{
		ID:            entry.ID,
		OsdUUID:       entry.OsdUUID,
		Type:          DatabasePartitionType,
		PartitionUUID: dbUUID.String(),
		SizeMB:        dbSize,
		OffsetMB:      offset,
	}
	metadataInfo.Partitions = append(metadataInfo.Partitions, dbPartitionInfo)

	return nil
}

func (m *MetadataDeviceInfo) GetPartitionArgs() []string {
	args := []string{}

	for partNum, part := range m.Partitions {
		partArgs := getPartitionArgs(partNum+1, part.PartitionUUID, part.OffsetMB, part.SizeMB, getPartitionLabel(part.ID, part.Type))
		args = append(args, partArgs...)
	}

	// append args for the whole device
	args = append(args, []string{fmt.Sprintf("--disk-guid=%s", m.DiskUUID), "/dev/" + m.Device}...)

	return args
}

func (e *PerfSchemeEntry) GetPartitionArgs() []string {
	// first determine if all the partitions are collocated
	collocated := e.IsCollocated()

	args := []string{}
	partNum := 1

	if collocated && e.StoreType == Bluestore {
		// partitions are collocated, create the metadata partitions on the same device
		walDetails := e.Partitions[WalPartitionType]
		partArgs := getPartitionArgsFromDetails(partNum, WalPartitionType, e.ID, walDetails)
		args = append(args, partArgs...)
		partNum++

		dbDetails := e.Partitions[DatabasePartitionType]
		partArgs = getPartitionArgsFromDetails(partNum, DatabasePartitionType, e.ID, dbDetails)
		args = append(args, partArgs...)
		partNum++
	}

	dataPartitionType := e.getDataPartitionType()

	// always create the data partition
	dataDetails := e.Partitions[dataPartitionType]
	dataPartArgs := getPartitionArgsFromDetails(partNum, dataPartitionType, e.ID, dataDetails)
	args = append(args, dataPartArgs...)

	// append args for the whole device
	args = append(args, []string{fmt.Sprintf("--disk-guid=%s", dataDetails.DiskUUID), "/dev/" + dataDetails.Device}...)

	return args
}

// determines if the given partition scheme entry is for a collocated OSD (all partitions on 1 device)
func (e *PerfSchemeEntry) IsCollocated() bool {
	collocated := true
	du := ""
	for _, details := range e.Partitions {
		if du == "" {
			du = details.DiskUUID
		} else if du != details.DiskUUID {
			// not all partitions are on the same disk, the partitions are not collocated
			collocated = false
			break
		}
	}

	return collocated
}

func (e *PerfSchemeEntry) getDataPartitionType() PartitionType {
	if e.StoreType == Filestore {
		return FilestoreDataPartitionType
	} else {
		return BlockPartitionType
	}
}

func (e *PerfSchemeEntry) getMetadataPartitionType() PartitionType {
	if e.StoreType == Filestore {
		return FilestoreJournalPartitionType
	} else {
		return DatabasePartitionType
	}
}

// Get the arguments necessary to create an sgdisk partition with the given parameters.
// number is the partition number.
// The offset and length are in MB. Under the covers this is translated to sectors.
// If the length is -1, all remaining space will be assigned to the
func getPartitionArgs(number int, guid string, offset, length int, label string) []string {
	const sectorsPerMB = 2048
	var newPart string
	if length == UseRemainingSpace {
		// The partition gets the remainder of the device
		newPart = fmt.Sprintf("--largest-new=%d", number)
	} else {
		// The partition has a specific offset and length
		newPart = fmt.Sprintf("--new=%d:%d:+%d", number, offset*sectorsPerMB, length*sectorsPerMB)
	}

	return []string{
		newPart,
		fmt.Sprintf("--change-name=%d:%s", number, label),
		fmt.Sprintf("--partition-guid=%d:%s", number, guid),
	}
}

func getPartitionArgsFromDetails(number int, partType PartitionType, id int, details *PerfSchemePartitionDetails) []string {
	return getPartitionArgs(number, details.PartitionUUID, details.OffsetMB, details.SizeMB, getPartitionLabel(id, partType))
}

func getPartitionLabel(id int, partType PartitionType) string {
	switch partType {
	case WalPartitionType:
		return fmt.Sprintf("ROOK-OSD%d-WAL", id)
	case DatabasePartitionType:
		return fmt.Sprintf("ROOK-OSD%d-DB", id)
	case BlockPartitionType:
		return fmt.Sprintf("ROOK-OSD%d-BLOCK", id)
	case FilestoreDataPartitionType:
		return fmt.Sprintf("ROOK-OSD%d-FS-DATA", id)
	case FilestoreJournalPartitionType:
		return fmt.Sprintf("ROOK-OSD%d-FS-JOURNAL", id)
	}

	return ""
}

func createBluestoreUUIDs() (*uuid.UUID, *uuid.UUID, *uuid.UUID, *uuid.UUID, error) {
	diskUUID, err := uuid.NewRandom()
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("failed to get disk uuid. %+v", err)
	}

	walUUID, err := uuid.NewRandom()
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("failed to get wal uuid. %+v", err)
	}

	dbUUID, err := uuid.NewRandom()
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("failed to get db uuid. %+v", err)
	}

	blockUUID, err := uuid.NewRandom()
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("failed to get block uuid. %+v", err)
	}

	return &diskUUID, &walUUID, &dbUUID, &blockUUID, nil
}

func createFilestoreUUIDs() (*uuid.UUID, *uuid.UUID, *uuid.UUID, error) {
	diskUUID, err := uuid.NewRandom()
	if err != nil {
		return nil, nil, nil, fmt.Errorf("failed to get disk uuid. %+v", err)
	}

	dataUUID, err := uuid.NewRandom()
	if err != nil {
		return nil, nil, nil, fmt.Errorf("failed to get data uuid. %+v", err)
	}

	journalUUID, err := uuid.NewRandom()
	if err != nil {
		return nil, nil, nil, fmt.Errorf("failed to get journal uuid. %+v", err)
	}

	return &diskUUID, &dataUUID, &journalUUID, nil
}