osd: create a default crushmap that is firefly compatible

This commit is contained in:
Jared Watts authored and Bassam Tabbara committed 2017-06-20 20:57:46 -07:00
1 parent fa3205fd98
commit d4c31e2799
12 files changed
+415 -119

No files matched your search

+1 -2
View File
@@ -25,7 +25,6 @@ import (
"github.com/prometheus/client_golang/prometheus"
ceph "github.com/rook/rook/pkg/ceph/client"
"github.com/rook/rook/pkg/ceph/mon"
"github.com/rook/rook/pkg/ceph/osd"
"github.com/rook/rook/pkg/clusterd"
)
@@ -83,7 +82,7 @@ type overallMonStatus struct {
// /crushmap
func (h *Handler) GetCrushMap(w http.ResponseWriter, r *http.Request) {
// get the crush map
crushmap, err := osd.GetCrushMap(h.context, h.config.ClusterInfo.Name)
crushmap, err := ceph.GetCrushMap(h.context, h.config.ClusterInfo.Name)
if err != nil {
logger.Errorf("failed to get crush map, err: %+v", err)
w.WriteHeader(http.StatusInternalServerError)
+192
View File
@@ -0,0 +1,192 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
"fmt"
"io/ioutil"
"os"
"regexp"
"strings"
"github.com/rook/rook/pkg/clusterd"
)
const defaultCrushMap = `# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable chooseleaf_vary_r 1
tunable chooseleaf_stable 0
tunable straw_calc_version 1
tunable allowed_bucket_algs 22
# types
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 region
type 10 root
# default bucket
root default {
id -1 # do not change unnecessarily
alg straw
hash 0 # rjenkins1
}
# rules
rule replicated_ruleset {
ruleset 0
type replicated
min_size 1
max_size 10
step take default
step chooseleaf firstn 0 type host
step emit
}
# end crush map
`
func GetCrushMap(context *clusterd.Context, clusterName string) (string, error) {
args := []string{"osd", "crush", "dump"}
buf, err := ExecuteCephCommand(context, clusterName, args)
if err != nil {
return "", fmt.Errorf("failed to get crush map. %v", err)
}
return string(buf), nil
}
func SetCrushMap(context *clusterd.Context, clusterName, compiledMap string) (string, error) {
args := []string{"osd", "setcrushmap", "-i", compiledMap}
buf, err := ExecuteCephCommand(context, clusterName, args)
if err != nil {
return string(buf), fmt.Errorf("failed to set compiled crushmap. %v", err)
}
return string(buf), nil
}
func SetCrushTunables(context *clusterd.Context, clusterName, profile string) (string, error) {
args := []string{"osd", "crush", "tunables", profile}
buf, err := ExecuteCephCommandPlain(context, clusterName, args)
if err != nil {
return "", fmt.Errorf("%+v, %s", err, string(buf))
}
return string(buf), nil
}
func CreateDefaultCrushMap(context *clusterd.Context, clusterName string) (string, error) {
// first set crush tunables to a firefly profile in order to support older clients
// (e.g., hyperkube uses a firefly rbd tool)
crushTunablesProfile := "firefly"
logger.Infof("setting crush tunables to %s", crushTunablesProfile)
output, err := SetCrushTunables(context, clusterName, crushTunablesProfile)
if err != nil {
return output, fmt.Errorf("failed to set crush tunables to profile %s: %+v", crushTunablesProfile, err)
} else {
logger.Infof("succeeded setting crush tunables to profile %s: %s", crushTunablesProfile, output)
}
// create a temp file that we will use to write the default decompiled crush map to
decompiledMap, err := ioutil.TempFile("", "")
if err != nil {
return "", fmt.Errorf("failed to open decompiled crush map temp file: %+v", err)
}
defer decompiledMap.Close()
defer os.Remove(decompiledMap.Name())
// write the default decompiled crush map to the temp file
_, err = decompiledMap.WriteString(defaultCrushMap)
if err != nil {
return "", fmt.Errorf("failed to write decompiled crush map to %s: %+v", decompiledMap.Name(), err)
}
// create a temp file to serve as the output file for the compilation process
compiledMap, err := ioutil.TempFile("", "")
if err != nil {
return "", fmt.Errorf("failed to open compiled crush map temp file: %+v", err)
}
defer compiledMap.Close()
defer os.Remove(compiledMap.Name())
// compile the crush map to an output file
args := []string{"-c", decompiledMap.Name(), "-o", compiledMap.Name()}
output, err = context.Executor.ExecuteCommandWithOutput("", CrushTool, args...)
if err != nil {
return output, fmt.Errorf("failed to compile crushmap from %s: %+v", decompiledMap.Name(), err)
}
// set the compiled crush map on the cluster
output, err = SetCrushMap(context, clusterName, compiledMap.Name())
if err != nil {
return output, fmt.Errorf("failed to set crushmap to %s: %+v", compiledMap.Name(), err)
}
return "", nil
}
func FormatLocation(location string) ([]string, error) {
var pairs []string
if location == "" {
pairs = []string{}
} else {
pairs = strings.Split(location, ",")
}
for _, p := range pairs {
if !isValidCrushFieldFormat(p) {
return nil, fmt.Errorf("CRUSH location field '%s' is not in a valid format", p)
}
}
// set a default root if it's not already set
if !isCrushFieldSet("root", pairs) {
pairs = append(pairs, formatProperty("root", "default"))
}
return pairs, nil
}
func isValidCrushFieldFormat(pair string) bool {
matched, err := regexp.MatchString("^.+=.+$", pair)
return matched && err == nil
}
func isCrushFieldSet(fieldName string, pairs []string) bool {
for _, p := range pairs {
kv := strings.Split(p, "=")
if len(kv) == 2 && kv[0] == fieldName && kv[1] != "" {
return true
}
}
return false
}
func formatProperty(name, value string) string {
return fmt.Sprintf("%s=%s", name, value)
}
+61
View File
@@ -0,0 +1,61 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
"testing"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/util"
exectest "github.com/rook/rook/pkg/util/exec/test"
"github.com/stretchr/testify/assert"
)
func TestGetCrushMap(t *testing.T) {
executor := &exectest.MockExecutor{}
response, err := GetCrushMap(&clusterd.Context{Executor: executor}, "rook")
assert.Nil(t, err)
assert.Equal(t, "", response)
}
func TestCrushLocation(t *testing.T) {
loc := "dc=datacenter1"
// test that root will get filled in with default/runtime values
res, err := FormatLocation(loc)
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
locSet := util.CreateSet(res)
assert.True(t, locSet.Contains("root=default"))
assert.True(t, locSet.Contains("dc=datacenter1"))
// test that if host name and root are already set they will be honored
loc = "root=otherRoot,dc=datacenter2,host=node123"
res, err = FormatLocation(loc)
assert.Nil(t, err)
assert.Equal(t, 3, len(res))
locSet = util.CreateSet(res)
assert.True(t, locSet.Contains("root=otherRoot"))
assert.True(t, locSet.Contains("dc=datacenter2"))
assert.True(t, locSet.Contains("host=node123"))
// test an invalid CRUSH location format
loc = "root=default,prop:value"
_, err = FormatLocation(loc)
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "is not in a valid format")
}
+1
View File
@@ -30,6 +30,7 @@ const (
AdminUsername = "client.admin"
CephTool = "ceph"
RBDTool = "rbd"
CrushTool = "crushtool"
)
// represents the response from a mon_status mon_command (subset of all available fields, only
-2
View File
@@ -69,7 +69,6 @@ type GlobalConfig struct {
OsdMaxObjectNamespaceLen int `ini:"osd max object namespace len,omitempty"`
OsdObjectStore string `ini:"osd objectstore"`
RbdDefaultFeatures int `ini:"rbd_default_features,omitempty"`
CrushtoolPath string `ini:"crushtool"`
FatalSignalHandlers string `ini:"fatal signal handlers"`
}
@@ -282,7 +281,6 @@ func CreateDefaultCephConfig(context *clusterd.Context, cluster *ClusterInfo, ru
OsdPoolDefaultPgpNum: 100,
OsdObjectStore: store,
RbdDefaultFeatures: 3,
CrushtoolPath: "",
FatalSignalHandlers: "false",
},
}
+43
View File
@@ -84,6 +84,11 @@ func (m *Leader) Configure(context *clusterd.Context, adminSecret string) error
return err
}
err = createInitialCrushMap(context, cluster)
if err != nil {
return err
}
if len(monitorsToRemove) == 0 {
// no monitors to remove
return nil
@@ -440,3 +445,41 @@ func WaitForQuorumWithMons(context *clusterd.Context, clusterName string, mons [
logger.Infof("Ceph monitors formed quorum")
return nil
}
func createInitialCrushMap(context *clusterd.Context, cluster *ClusterInfo) error {
createCrushMap := false
// check to see if the crush map has already been initialized
key := path.Join(CephKey, "crushMapInitialized")
resp, err := context.EtcdClient.Get(ctx.Background(), key, nil)
if err != nil {
if !util.IsEtcdKeyNotFound(err) {
return err
}
createCrushMap = true
} else {
if resp.Node.Value != "1" {
createCrushMap = true
}
}
if !createCrushMap {
// no need to create the crushmap, bail out
return nil
}
logger.Info("creating initial crushmap")
out, err := client.CreateDefaultCrushMap(context, cluster.Name)
if err != nil {
return fmt.Errorf("failed to create initial crushmap: %+v. output: %s", err, out)
}
logger.Info("created initial crushmap")
// save the fact that we've created the initial crush map so we don't do it again
if _, err := context.EtcdClient.Set(ctx.Background(), key, "1", nil); err != nil {
return err
}
return nil
}
+25
View File
@@ -149,6 +149,9 @@ func TestMoveUnhealthyMonitor(t *testing.T) {
Executor: executor,
}
// mock the initial crush map being created
etcdClient.SetValue("/rook/services/ceph/crushMapInitialized", "1")
// mock the agent responses that the deployments were successful to start mons and osds
etcdClient.WatcherResponses["/rook/_notify/a/monitor/status"] = "succeeded"
etcdClient.WatcherResponses["/rook/_notify/b/monitor/status"] = "succeeded"
@@ -283,3 +286,25 @@ func TestUnhealthyMon(t *testing.T) {
assert.Equal(t, 1, len(bad))
assert.Equal(t, cmonName, bad["c"].Name)
}
func TestCreateInitialCrushMap(t *testing.T) {
etcdClient := util.NewMockEtcdClient()
executor := &exectest.MockExecutor{}
context := &clusterd.Context{
DirectContext: clusterd.DirectContext{EtcdClient: etcdClient},
Executor: executor,
}
clusterInfo := &ClusterInfo{Name: "cluster30983"}
// calling the first time should result in a key being set that says we've initialized the crush map
err := createInitialCrushMap(context, clusterInfo)
assert.Nil(t, err)
assert.Equal(t, "1", etcdClient.GetValue("/rook/services/ceph/crushMapInitialized"))
// now call again, since we've already done it once we should not try to create the crushmap again
executor.MockExecuteCommandWithOutputFile = func(actionName string, command string, outFileArg string, args ...string) (string, error) {
return "", fmt.Errorf("crushmap was already created, we shouldn't be calling this again")
}
err = createInitialCrushMap(context, clusterInfo)
assert.Nil(t, err)
}
-78
View File
@@ -1,78 +0,0 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package osd
import (
"fmt"
"regexp"
"strings"
"github.com/rook/rook/pkg/ceph/client"
"github.com/rook/rook/pkg/clusterd"
)
// calls osd getcrushmap
func GetCrushMap(context *clusterd.Context, clusterName string) (string, error) {
args := []string{"osd", "crush", "dump"}
buf, err := client.ExecuteCephCommand(context, clusterName, args)
if err != nil {
return "", fmt.Errorf("failed to get crush map. %v", err)
}
return string(buf), nil
}
func formatLocation(location string) ([]string, error) {
var pairs []string
if location == "" {
pairs = []string{}
} else {
pairs = strings.Split(location, ",")
}
for _, p := range pairs {
if !isValidCrushFieldFormat(p) {
return nil, fmt.Errorf("CRUSH location field '%s' is not in a valid format", p)
}
}
// set a default root if it's not already set
if !isCrushFieldSet("root", pairs) {
pairs = append(pairs, formatProperty("root", "default"))
}
return pairs, nil
}
func isValidCrushFieldFormat(pair string) bool {
matched, err := regexp.MatchString("^.+=.+$", pair)
return matched && err == nil
}
func isCrushFieldSet(fieldName string, pairs []string) bool {
for _, p := range pairs {
kv := strings.Split(p, "=")
if len(kv) == 2 && kv[0] == fieldName && kv[1] != "" {
return true
}
}
return false
}
func formatProperty(name, value string) string {
return fmt.Sprintf("%s=%s", name, value)
}
-36
View File
@@ -74,39 +74,3 @@ func testCrushMapHelper(t *testing.T, storeConfig *StoreConfig) {
// location should have been stored in etcd as well
assert.Equal(t, location, etcdClient.GetValue("/rook/nodes/config/node1/location"))
}
func TestGetCrushMap(t *testing.T) {
executor := &exectest.MockExecutor{}
response, err := GetCrushMap(&clusterd.Context{Executor: executor}, "rook")
assert.Nil(t, err)
assert.Equal(t, "", response)
}
func TestCrushLocation(t *testing.T) {
loc := "dc=datacenter1"
// test that root will get filled in with default/runtime values
res, err := formatLocation(loc)
assert.Nil(t, err)
assert.Equal(t, 2, len(res))
locSet := util.CreateSet(res)
assert.True(t, locSet.Contains("root=default"))
assert.True(t, locSet.Contains("dc=datacenter1"))
// test that if host name and root are already set they will be honored
loc = "root=otherRoot,dc=datacenter2,host=node123"
res, err = formatLocation(loc)
assert.Nil(t, err)
assert.Equal(t, 3, len(res))
locSet = util.CreateSet(res)
assert.True(t, locSet.Contains("root=otherRoot"))
assert.True(t, locSet.Contains("dc=datacenter2"))
assert.True(t, locSet.Contains("host=node123"))
// test an invalid CRUSH location format
loc = "root=default,prop:value"
_, err = formatLocation(loc)
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "is not in a valid format")
}
+1 -1
View File
@@ -604,7 +604,7 @@ func addOSDToCrushMap(context *clusterd.Context, config *osdConfig, clusterName,
weight, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", weight), 64)
osdEntity := fmt.Sprintf("osd.%d", osdID)
locArgs, err := formatLocation(location)
locArgs, err := client.FormatLocation(location)
if err != nil {
return err
}
+68
View File
@@ -40,6 +40,11 @@ import (
"k8s.io/client-go/pkg/api/v1"
)
const (
crushConfigMapName = "crush-config"
crushmapCreatedKey = "initialCrushMapCreated"
)
var (
healthCheckInterval = 10 * time.Second
clientTimeout = 15 * time.Second
@@ -92,6 +97,11 @@ func (c *Cluster) CreateInstance() error {
return fmt.Errorf("failed to start the mons. %+v", err)
}
err = c.createInitialCrushMap()
if err != nil {
return fmt.Errorf("failed to create initial crushmap: %+v", err)
}
c.mgrs = mgr.New(c.context, c.Namespace, c.Spec.VersionTag)
err = c.mgrs.Start()
if err != nil {
@@ -137,6 +147,64 @@ func (c *Cluster) Monitor(stopCh <-chan struct{}) {
}
}
func (c *Cluster) createInitialCrushMap() error {
configMapExists := false
createCrushMap := false
cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(crushConfigMapName, metav1.GetOptions{})
if err != nil {
if !errors.IsNotFound(err) {
return err
}
// crush config map was not found, meaning we haven't created the initial crush map
createCrushMap = true
} else {
// crush config map was found, look in it to verify we've created the initial crush map
configMapExists = true
val, ok := cm.Data[crushmapCreatedKey]
if !ok {
createCrushMap = true
} else if val != "1" {
createCrushMap = true
}
}
if !createCrushMap {
// no need to create the crushmap, bail out
return nil
}
logger.Info("creating initial crushmap")
out, err := client.CreateDefaultCrushMap(c.context, c.Namespace)
if err != nil {
return fmt.Errorf("failed to create initial crushmap: %+v. output: %s", err, out)
}
logger.Info("created initial crushmap")
// save the fact that we've created the initial crushmap to a configmap
configMap := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: crushConfigMapName,
Namespace: c.Namespace,
},
Data: map[string]string{crushmapCreatedKey: "1"},
}
if !configMapExists {
if _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(configMap); err != nil {
return fmt.Errorf("failed to create configmap %s: %+v", crushConfigMapName, err)
}
} else {
if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(configMap); err != nil {
return fmt.Errorf("failed to update configmap %s: %+v", crushConfigMapName, err)
}
}
return nil
}
func (c *Cluster) createClientAccess(clusterInfo *cephmon.ClusterInfo) error {
// create a user for rbd clients
name := fmt.Sprintf("%s-rook-user", c.Namespace)
+23
View File
@@ -57,3 +57,26 @@ func TestCreateSecrets(t *testing.T) {
assert.Equal(t, secretName, secret.Name)
assert.Equal(t, 1, len(secret.StringData))
}
func TestCreateInitialCrushMap(t *testing.T) {
clientset := testop.New(3)
executor := &exectest.MockExecutor{}
c := &Cluster{}
c.Namespace = "rook294"
c.Init(&clusterd.Context{KubeContext: clusterd.KubeContext{Clientset: clientset}, Executor: executor})
// create the initial crush map and verify that a configmap value was created that says the crush map was created
err := c.createInitialCrushMap()
assert.Nil(t, err)
cm, err := clientset.CoreV1().ConfigMaps(c.Namespace).Get(crushConfigMapName, metav1.GetOptions{})
assert.Nil(t, err)
assert.NotNil(t, cm)
assert.Equal(t, "1", cm.Data[crushmapCreatedKey])
// the crushmap should NOT get created again
executor.MockExecuteCommandWithOutputFile = func(actionName string, command string, outFileArg string, args ...string) (string, error) {
return "", fmt.Errorf("crushmap was already created, we shouldn't be calling this again")
}
err = c.createInitialCrushMap()
assert.Nil(t, err)
}