diff --git a/pkg/api/handlers.go b/pkg/api/handlers.go index 74f4a4644..11bdbc895 100644 --- a/pkg/api/handlers.go +++ b/pkg/api/handlers.go @@ -25,7 +25,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ceph "github.com/rook/rook/pkg/ceph/client" "github.com/rook/rook/pkg/ceph/mon" - "github.com/rook/rook/pkg/ceph/osd" "github.com/rook/rook/pkg/clusterd" ) @@ -83,7 +82,7 @@ type overallMonStatus struct { // /crushmap func (h *Handler) GetCrushMap(w http.ResponseWriter, r *http.Request) { // get the crush map - crushmap, err := osd.GetCrushMap(h.context, h.config.ClusterInfo.Name) + crushmap, err := ceph.GetCrushMap(h.context, h.config.ClusterInfo.Name) if err != nil { logger.Errorf("failed to get crush map, err: %+v", err) w.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/ceph/client/crush.go b/pkg/ceph/client/crush.go new file mode 100644 index 000000000..17a689e84 --- /dev/null +++ b/pkg/ceph/client/crush.go @@ -0,0 +1,192 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package client + +import ( + "fmt" + "io/ioutil" + "os" + "regexp" + "strings" + + "github.com/rook/rook/pkg/clusterd" +) + +const defaultCrushMap = `# begin crush map +tunable choose_local_tries 0 +tunable choose_local_fallback_tries 0 +tunable choose_total_tries 50 +tunable chooseleaf_descend_once 1 +tunable chooseleaf_vary_r 1 +tunable chooseleaf_stable 0 +tunable straw_calc_version 1 +tunable allowed_bucket_algs 22 + +# types +type 0 osd +type 1 host +type 2 chassis +type 3 rack +type 4 row +type 5 pdu +type 6 pod +type 7 room +type 8 datacenter +type 9 region +type 10 root + +# default bucket +root default { + id -1 # do not change unnecessarily + alg straw + hash 0 # rjenkins1 +} + +# rules +rule replicated_ruleset { + ruleset 0 + type replicated + min_size 1 + max_size 10 + step take default + step chooseleaf firstn 0 type host + step emit +} + +# end crush map +` + +func GetCrushMap(context *clusterd.Context, clusterName string) (string, error) { + args := []string{"osd", "crush", "dump"} + buf, err := ExecuteCephCommand(context, clusterName, args) + if err != nil { + return "", fmt.Errorf("failed to get crush map. %v", err) + } + + return string(buf), nil +} + +func SetCrushMap(context *clusterd.Context, clusterName, compiledMap string) (string, error) { + args := []string{"osd", "setcrushmap", "-i", compiledMap} + buf, err := ExecuteCephCommand(context, clusterName, args) + if err != nil { + return string(buf), fmt.Errorf("failed to set compiled crushmap. %v", err) + } + + return string(buf), nil +} + +func SetCrushTunables(context *clusterd.Context, clusterName, profile string) (string, error) { + args := []string{"osd", "crush", "tunables", profile} + buf, err := ExecuteCephCommandPlain(context, clusterName, args) + if err != nil { + return "", fmt.Errorf("%+v, %s", err, string(buf)) + } + + return string(buf), nil +} + +func CreateDefaultCrushMap(context *clusterd.Context, clusterName string) (string, error) { + // first set crush tunables to a firefly profile in order to support older clients + // (e.g., hyperkube uses a firefly rbd tool) + crushTunablesProfile := "firefly" + logger.Infof("setting crush tunables to %s", crushTunablesProfile) + output, err := SetCrushTunables(context, clusterName, crushTunablesProfile) + if err != nil { + return output, fmt.Errorf("failed to set crush tunables to profile %s: %+v", crushTunablesProfile, err) + } else { + logger.Infof("succeeded setting crush tunables to profile %s: %s", crushTunablesProfile, output) + } + + // create a temp file that we will use to write the default decompiled crush map to + decompiledMap, err := ioutil.TempFile("", "") + if err != nil { + return "", fmt.Errorf("failed to open decompiled crush map temp file: %+v", err) + } + defer decompiledMap.Close() + defer os.Remove(decompiledMap.Name()) + + // write the default decompiled crush map to the temp file + _, err = decompiledMap.WriteString(defaultCrushMap) + if err != nil { + return "", fmt.Errorf("failed to write decompiled crush map to %s: %+v", decompiledMap.Name(), err) + } + + // create a temp file to serve as the output file for the compilation process + compiledMap, err := ioutil.TempFile("", "") + if err != nil { + return "", fmt.Errorf("failed to open compiled crush map temp file: %+v", err) + } + defer compiledMap.Close() + defer os.Remove(compiledMap.Name()) + + // compile the crush map to an output file + args := []string{"-c", decompiledMap.Name(), "-o", compiledMap.Name()} + output, err = context.Executor.ExecuteCommandWithOutput("", CrushTool, args...) + if err != nil { + return output, fmt.Errorf("failed to compile crushmap from %s: %+v", decompiledMap.Name(), err) + } + + // set the compiled crush map on the cluster + output, err = SetCrushMap(context, clusterName, compiledMap.Name()) + if err != nil { + return output, fmt.Errorf("failed to set crushmap to %s: %+v", compiledMap.Name(), err) + } + + return "", nil +} + +func FormatLocation(location string) ([]string, error) { + var pairs []string + if location == "" { + pairs = []string{} + } else { + pairs = strings.Split(location, ",") + } + + for _, p := range pairs { + if !isValidCrushFieldFormat(p) { + return nil, fmt.Errorf("CRUSH location field '%s' is not in a valid format", p) + } + } + + // set a default root if it's not already set + if !isCrushFieldSet("root", pairs) { + pairs = append(pairs, formatProperty("root", "default")) + } + + return pairs, nil +} + +func isValidCrushFieldFormat(pair string) bool { + matched, err := regexp.MatchString("^.+=.+$", pair) + return matched && err == nil +} + +func isCrushFieldSet(fieldName string, pairs []string) bool { + for _, p := range pairs { + kv := strings.Split(p, "=") + if len(kv) == 2 && kv[0] == fieldName && kv[1] != "" { + return true + } + } + + return false +} + +func formatProperty(name, value string) string { + return fmt.Sprintf("%s=%s", name, value) +} diff --git a/pkg/ceph/client/crush_test.go b/pkg/ceph/client/crush_test.go new file mode 100644 index 000000000..aa26cebc8 --- /dev/null +++ b/pkg/ceph/client/crush_test.go @@ -0,0 +1,61 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package client + +import ( + "testing" + + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/util" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" +) + +func TestGetCrushMap(t *testing.T) { + executor := &exectest.MockExecutor{} + response, err := GetCrushMap(&clusterd.Context{Executor: executor}, "rook") + + assert.Nil(t, err) + assert.Equal(t, "", response) +} + +func TestCrushLocation(t *testing.T) { + loc := "dc=datacenter1" + + // test that root will get filled in with default/runtime values + res, err := FormatLocation(loc) + assert.Nil(t, err) + assert.Equal(t, 2, len(res)) + locSet := util.CreateSet(res) + assert.True(t, locSet.Contains("root=default")) + assert.True(t, locSet.Contains("dc=datacenter1")) + + // test that if host name and root are already set they will be honored + loc = "root=otherRoot,dc=datacenter2,host=node123" + res, err = FormatLocation(loc) + assert.Nil(t, err) + assert.Equal(t, 3, len(res)) + locSet = util.CreateSet(res) + assert.True(t, locSet.Contains("root=otherRoot")) + assert.True(t, locSet.Contains("dc=datacenter2")) + assert.True(t, locSet.Contains("host=node123")) + + // test an invalid CRUSH location format + loc = "root=default,prop:value" + _, err = FormatLocation(loc) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "is not in a valid format") +} diff --git a/pkg/ceph/client/mon.go b/pkg/ceph/client/mon.go index e5833f0a6..c5a6d3e37 100644 --- a/pkg/ceph/client/mon.go +++ b/pkg/ceph/client/mon.go @@ -30,6 +30,7 @@ const ( AdminUsername = "client.admin" CephTool = "ceph" RBDTool = "rbd" + CrushTool = "crushtool" ) // represents the response from a mon_status mon_command (subset of all available fields, only diff --git a/pkg/ceph/mon/config.go b/pkg/ceph/mon/config.go index b547628e3..36ccd6a4e 100644 --- a/pkg/ceph/mon/config.go +++ b/pkg/ceph/mon/config.go @@ -69,7 +69,6 @@ type GlobalConfig struct { OsdMaxObjectNamespaceLen int `ini:"osd max object namespace len,omitempty"` OsdObjectStore string `ini:"osd objectstore"` RbdDefaultFeatures int `ini:"rbd_default_features,omitempty"` - CrushtoolPath string `ini:"crushtool"` FatalSignalHandlers string `ini:"fatal signal handlers"` } @@ -282,7 +281,6 @@ func CreateDefaultCephConfig(context *clusterd.Context, cluster *ClusterInfo, ru OsdPoolDefaultPgpNum: 100, OsdObjectStore: store, RbdDefaultFeatures: 3, - CrushtoolPath: "", FatalSignalHandlers: "false", }, } diff --git a/pkg/ceph/mon/leader.go b/pkg/ceph/mon/leader.go index 97862c874..189b57f16 100644 --- a/pkg/ceph/mon/leader.go +++ b/pkg/ceph/mon/leader.go @@ -84,6 +84,11 @@ func (m *Leader) Configure(context *clusterd.Context, adminSecret string) error return err } + err = createInitialCrushMap(context, cluster) + if err != nil { + return err + } + if len(monitorsToRemove) == 0 { // no monitors to remove return nil @@ -440,3 +445,41 @@ func WaitForQuorumWithMons(context *clusterd.Context, clusterName string, mons [ logger.Infof("Ceph monitors formed quorum") return nil } + +func createInitialCrushMap(context *clusterd.Context, cluster *ClusterInfo) error { + createCrushMap := false + + // check to see if the crush map has already been initialized + key := path.Join(CephKey, "crushMapInitialized") + resp, err := context.EtcdClient.Get(ctx.Background(), key, nil) + if err != nil { + if !util.IsEtcdKeyNotFound(err) { + return err + } + createCrushMap = true + } else { + if resp.Node.Value != "1" { + createCrushMap = true + } + } + + if !createCrushMap { + // no need to create the crushmap, bail out + return nil + } + + logger.Info("creating initial crushmap") + out, err := client.CreateDefaultCrushMap(context, cluster.Name) + if err != nil { + return fmt.Errorf("failed to create initial crushmap: %+v. output: %s", err, out) + } + + logger.Info("created initial crushmap") + + // save the fact that we've created the initial crush map so we don't do it again + if _, err := context.EtcdClient.Set(ctx.Background(), key, "1", nil); err != nil { + return err + } + + return nil +} diff --git a/pkg/ceph/mon/leader_test.go b/pkg/ceph/mon/leader_test.go index f8092aa14..3356ab955 100644 --- a/pkg/ceph/mon/leader_test.go +++ b/pkg/ceph/mon/leader_test.go @@ -149,6 +149,9 @@ func TestMoveUnhealthyMonitor(t *testing.T) { Executor: executor, } + // mock the initial crush map being created + etcdClient.SetValue("/rook/services/ceph/crushMapInitialized", "1") + // mock the agent responses that the deployments were successful to start mons and osds etcdClient.WatcherResponses["/rook/_notify/a/monitor/status"] = "succeeded" etcdClient.WatcherResponses["/rook/_notify/b/monitor/status"] = "succeeded" @@ -283,3 +286,25 @@ func TestUnhealthyMon(t *testing.T) { assert.Equal(t, 1, len(bad)) assert.Equal(t, cmonName, bad["c"].Name) } + +func TestCreateInitialCrushMap(t *testing.T) { + etcdClient := util.NewMockEtcdClient() + executor := &exectest.MockExecutor{} + context := &clusterd.Context{ + DirectContext: clusterd.DirectContext{EtcdClient: etcdClient}, + Executor: executor, + } + clusterInfo := &ClusterInfo{Name: "cluster30983"} + + // calling the first time should result in a key being set that says we've initialized the crush map + err := createInitialCrushMap(context, clusterInfo) + assert.Nil(t, err) + assert.Equal(t, "1", etcdClient.GetValue("/rook/services/ceph/crushMapInitialized")) + + // now call again, since we've already done it once we should not try to create the crushmap again + executor.MockExecuteCommandWithOutputFile = func(actionName string, command string, outFileArg string, args ...string) (string, error) { + return "", fmt.Errorf("crushmap was already created, we shouldn't be calling this again") + } + err = createInitialCrushMap(context, clusterInfo) + assert.Nil(t, err) +} diff --git a/pkg/ceph/osd/crushmap.go b/pkg/ceph/osd/crushmap.go deleted file mode 100644 index f28563b59..000000000 --- a/pkg/ceph/osd/crushmap.go +++ /dev/null @@ -1,78 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package osd - -import ( - "fmt" - "regexp" - "strings" - - "github.com/rook/rook/pkg/ceph/client" - "github.com/rook/rook/pkg/clusterd" -) - -// calls osd getcrushmap -func GetCrushMap(context *clusterd.Context, clusterName string) (string, error) { - args := []string{"osd", "crush", "dump"} - buf, err := client.ExecuteCephCommand(context, clusterName, args) - if err != nil { - return "", fmt.Errorf("failed to get crush map. %v", err) - } - - return string(buf), nil -} - -func formatLocation(location string) ([]string, error) { - var pairs []string - if location == "" { - pairs = []string{} - } else { - pairs = strings.Split(location, ",") - } - - for _, p := range pairs { - if !isValidCrushFieldFormat(p) { - return nil, fmt.Errorf("CRUSH location field '%s' is not in a valid format", p) - } - } - - // set a default root if it's not already set - if !isCrushFieldSet("root", pairs) { - pairs = append(pairs, formatProperty("root", "default")) - } - - return pairs, nil -} - -func isValidCrushFieldFormat(pair string) bool { - matched, err := regexp.MatchString("^.+=.+$", pair) - return matched && err == nil -} - -func isCrushFieldSet(fieldName string, pairs []string) bool { - for _, p := range pairs { - kv := strings.Split(p, "=") - if len(kv) == 2 && kv[0] == fieldName && kv[1] != "" { - return true - } - } - - return false -} - -func formatProperty(name, value string) string { - return fmt.Sprintf("%s=%s", name, value) -} diff --git a/pkg/ceph/osd/crushmap_test.go b/pkg/ceph/osd/crushmap_test.go index 757c8829f..ded48ea92 100644 --- a/pkg/ceph/osd/crushmap_test.go +++ b/pkg/ceph/osd/crushmap_test.go @@ -74,39 +74,3 @@ func testCrushMapHelper(t *testing.T, storeConfig *StoreConfig) { // location should have been stored in etcd as well assert.Equal(t, location, etcdClient.GetValue("/rook/nodes/config/node1/location")) } - -func TestGetCrushMap(t *testing.T) { - executor := &exectest.MockExecutor{} - response, err := GetCrushMap(&clusterd.Context{Executor: executor}, "rook") - - assert.Nil(t, err) - assert.Equal(t, "", response) -} - -func TestCrushLocation(t *testing.T) { - loc := "dc=datacenter1" - - // test that root will get filled in with default/runtime values - res, err := formatLocation(loc) - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - locSet := util.CreateSet(res) - assert.True(t, locSet.Contains("root=default")) - assert.True(t, locSet.Contains("dc=datacenter1")) - - // test that if host name and root are already set they will be honored - loc = "root=otherRoot,dc=datacenter2,host=node123" - res, err = formatLocation(loc) - assert.Nil(t, err) - assert.Equal(t, 3, len(res)) - locSet = util.CreateSet(res) - assert.True(t, locSet.Contains("root=otherRoot")) - assert.True(t, locSet.Contains("dc=datacenter2")) - assert.True(t, locSet.Contains("host=node123")) - - // test an invalid CRUSH location format - loc = "root=default,prop:value" - _, err = formatLocation(loc) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), "is not in a valid format") -} diff --git a/pkg/ceph/osd/device.go b/pkg/ceph/osd/device.go index c1d5598ff..010b80940 100644 --- a/pkg/ceph/osd/device.go +++ b/pkg/ceph/osd/device.go @@ -604,7 +604,7 @@ func addOSDToCrushMap(context *clusterd.Context, config *osdConfig, clusterName, weight, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", weight), 64) osdEntity := fmt.Sprintf("osd.%d", osdID) - locArgs, err := formatLocation(location) + locArgs, err := client.FormatLocation(location) if err != nil { return err } diff --git a/pkg/operator/cluster/cluster.go b/pkg/operator/cluster/cluster.go index 3d412bc9b..3406a1fdd 100644 --- a/pkg/operator/cluster/cluster.go +++ b/pkg/operator/cluster/cluster.go @@ -40,6 +40,11 @@ import ( "k8s.io/client-go/pkg/api/v1" ) +const ( + crushConfigMapName = "crush-config" + crushmapCreatedKey = "initialCrushMapCreated" +) + var ( healthCheckInterval = 10 * time.Second clientTimeout = 15 * time.Second @@ -92,6 +97,11 @@ func (c *Cluster) CreateInstance() error { return fmt.Errorf("failed to start the mons. %+v", err) } + err = c.createInitialCrushMap() + if err != nil { + return fmt.Errorf("failed to create initial crushmap: %+v", err) + } + c.mgrs = mgr.New(c.context, c.Namespace, c.Spec.VersionTag) err = c.mgrs.Start() if err != nil { @@ -137,6 +147,64 @@ func (c *Cluster) Monitor(stopCh <-chan struct{}) { } } +func (c *Cluster) createInitialCrushMap() error { + configMapExists := false + createCrushMap := false + + cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(crushConfigMapName, metav1.GetOptions{}) + if err != nil { + if !errors.IsNotFound(err) { + return err + } + + // crush config map was not found, meaning we haven't created the initial crush map + createCrushMap = true + } else { + // crush config map was found, look in it to verify we've created the initial crush map + configMapExists = true + val, ok := cm.Data[crushmapCreatedKey] + if !ok { + createCrushMap = true + } else if val != "1" { + createCrushMap = true + } + } + + if !createCrushMap { + // no need to create the crushmap, bail out + return nil + } + + logger.Info("creating initial crushmap") + out, err := client.CreateDefaultCrushMap(c.context, c.Namespace) + if err != nil { + return fmt.Errorf("failed to create initial crushmap: %+v. output: %s", err, out) + } + + logger.Info("created initial crushmap") + + // save the fact that we've created the initial crushmap to a configmap + configMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: crushConfigMapName, + Namespace: c.Namespace, + }, + Data: map[string]string{crushmapCreatedKey: "1"}, + } + + if !configMapExists { + if _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(configMap); err != nil { + return fmt.Errorf("failed to create configmap %s: %+v", crushConfigMapName, err) + } + } else { + if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(configMap); err != nil { + return fmt.Errorf("failed to update configmap %s: %+v", crushConfigMapName, err) + } + } + + return nil +} + func (c *Cluster) createClientAccess(clusterInfo *cephmon.ClusterInfo) error { // create a user for rbd clients name := fmt.Sprintf("%s-rook-user", c.Namespace) diff --git a/pkg/operator/cluster/cluster_test.go b/pkg/operator/cluster/cluster_test.go index 7e2376099..118237b91 100644 --- a/pkg/operator/cluster/cluster_test.go +++ b/pkg/operator/cluster/cluster_test.go @@ -57,3 +57,26 @@ func TestCreateSecrets(t *testing.T) { assert.Equal(t, secretName, secret.Name) assert.Equal(t, 1, len(secret.StringData)) } + +func TestCreateInitialCrushMap(t *testing.T) { + clientset := testop.New(3) + executor := &exectest.MockExecutor{} + c := &Cluster{} + c.Namespace = "rook294" + c.Init(&clusterd.Context{KubeContext: clusterd.KubeContext{Clientset: clientset}, Executor: executor}) + + // create the initial crush map and verify that a configmap value was created that says the crush map was created + err := c.createInitialCrushMap() + assert.Nil(t, err) + cm, err := clientset.CoreV1().ConfigMaps(c.Namespace).Get(crushConfigMapName, metav1.GetOptions{}) + assert.Nil(t, err) + assert.NotNil(t, cm) + assert.Equal(t, "1", cm.Data[crushmapCreatedKey]) + + // the crushmap should NOT get created again + executor.MockExecuteCommandWithOutputFile = func(actionName string, command string, outFileArg string, args ...string) (string, error) { + return "", fmt.Errorf("crushmap was already created, we shouldn't be calling this again") + } + err = c.createInitialCrushMap() + assert.Nil(t, err) +}