From e2155afa0fefa8b10490aebe31ef781d959bd177 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Tue, 8 Oct 2019 13:30:40 -0600 Subject: [PATCH] ceph: osd: use mon DB config for dir-based OSDs As with the rest of the Ceph daemons, use the new config patters for dir-based OSDs. Do not generate a config to use in an init container; instead use CLI flags and the mon config database. Signed-off-by: Blaine Gardner --- PendingReleaseNotes.md | 2 ++ pkg/operator/ceph/cluster/osd/osd_test.go | 12 +++++++- pkg/operator/ceph/cluster/osd/spec.go | 35 +++++++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 7ee4aad22..55f9e0310 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -15,6 +15,8 @@ - Ceph OSD's admin socket is now placed under Ceph's default system location `/run/ceph`. - The on-host log directory for OSDs was set incorrectly to `//log`; fix this to be `/log/`, the same as other daemons. + - Use the mon configuration database for directory-based OSDs, and do not generate a config + ### EdgeFS diff --git a/pkg/operator/ceph/cluster/osd/osd_test.go b/pkg/operator/ceph/cluster/osd/osd_test.go index 06076f5fb..544aacf0a 100644 --- a/pkg/operator/ceph/cluster/osd/osd_test.go +++ b/pkg/operator/ceph/cluster/osd/osd_test.go @@ -251,7 +251,17 @@ func TestDiscoverOSDs(t *testing.T) { clusterInfo := &cephconfig.ClusterInfo{ CephVersion: cephver.Nautilus, } - c := New(clusterInfo, &clusterd.Context{}, "ns", "myversion", cephv1.CephVersionSpec{}, + executor := &exectest.MockExecutor{ + MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) { + logger.Infof("Command: %s %v", command, args) + // very simple stub that just reports success + return "", nil + }, + } + context := &clusterd.Context{ + Executor: executor, + } + c := New(clusterInfo, context, "ns", "myversion", cephv1.CephVersionSpec{}, rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, rookalpha.Annotations{}, cephv1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{}, false, false) node1 := "n1" node2 := "n2" diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 4c28c2140..3ae245f72 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -27,6 +27,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + cephclient "github.com/rook/rook/pkg/daemon/client" opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" opconfig "github.com/rook/rook/pkg/operator/ceph/config" @@ -265,6 +266,32 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo) (*apps.Dep {Name: "CEPH_VOLUME_DEBUG", Value: "1"}, }...) + } else if osd.IsDirectory { + command = []string{"ceph-osd"} + args = opconfig.DefaultFlags(c.clusterInfo.FSID, path.Join(osd.DataPath, "keyring"), c.clusterInfo.CephVersion) + args = append(args, + "--foreground", + "--id", osdID, + "--osd-data", osd.DataPath, + "--osd-uuid", osd.UUID, + "--setuser", "ceph", + "--setgroup", "ceph", + "--osd-objectstore", "filestore", /* always? */ + "--osd-max-object-name-len", "256", + "--osd-max-object-namespace-len", "64", + ) + + // Set configs in mon cfg database + monstore := opconfig.GetMonStore(c.context, c.Namespace) + who := fmt.Sprintf("osd.%s", osdID) + // crush_location + loc, err := getCRUSHLocation(osdProps) + if err != nil { + return nil, fmt.Errorf("failed to determine CRUSH location for osd.%s. %+v", osdID, err) + } + if err := monstore.Set(who, "crush_location", loc); err != nil { + return nil, fmt.Errorf("failed to set CRUSH location for osd.%s. %+v", osdID, err) + } } else { // other osds can launch the osd daemon directly command = []string{"ceph-osd"} @@ -821,3 +848,11 @@ func (c *Cluster) getOSDLabels(osdID int, failureDomainValue string, portable bo portableKey: strconv.FormatBool(portable), } } + +func getCRUSHLocation(p osdProperties) (string, error) { + locs, err := cephclient.FormatLocation(p.location, p.crushHostname) + if err != nil { + return "", err + } + return strings.Join(locs, " "), nil +}