From b49b6393bc80267bf691730f8eb7e0e53eadb3f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Wed, 7 Oct 2020 16:29:25 +0200 Subject: [PATCH] ceph: tune osd tcmalloc thread cache bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can now tune the value of TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES by setting an annotation on the CephCluster CRD like so: ``` annotation: osd: TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "134217728" ``` Alternatively, if nothing is specified Rook will read the value of the file `/etc/sysconfig/ceph`. Closes: https://github.com/rook/rook/issues/6134 Signed-off-by: Sébastien Han --- go.mod | 2 +- pkg/operator/ceph/cluster/osd/envs.go | 43 ++++++++++++++++---- pkg/operator/ceph/cluster/osd/envs_test.go | 47 ++++++++++++++++++++++ pkg/operator/ceph/cluster/osd/spec.go | 8 ++++ pkg/operator/ceph/cluster/osd/spec_test.go | 8 +++- 5 files changed, 98 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 888dcac0a..c2f51d984 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( golang.org/x/text v0.3.3 // indirect golang.org/x/tools v0.0.0-20200319210407-521f4a0cd458 // indirect google.golang.org/grpc v1.26.0 // indirect - gopkg.in/ini.v1 v1.51.1 // indirect + gopkg.in/ini.v1 v1.51.1 k8s.io/api v0.17.2 k8s.io/apiextensions-apiserver v0.17.2 k8s.io/apimachinery v0.17.2 diff --git a/pkg/operator/ceph/cluster/osd/envs.go b/pkg/operator/ceph/cluster/osd/envs.go index 2661bc783..93a887663 100644 --- a/pkg/operator/ceph/cluster/osd/envs.go +++ b/pkg/operator/ceph/cluster/osd/envs.go @@ -22,6 +22,7 @@ import ( "github.com/pkg/errors" opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/k8sutil" + "gopkg.in/ini.v1" v1 "k8s.io/api/core/v1" ) @@ -33,14 +34,19 @@ const ( EncryptedDeviceEnvVarName = "ROOK_ENCRYPTED_DEVICE" // CephVolumeEncryptedKeyEnvVarName is the env variable used by ceph-volume to encrypt the OSD (raw mode) // Hardcoded in ceph-volume do NOT touch - CephVolumeEncryptedKeyEnvVarName = "CEPH_VOLUME_DMCRYPT_SECRET" - osdMetadataDeviceEnvVarName = "ROOK_METADATA_DEVICE" - osdWalDeviceEnvVarName = "ROOK_WAL_DEVICE" - pvcBackedOSDVarName = "ROOK_PVC_BACKED_OSD" - blockPathVarName = "ROOK_BLOCK_PATH" - cvModeVarName = "ROOK_CV_MODE" - lvBackedPVVarName = "ROOK_LV_BACKED_PV" - CrushDeviceClassVarName = "ROOK_OSD_CRUSH_DEVICE_CLASS" + CephVolumeEncryptedKeyEnvVarName = "CEPH_VOLUME_DMCRYPT_SECRET" + osdMetadataDeviceEnvVarName = "ROOK_METADATA_DEVICE" + osdWalDeviceEnvVarName = "ROOK_WAL_DEVICE" + pvcBackedOSDVarName = "ROOK_PVC_BACKED_OSD" + blockPathVarName = "ROOK_BLOCK_PATH" + cvModeVarName = "ROOK_CV_MODE" + lvBackedPVVarName = "ROOK_LV_BACKED_PV" + CrushDeviceClassVarName = "ROOK_OSD_CRUSH_DEVICE_CLASS" + tcmallocMaxTotalThreadCacheBytesEnv = "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES" +) + +var ( + cephEnvConfigFile = "/etc/sysconfig/ceph" ) func (c *Cluster) getConfigEnvVars(osdProps osdProperties, dataDir string) []v1.EnvVar { @@ -196,3 +202,24 @@ func osdActivateEnvVar() []v1.EnvVar { return append(cephVolumeEnvVar(), monEnvVars...) } + +func getTcmallocMaxTotalThreadCacheBytes(tcmallocMaxTotalThreadCacheBytes string) v1.EnvVar { + var value string + // If empty we read the default value from the file coming with the package + if tcmallocMaxTotalThreadCacheBytes == "" { + value = getTcmallocMaxTotalThreadCacheBytesFromFile() + } else { + value = tcmallocMaxTotalThreadCacheBytes + } + + return v1.EnvVar{Name: tcmallocMaxTotalThreadCacheBytesEnv, Value: value} +} + +func getTcmallocMaxTotalThreadCacheBytesFromFile() string { + iniCephEnvConfigFile, err := ini.Load(cephEnvConfigFile) + if err != nil { + return "" + } + + return iniCephEnvConfigFile.Section("").Key(tcmallocMaxTotalThreadCacheBytesEnv).String() +} diff --git a/pkg/operator/ceph/cluster/osd/envs_test.go b/pkg/operator/ceph/cluster/osd/envs_test.go index 13a28fa29..600935c83 100644 --- a/pkg/operator/ceph/cluster/osd/envs_test.go +++ b/pkg/operator/ceph/cluster/osd/envs_test.go @@ -17,11 +17,34 @@ limitations under the License. package osd import ( + "io/ioutil" + "os" "testing" "github.com/stretchr/testify/assert" ) +var ( + sysconfig = []byte(`# /etc/sysconfig/ceph +# +# Environment file for ceph daemon systemd unit files. +# + +# Increase tcmalloc cache size +TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728 + +## automatically restart systemd units on upgrade +# +# By default, it is left to the administrator to restart +# ceph daemons (or their related systemd units) manually +# when the 'ceph' package is upgraded. By setting this +# parameter to "yes", package upgrade will trigger a +# "systemctl try-restart" on all the ceph systemd units +# currently active on the node. +# +CEPH_AUTO_RESTART_ON_UPGRADE=no`) +) + func TestCephVolumeEnvVar(t *testing.T) { cvEnv := cephVolumeEnvVar() assert.Equal(t, "CEPH_VOLUME_DEBUG", cvEnv[0].Name) @@ -45,3 +68,27 @@ func TestOsdActivateEnvVar(t *testing.T) { assert.Equal(t, "CEPH_ARGS", osdActivateEnv[4].Name) assert.Equal(t, "-m $(ROOK_CEPH_MON_HOST)", osdActivateEnv[4].Value) } + +func TestGetTcmallocMaxTotalThreadCacheBytes(t *testing.T) { + // No file, nothing + v := getTcmallocMaxTotalThreadCacheBytes("") + assert.Equal(t, "", v.Value) + + // File and arg are empty so we can an empty value + file, err := ioutil.TempFile("", "") + assert.NoError(t, err) + defer os.Remove(file.Name()) + cephEnvConfigFile = file.Name() + v = getTcmallocMaxTotalThreadCacheBytes("") + assert.Equal(t, "", v.Value) + + // Arg is not empty + v = getTcmallocMaxTotalThreadCacheBytes("67108864") + assert.Equal(t, "67108864", v.Value) + + // Read the file now + err = ioutil.WriteFile(file.Name(), sysconfig, 0444) + assert.NoError(t, err) + v = getTcmallocMaxTotalThreadCacheBytes("") + assert.Equal(t, "134217728", v.Value) +} diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 7031d3a11..41ac9f5a4 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -242,6 +242,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC {Name: "ROOK_OSD_ID", Value: osdID}, {Name: "ROOK_CEPH_VERSION", Value: c.clusterInfo.CephVersion.CephVersionFormatted()}, {Name: "ROOK_IS_DEVICE", Value: "true"}, + getTcmallocMaxTotalThreadCacheBytes(""), }...) var command []string @@ -540,6 +541,13 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC osdProps.placement.ApplyToPodSpec(&deployment.Spec.Template.Spec) } + // Change TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES if the OSD has been annotated with a value + osdAnnotations := cephv1.GetOSDAnnotations(c.spec.Annotations) + tcmallocMaxTotalThreadCacheBytes, ok := osdAnnotations[tcmallocMaxTotalThreadCacheBytesEnv] + if ok && tcmallocMaxTotalThreadCacheBytes != "" { + deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, getTcmallocMaxTotalThreadCacheBytes(tcmallocMaxTotalThreadCacheBytes)) + } + return deployment, nil } diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index ed9dd5e28..7ae3e5a8f 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -93,6 +93,11 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { PriorityClassNames: map[rookv1.KeyType]string{ cephv1.KeyOSD: "my-priority-class", }, + Annotations: rookv1.AnnotationsSpec{ + "osd": map[string]string{ + "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES": "134217728", + }, + }, } c := New(context, clusterInfo, spec, "rook/rook:myversion") @@ -145,7 +150,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, AppName, deployment.Spec.Template.ObjectMeta.Name) assert.Equal(t, AppName, deployment.Spec.Template.ObjectMeta.Labels["app"]) assert.Equal(t, c.clusterInfo.Namespace, deployment.Spec.Template.ObjectMeta.Labels["rook_cluster"]) - assert.Equal(t, 0, len(deployment.Spec.Template.ObjectMeta.Annotations)) + assert.Equal(t, 1, len(deployment.Spec.Template.ObjectMeta.Annotations)) assert.Equal(t, 2, len(deployment.Spec.Template.Spec.InitContainers)) initCont := deployment.Spec.Template.Spec.InitContainers[0] @@ -158,6 +163,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, spec.CephVersion.Image, cont.Image) assert.Equal(t, 7, len(cont.VolumeMounts)) assert.Equal(t, "ceph-osd", cont.Command[0]) + verifyEnvVar(t, cont.Env, "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES", "134217728", true) // Test OSD on PVC with LVM osdProp = osdProperties{