Merge pull request #6397 from leseb/fix-6134

ceph: tune osd tcmalloc thread cache bytes

go.mod

@@ -32,7 +32,7 @@ require (
 	golang.org/x/text v0.3.3 // indirect
 	golang.org/x/tools v0.0.0-20200319210407-521f4a0cd458 // indirect
 	google.golang.org/grpc v1.26.0 // indirect
-	gopkg.in/ini.v1 v1.51.1 // indirect
+	gopkg.in/ini.v1 v1.51.1
 	k8s.io/api v0.17.2
 	k8s.io/apiextensions-apiserver v0.17.2
 	k8s.io/apimachinery v0.17.2

pkg/operator/ceph/cluster/osd/envs.go

@@ -22,6 +22,7 @@ import (
 	"github.com/pkg/errors"
 	opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon"
 	"github.com/rook/rook/pkg/operator/k8sutil"
+	"gopkg.in/ini.v1"
 	v1 "k8s.io/api/core/v1"
 )
 
@@ -33,14 +34,19 @@ const (
 	EncryptedDeviceEnvVarName = "ROOK_ENCRYPTED_DEVICE"
 	// CephVolumeEncryptedKeyEnvVarName is the env variable used by ceph-volume to encrypt the OSD (raw mode)
 	// Hardcoded in ceph-volume do NOT touch
-	CephVolumeEncryptedKeyEnvVarName = "CEPH_VOLUME_DMCRYPT_SECRET"
-	osdMetadataDeviceEnvVarName      = "ROOK_METADATA_DEVICE"
-	osdWalDeviceEnvVarName           = "ROOK_WAL_DEVICE"
-	pvcBackedOSDVarName              = "ROOK_PVC_BACKED_OSD"
-	blockPathVarName                 = "ROOK_BLOCK_PATH"
-	cvModeVarName                    = "ROOK_CV_MODE"
-	lvBackedPVVarName                = "ROOK_LV_BACKED_PV"
-	CrushDeviceClassVarName          = "ROOK_OSD_CRUSH_DEVICE_CLASS"
+	CephVolumeEncryptedKeyEnvVarName    = "CEPH_VOLUME_DMCRYPT_SECRET"
+	osdMetadataDeviceEnvVarName         = "ROOK_METADATA_DEVICE"
+	osdWalDeviceEnvVarName              = "ROOK_WAL_DEVICE"
+	pvcBackedOSDVarName                 = "ROOK_PVC_BACKED_OSD"
+	blockPathVarName                    = "ROOK_BLOCK_PATH"
+	cvModeVarName                       = "ROOK_CV_MODE"
+	lvBackedPVVarName                   = "ROOK_LV_BACKED_PV"
+	CrushDeviceClassVarName             = "ROOK_OSD_CRUSH_DEVICE_CLASS"
+	tcmallocMaxTotalThreadCacheBytesEnv = "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES"
+)
+
+var (
+	cephEnvConfigFile = "/etc/sysconfig/ceph"
 )
 
 func (c *Cluster) getConfigEnvVars(osdProps osdProperties, dataDir string) []v1.EnvVar {
@@ -196,3 +202,24 @@ func osdActivateEnvVar() []v1.EnvVar {
 
 	return append(cephVolumeEnvVar(), monEnvVars...)
 }
+
+func getTcmallocMaxTotalThreadCacheBytes(tcmallocMaxTotalThreadCacheBytes string) v1.EnvVar {
+	var value string
+	// If empty we read the default value from the file coming with the package
+	if tcmallocMaxTotalThreadCacheBytes == "" {
+		value = getTcmallocMaxTotalThreadCacheBytesFromFile()
+	} else {
+		value = tcmallocMaxTotalThreadCacheBytes
+	}
+
+	return v1.EnvVar{Name: tcmallocMaxTotalThreadCacheBytesEnv, Value: value}
+}
+
+func getTcmallocMaxTotalThreadCacheBytesFromFile() string {
+	iniCephEnvConfigFile, err := ini.Load(cephEnvConfigFile)
+	if err != nil {
+		return ""
+	}
+
+	return iniCephEnvConfigFile.Section("").Key(tcmallocMaxTotalThreadCacheBytesEnv).String()
+}

pkg/operator/ceph/cluster/osd/envs_test.go

@@ -17,11 +17,34 @@ limitations under the License.
 package osd
 
 import (
+	"io/ioutil"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
+var (
+	sysconfig = []byte(`# /etc/sysconfig/ceph
+#
+# Environment file for ceph daemon systemd unit files.
+#
+
+# Increase tcmalloc cache size
+TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728
+
+## automatically restart systemd units on upgrade
+#
+# By default, it is left to the administrator to restart
+# ceph daemons (or their related systemd units) manually
+# when the 'ceph' package is upgraded. By setting this
+# parameter to "yes", package upgrade will trigger a
+# "systemctl try-restart" on all the ceph systemd units
+# currently active on the node.
+#
+CEPH_AUTO_RESTART_ON_UPGRADE=no`)
+)
+
 func TestCephVolumeEnvVar(t *testing.T) {
 	cvEnv := cephVolumeEnvVar()
 	assert.Equal(t, "CEPH_VOLUME_DEBUG", cvEnv[0].Name)
@@ -45,3 +68,27 @@ func TestOsdActivateEnvVar(t *testing.T) {
 	assert.Equal(t, "CEPH_ARGS", osdActivateEnv[4].Name)
 	assert.Equal(t, "-m $(ROOK_CEPH_MON_HOST)", osdActivateEnv[4].Value)
 }
+
+func TestGetTcmallocMaxTotalThreadCacheBytes(t *testing.T) {
+	// No file, nothing
+	v := getTcmallocMaxTotalThreadCacheBytes("")
+	assert.Equal(t, "", v.Value)
+
+	// File and arg are empty so we can an empty value
+	file, err := ioutil.TempFile("", "")
+	assert.NoError(t, err)
+	defer os.Remove(file.Name())
+	cephEnvConfigFile = file.Name()
+	v = getTcmallocMaxTotalThreadCacheBytes("")
+	assert.Equal(t, "", v.Value)
+
+	// Arg is not empty
+	v = getTcmallocMaxTotalThreadCacheBytes("67108864")
+	assert.Equal(t, "67108864", v.Value)
+
+	// Read the file now
+	err = ioutil.WriteFile(file.Name(), sysconfig, 0444)
+	assert.NoError(t, err)
+	v = getTcmallocMaxTotalThreadCacheBytes("")
+	assert.Equal(t, "134217728", v.Value)
+}

pkg/operator/ceph/cluster/osd/spec.go

@@ -242,6 +242,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
 		{Name: "ROOK_OSD_ID", Value: osdID},
 		{Name: "ROOK_CEPH_VERSION", Value: c.clusterInfo.CephVersion.CephVersionFormatted()},
 		{Name: "ROOK_IS_DEVICE", Value: "true"},
+		getTcmallocMaxTotalThreadCacheBytes(""),
 	}...)
 
 	var command []string
@@ -540,6 +541,13 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
 		osdProps.placement.ApplyToPodSpec(&deployment.Spec.Template.Spec)
 	}
 
+	// Change TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES if the OSD has been annotated with a value
+	osdAnnotations := cephv1.GetOSDAnnotations(c.spec.Annotations)
+	tcmallocMaxTotalThreadCacheBytes, ok := osdAnnotations[tcmallocMaxTotalThreadCacheBytesEnv]
+	if ok && tcmallocMaxTotalThreadCacheBytes != "" {
+		deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, getTcmallocMaxTotalThreadCacheBytes(tcmallocMaxTotalThreadCacheBytes))
+	}
+
 	return deployment, nil
 }
 

pkg/operator/ceph/cluster/osd/spec_test.go

@@ -93,6 +93,11 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
 		PriorityClassNames: map[rookv1.KeyType]string{
 			cephv1.KeyOSD: "my-priority-class",
 		},
+		Annotations: rookv1.AnnotationsSpec{
+			"osd": map[string]string{
+				"TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES": "134217728",
+			},
+		},
 	}
 	c := New(context, clusterInfo, spec, "rook/rook:myversion")
 
@@ -145,7 +150,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
 	assert.Equal(t, AppName, deployment.Spec.Template.ObjectMeta.Name)
 	assert.Equal(t, AppName, deployment.Spec.Template.ObjectMeta.Labels["app"])
 	assert.Equal(t, c.clusterInfo.Namespace, deployment.Spec.Template.ObjectMeta.Labels["rook_cluster"])
-	assert.Equal(t, 0, len(deployment.Spec.Template.ObjectMeta.Annotations))
+	assert.Equal(t, 1, len(deployment.Spec.Template.ObjectMeta.Annotations))
 
 	assert.Equal(t, 2, len(deployment.Spec.Template.Spec.InitContainers))
 	initCont := deployment.Spec.Template.Spec.InitContainers[0]
@@ -158,6 +163,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
 	assert.Equal(t, spec.CephVersion.Image, cont.Image)
 	assert.Equal(t, 7, len(cont.VolumeMounts))
 	assert.Equal(t, "ceph-osd", cont.Command[0])
+	verifyEnvVar(t, cont.Env, "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES", "134217728", true)
 
 	// Test OSD on PVC with LVM
 	osdProp = osdProperties{