YugabyteDB: Add resource limits to YugabyteDB pods

Current YugabyteDB operator code creates Master and TServer pods without any resource requests/limits (specifically CPU and memory).
This causes the operator to run into soft/hard memory limit issue. The fix adds recommended resource requests and limits as defaults to the pods it creates.

Closes: https://github.com/yugabyte/yugabyte-db/issues/3884
Signed-off-by: Sameer Kulkarni <samkulkarni20@gmail.com>

Documentation/yugabytedb-cluster-crd.md

@@ -21,6 +21,14 @@ spec:
   master:
     # Replica count for Master.
     replicas: 3
+    # optional. Default values for resource are as below
+    resource:
+      requests:
+        cpu: 2
+        memory: 2Gi
+      limits:
+        cpu: 2
+        memory: 2Gi
     # Mentioning network ports is optional. If some or all ports are not specified, then they will be defaulted to below-mentioned values, except for tserver-ui.
     network:
       ports:
@@ -41,6 +49,14 @@ spec:
   tserver:
     # Replica count for TServer
     replicas: 3
+    # optional. Default values for resource are as below
+    resource:
+      requests:
+        cpu: 2
+        memory: 4Gi
+      limits:
+        cpu: 2
+        memory: 4Gi
     # Mentioning network ports is optional. If some or all ports are not specified, then they will be defaulted to below-mentioned values, except for tserver-ui.
     # For tserver-ui a cluster ip service will be created if the yb-tserver-ui port is explicitly mentioned. If it is not specified, only StatefulSet & headless service will be created for TServer. TServer ClusterIP service creation will be skipped. Whereas for Master, all 3 kubernetes objects will always be created.
     network:
@@ -78,6 +94,11 @@ Specify Master/TServer specific attributes under `master`/`tserver`. The valid a
 
 Specify replica count for `master` & `tserver` pods under `replicas` field. This is a **required** field.
 
+### Resource
+
+Specify resource requests and limits for CPU & Memory. If provided, the given resource values will be used. If omitted the default CPU request & limit will be 2, whereas default memory request & limit will be 2Gi & 4Gi for Master & TServer, respectively. You may override these values for dev environments where the cluster doesn't have CPU/Memory as per the defaults.
+**Though it is recommended to use the defaults for production-like deployments.**
+
 ### Network
 
 `network` field accepts `NetworkSpec` to be specified which describes YugabyteDB network settings. This is an **optional** field. Default network settings will be used, if any or all of the acceptable values are absent.

PendingReleaseNotes.md

@@ -21,6 +21,8 @@
 
 ### YugabyteDB
 
+- Master and TServer pods for YugabyteDB will have resources requests and limits specified as per YugabyteDB recommendations. This will help avoid the soft/hard memory limit issue.
+
 ### Cassandra
 - Added [JMX Prometheus exporter](https://github.com/prometheus/jmx_exporter) support.
 

cluster/examples/kubernetes/yugabytedb/cluster.yaml

@@ -12,6 +12,13 @@ spec:
   master:
     # Replica count for Master.
     replicas: 3
+    resource:
+      requests:
+        cpu: 2
+        memory: 2Gi
+      limits:
+        cpu: 2
+        memory: 2Gi
     # Mentioning network ports is optional. If some or all ports are not specified, then they will be defaulted to below-mentioned values, except for tserver-ui.
     network:
       ports:
@@ -32,6 +39,13 @@ spec:
   tserver:
     # Replica count for TServer
     replicas: 3
+    resource:
+      requests:
+        cpu: 2
+        memory: 4Gi
+      limits:
+        cpu: 2
+        memory: 4Gi
     # Mentioning network ports is optional. If some or all ports are not specified, then they will be defaulted to below-mentioned values, except for tserver-ui.
     # For tserver-ui a cluster ip service will be created if the yb-tserver-ui port is explicitly mentioned. If it is not specified, only StatefulSet & headless service will be created for TServer. TServer ClusterIP service creation will be skipped. Whereas for Master, all 3 kubernetes objects will always be created.
     network:

pkg/apis/yugabytedb.rook.io/v1alpha1/types.go

@@ -54,6 +54,7 @@ type YBClusterSpec struct {
 
 type ServerSpec struct {
 	Replicas            int32                    `json:"replicas,omitempty"`
+	Resource            v1.ResourceRequirements  `json:"resource,omitempty"`
 	Network             NetworkSpec              `json:"network,omitempty"`
 	VolumeClaimTemplate v1.PersistentVolumeClaim `json:"volumeClaimTemplate,omitempty"`
 }

pkg/apis/yugabytedb.rook.io/v1alpha1/zz_generated.deepcopy.go

@@ -65,6 +65,7 @@ func (in *PortSpec) DeepCopy() *PortSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ServerSpec) DeepCopyInto(out *ServerSpec) {
 	*out = *in
+	in.Resource.DeepCopyInto(&out.Resource)
 	in.Network.DeepCopyInto(&out.Network)
 	in.VolumeClaimTemplate.DeepCopyInto(&out.VolumeClaimTemplate)
 	return

pkg/operator/yugabytedb/controller.go

@@ -26,6 +26,7 @@ import (
 	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -72,6 +73,9 @@ const (
 	envPodName                  = "POD_NAME"
 	envPodNameVal               = "metadata.name"
 	yugabyteDBImageName         = "yugabytedb/yugabyte:2.0.10.0-b4"
+	podCPULimitDefault          = "2"
+	masterMemLimitDefault       = "2Gi"
+	tserverMemLimitDefault      = "4Gi"
 )
 
 var ClusterResource = k8sutil.CustomResource{
@@ -179,6 +183,35 @@ func validateClusterSpec(spec yugabytedbv1alpha1.YBClusterSpec) error {
 		return fmt.Errorf("VolumeClaimTemplate unavailable in TServer spec.")
 	}
 
+	validateResourceSpec(spec.Master.Resource, false)
+	validateResourceSpec(spec.TServer.Resource, true)
+
+	return nil
+}
+
+func validateResourceSpec(resourceSpec v1.ResourceRequirements, isTServer bool) error {
+	if (&resourceSpec.Requests == nil || len(resourceSpec.Requests) != 0) && (&resourceSpec.Limits == nil || len(resourceSpec.Limits) != 0) {
+		reqCPU, reqOk := resourceSpec.Requests[v1.ResourceCPU]
+		limCPU, limOk := resourceSpec.Limits[v1.ResourceCPU]
+		podName := "Master"
+
+		if isTServer {
+			podName = "TServer"
+		}
+
+		// q.Cmp(y Quantity) returns -1 if q < y
+		if reqOk && limOk && (&reqCPU).Cmp(limCPU) < 0 {
+			return fmt.Errorf("%s Requested CPU cannot be greater than CPU limit", podName)
+		}
+
+		reqMem, reqOk := resourceSpec.Requests[v1.ResourceMemory]
+		limMem, limOk := resourceSpec.Limits[v1.ResourceMemory]
+		// q.Cmp(y Quantity) returns -1 if q < y
+		if reqOk && limOk && (&reqMem).Cmp(limMem) < 0 {
+			return fmt.Errorf("%s Requested Memory cannot be greater than Memory limit", podName)
+		}
+	}
+
 	return nil
 }
 
@@ -328,7 +361,7 @@ func getPortsFromSpec(networkSpec yugabytedbv1alpha1.NetworkSpec) (clusterPort *
 	return &ports, nil
 }
 
-func createMasterContainerCommand(namespace, serviceName string, masterCompleteName string, grpcPort, replicas int32) []string {
+func createMasterContainerCommand(namespace, serviceName string, masterCompleteName string, grpcPort, replicas int32, resources v1.ResourceRequirements) []string {
 	command := []string{
 		"/home/yugabyte/bin/yb-master",
 		fmt.Sprintf("--fs_data_dirs=%s", volumeMountPath),
@@ -340,10 +373,17 @@ func createMasterContainerCommand(namespace, serviceName string, masterCompleteN
 		fmt.Sprintf("--replication_factor=%d", replicas),
 		"--logtostderr",
 	}
+
+	if &resources.Limits != nil {
+		if memLimit, ok := getMemoryLimitBytes(resources); ok {
+			command = append(command, fmt.Sprintf("--memory_limit_hard_bytes=%d", memLimit))
+		}
+	}
+
 	return command
 }
 
-func createTServerContainerCommand(namespace, serviceName, masterServiceName string, masterCompleteName string, masterGRPCPort, tserverGRPCPort, pgsqlPort, replicas int32) []string {
+func createTServerContainerCommand(namespace, serviceName, masterServiceName string, masterCompleteName string, masterGRPCPort, tserverGRPCPort, pgsqlPort, replicas int32, resources v1.ResourceRequirements) []string {
 	command := []string{
 		"/home/yugabyte/bin/yb-tserver",
 		fmt.Sprintf("--fs_data_dirs=%s", volumeMountPath),
@@ -355,9 +395,27 @@ func createTServerContainerCommand(namespace, serviceName, masterServiceName str
 		"--enable_ysql=true",
 		"--logtostderr",
 	}
+
+	if &resources.Limits != nil {
+		if memLimit, ok := getMemoryLimitBytes(resources); ok {
+			command = append(command, fmt.Sprintf("--memory_limit_hard_bytes=%d", memLimit))
+		}
+	}
+
 	return command
 }
 
+func getMemoryLimitBytes(resources v1.ResourceRequirements) (int64, bool) {
+	if memLimit, ok := resources.Limits[v1.ResourceMemory]; ok {
+		memBytes, _ := (&memLimit).AsInt64()
+		// Set DB mem limit at 85% of pod memory. 1024 * 85% ~= 870
+		memBytesAdjusted := memBytes * 870 / 1024
+		return memBytesAdjusted, true
+	}
+
+	return int64(0), false
+}
+
 func getMasterAddresses(masterCompleteName string, serviceName string, namespace string, replicas int32, masterGRPCPort int32) string {
 	masterAddrs := []string{}
 	for i := int32(0); i < replicas; i++ {
@@ -417,3 +475,38 @@ func createTServerContainerPortsList(clusterPortsSpec *clusterPorts) []v1.Contai
 func (c *cluster) addCRNameSuffix(str string) string {
 	return fmt.Sprintf("%s-%s", str, c.name)
 }
+
+func getResourceSpec(resourceSpec v1.ResourceRequirements, isTServerStatefulset bool) v1.ResourceRequirements {
+	resources := resourceSpec
+
+	if &resources == nil {
+		resources = v1.ResourceRequirements{
+			Requests: make(map[v1.ResourceName]resource.Quantity),
+			Limits:   make(map[v1.ResourceName]resource.Quantity),
+		}
+	}
+
+	if len(resources.Requests) != 0 || len(resources.Limits) != 0 {
+		return resources
+	}
+
+	memoryLimit := masterMemLimitDefault
+
+	if isTServerStatefulset {
+		memoryLimit = tserverMemLimitDefault
+	}
+
+	if &resources.Requests == nil || len(resources.Requests) == 0 {
+		resources.Requests = make(map[v1.ResourceName]resource.Quantity)
+		resources.Requests[v1.ResourceCPU] = resource.MustParse(podCPULimitDefault)
+		resources.Requests[v1.ResourceMemory] = resource.MustParse(memoryLimit)
+	}
+
+	if &resources.Limits == nil || len(resources.Limits) == 0 {
+		resources.Limits = make(map[v1.ResourceName]resource.Quantity)
+		resources.Limits[v1.ResourceCPU] = resource.MustParse(podCPULimitDefault)
+		resources.Limits[v1.ResourceMemory] = resource.MustParse(memoryLimit)
+	}
+
+	return resources
+}

pkg/operator/yugabytedb/controller_test.go

@@ -260,36 +260,40 @@ func TestGetPortsFromSpec(t *testing.T) {
 
 func TestCreateMasterContainerCommand(t *testing.T) {
 	replicationFactor := int32(3)
+	resources := v1.ResourceRequirements{}
 
 	expectedCommand := getMasterContainerCommand(replicationFactor)
-	actualCommand := createMasterContainerCommand("default", masterNamePlural, masterName, int32(7100), replicationFactor)
+	actualCommand := createMasterContainerCommand("default", masterNamePlural, masterName, int32(7100), replicationFactor, resources)
 
 	assert.Equal(t, expectedCommand, actualCommand)
 }
 
 func TestCreateTServerContainerCommand(t *testing.T) {
 	replicationFactor := int32(3)
+	resources := v1.ResourceRequirements{}
 
 	expectedCommand := getTserverContainerCommand(replicationFactor)
-	actualCommand := createTServerContainerCommand("default", tserverNamePlural, masterNamePlural, masterName, int32(7100), int32(9100), int32(5433), replicationFactor)
+	actualCommand := createTServerContainerCommand("default", tserverNamePlural, masterNamePlural, masterName, int32(7100), int32(9100), int32(5433), replicationFactor, resources)
 
 	assert.Equal(t, expectedCommand, actualCommand)
 }
 
 func TestCreateMasterContainerCommandRF1(t *testing.T) {
 	replicationFactor := int32(1)
+	resources := v1.ResourceRequirements{}
 
 	expectedCommand := getMasterContainerCommand(replicationFactor)
-	actualCommand := createMasterContainerCommand("default", masterNamePlural, masterName, int32(7100), replicationFactor)
+	actualCommand := createMasterContainerCommand("default", masterNamePlural, masterName, int32(7100), replicationFactor, resources)
 
 	assert.Equal(t, expectedCommand, actualCommand)
 }
 
 func TestCreateTServerContainerCommandRF1(t *testing.T) {
 	replicationFactor := int32(1)
+	resources := v1.ResourceRequirements{}
 
 	expectedCommand := getTserverContainerCommand(replicationFactor)
-	actualCommand := createTServerContainerCommand("default", tserverNamePlural, masterNamePlural, masterName, int32(7100), int32(9100), int32(5433), replicationFactor)
+	actualCommand := createTServerContainerCommand("default", tserverNamePlural, masterNamePlural, masterName, int32(7100), int32(9100), int32(5433), replicationFactor, resources)
 
 	assert.Equal(t, expectedCommand, actualCommand)
 }
@@ -402,6 +406,23 @@ func TestOnAdd(t *testing.T) {
 		{Name: masterContainerRPCPortName, ContainerPort: masterRPCPortDefault},
 	}
 	assert.Equal(t, expectedContainerPorts, container.Ports)
+	assert.NotNil(t, container.Resources)
+	assert.Equal(t, 2, len(container.Resources.Requests))
+	assert.Equal(t, 2, len(container.Resources.Limits))
+
+	reqCPU, reqOk := container.Resources.Requests[v1.ResourceCPU]
+	limCPU, limOk := container.Resources.Limits[v1.ResourceCPU]
+	assert.True(t, reqOk)
+	assert.True(t, limOk)
+	assert.Equal(t, 0, (&reqCPU).Cmp(resource.MustParse(podCPULimitDefault)))
+	assert.Equal(t, 0, (&limCPU).Cmp(resource.MustParse(podCPULimitDefault)))
+
+	reqMem, reqOk := container.Resources.Requests[v1.ResourceMemory]
+	limMem, limOk := container.Resources.Limits[v1.ResourceMemory]
+	assert.True(t, reqOk)
+	assert.True(t, reqOk)
+	assert.Equal(t, 0, (&reqMem).Cmp(resource.MustParse(masterMemLimitDefault)))
+	assert.Equal(t, 0, (&limMem).Cmp(resource.MustParse(masterMemLimitDefault)))
 
 	volumeMountName := addCRNameSuffix(cluster.Spec.Master.VolumeClaimTemplate.Name)
 	expectedVolumeMounts := []v1.VolumeMount{{Name: volumeMountName, MountPath: volumeMountPath}}
@@ -440,6 +461,7 @@ func TestOnAdd(t *testing.T) {
 		"--enable_ysql=true",
 		"--replication_factor=3",
 		"--logtostderr",
+		"--memory_limit_hard_bytes=1824522240",
 	}
 	assert.Equal(t, expectedCommand, container.Command)
 
@@ -473,6 +495,23 @@ func TestOnAdd(t *testing.T) {
 		{Name: postgresPortName, ContainerPort: tserverPostgresPortDefault},
 	}
 	assert.Equal(t, expectedContainerPorts, container.Ports)
+	assert.NotNil(t, container.Resources)
+	assert.Equal(t, 2, len(container.Resources.Requests))
+	assert.Equal(t, 2, len(container.Resources.Limits))
+
+	reqCPU, reqOk = container.Resources.Requests[v1.ResourceCPU]
+	limCPU, limOk = container.Resources.Limits[v1.ResourceCPU]
+	assert.True(t, reqOk)
+	assert.True(t, limOk)
+	assert.Equal(t, 0, (&reqCPU).Cmp(resource.MustParse(podCPULimitDefault)))
+	assert.Equal(t, 0, (&limCPU).Cmp(resource.MustParse(podCPULimitDefault)))
+
+	reqMem, reqOk = container.Resources.Requests[v1.ResourceMemory]
+	limMem, limOk = container.Resources.Limits[v1.ResourceMemory]
+	assert.True(t, reqOk)
+	assert.True(t, reqOk)
+	assert.Equal(t, 0, (&reqMem).Cmp(resource.MustParse(tserverMemLimitDefault)))
+	assert.Equal(t, 0, (&limMem).Cmp(resource.MustParse(tserverMemLimitDefault)))
 
 	volumeMountName = addCRNameSuffix(cluster.Spec.TServer.VolumeClaimTemplate.Name)
 	expectedVolumeMounts = []v1.VolumeMount{{Name: volumeMountName, MountPath: volumeMountPath}}
@@ -511,6 +550,7 @@ func TestOnAdd(t *testing.T) {
 		fmt.Sprintf("--tserver_master_addrs=%s", getMasterAddresses(addCRNameSuffix(masterName), addCRNameSuffix(masterNamePlural), namespace, int32(3), int32(7100))),
 		"--enable_ysql=true",
 		"--logtostderr",
+		"--memory_limit_hard_bytes=3649044480",
 	}
 	assert.Equal(t, expectedCommand, container.Command)
 

pkg/operator/yugabytedb/create_controller.go

@@ -291,9 +291,10 @@ func createPodSpec(cluster *cluster, containerImage string, isTServerStatefulset
 }
 
 func createContainer(cluster *cluster, containerImage string, isTServerStatefulset bool, name, serviceName string) v1.Container {
+	resources := getResourceSpec(cluster.spec.Master.Resource, isTServerStatefulset)
 	ports, _ := getPortsFromSpec(cluster.spec.Master.Network)
 	masterCompleteName := cluster.addCRNameSuffix(masterName)
-	command := createMasterContainerCommand(cluster.namespace, serviceName, masterCompleteName, ports.masterPorts.rpc, cluster.spec.Master.Replicas)
+	command := createMasterContainerCommand(cluster.namespace, serviceName, masterCompleteName, ports.masterPorts.rpc, cluster.spec.Master.Replicas, resources)
 	containerPorts := createMasterContainerPortsList(ports)
 	volumeMountName := cluster.addCRNameSuffix(cluster.spec.Master.VolumeClaimTemplate.Name)
 
@@ -302,7 +303,8 @@ func createContainer(cluster *cluster, containerImage string, isTServerStatefuls
 		masterCompleteName := cluster.addCRNameSuffix(masterName)
 		masterRPCPort := ports.masterPorts.rpc
 		ports, _ = getPortsFromSpec(cluster.spec.TServer.Network)
-		command = createTServerContainerCommand(cluster.namespace, serviceName, masterServiceName, masterCompleteName, masterRPCPort, ports.tserverPorts.rpc, ports.tserverPorts.postgres, cluster.spec.TServer.Replicas)
+		resources = getResourceSpec(cluster.spec.TServer.Resource, isTServerStatefulset)
+		command = createTServerContainerCommand(cluster.namespace, serviceName, masterServiceName, masterCompleteName, masterRPCPort, ports.tserverPorts.rpc, ports.tserverPorts.postgres, cluster.spec.TServer.Replicas, resources)
 		containerPorts = createTServerContainerPortsList(ports)
 		volumeMountName = cluster.addCRNameSuffix(cluster.spec.TServer.VolumeClaimTemplate.Name)
 	}
@@ -333,8 +335,9 @@ func createContainer(cluster *cluster, containerImage string, isTServerStatefuls
 				},
 			},
 		},
-		Command: command,
-		Ports:   containerPorts,
+		Resources: resources,
+		Command:   command,
+		Ports:     containerPorts,
 		VolumeMounts: []v1.VolumeMount{
 			{
 				Name:      volumeMountName,

pkg/operator/yugabytedb/update_controller.go

@@ -196,7 +196,8 @@ func (c *ClusterController) updateStatefulSet(newCluster *cluster, isTServerStat
 	vct := *newCluster.spec.Master.VolumeClaimTemplate.DeepCopy()
 	vct.Name = newCluster.addCRNameSuffix(vct.Name)
 	volumeClaimTemplates := []v1.PersistentVolumeClaim{vct}
-	command := createMasterContainerCommand(newCluster.namespace, masterServiceName, masterCompleteName, ports.masterPorts.rpc, newCluster.spec.Master.Replicas)
+	resources := getResourceSpec(newCluster.spec.Master.Resource, isTServerStatefulset)
+	command := createMasterContainerCommand(newCluster.namespace, masterServiceName, masterCompleteName, ports.masterPorts.rpc, newCluster.spec.Master.Replicas, resources)
 	containerPorts := createMasterContainerPortsList(ports)
 
 	if isTServerStatefulset {
@@ -215,8 +216,9 @@ func (c *ClusterController) updateStatefulSet(newCluster *cluster, isTServerStat
 		vct = *newCluster.spec.TServer.VolumeClaimTemplate.DeepCopy()
 		vct.Name = newCluster.addCRNameSuffix(vct.Name)
 		volumeClaimTemplates = []v1.PersistentVolumeClaim{vct}
+		resources = getResourceSpec(newCluster.spec.Master.Resource, isTServerStatefulset)
 		command = createTServerContainerCommand(newCluster.namespace, tserverServiceName, masterServiceName, masterCompleteName,
-			masterRPCPort, ports.tserverPorts.rpc, ports.tserverPorts.postgres, newCluster.spec.TServer.Replicas)
+			masterRPCPort, ports.tserverPorts.rpc, ports.tserverPorts.postgres, newCluster.spec.TServer.Replicas, resources)
 		containerPorts = createTServerContainerPortsList(ports)
 	}
 
@@ -228,6 +230,7 @@ func (c *ClusterController) updateStatefulSet(newCluster *cluster, isTServerStat
 
 	sfs.Spec.Replicas = &replicas
 	sfs.Spec.Template.Spec.Containers[0].Command = command
+	sfs.Spec.Template.Spec.Containers[0].Resources = resources
 	sfs.Spec.Template.Spec.Containers[0].Ports = containerPorts
 	sfs.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name = vct.Name
 	sfs.Spec.VolumeClaimTemplates = volumeClaimTemplates

tests/framework/installer/yugabytedb_manifests.go

@@ -148,6 +148,13 @@ metadata:
 spec:
   master:
     replicas: ` + strconv.Itoa(replicaCount) + `
+    resource:
+      requests:
+        cpu: 300m
+        memory: 256Mi
+      limits:
+        cpu: 300m
+        memory: 256Mi
     network:
       ports:
         - name: yb-master-ui
@@ -164,6 +171,13 @@ spec:
             storage: 10Mi
   tserver:
     replicas: ` + strconv.Itoa(replicaCount) + `
+    resource:
+      requests:
+        cpu: 300m
+        memory: 256Mi
+      limits:
+        cpu: 300m
+        memory: 256Mi
     network:
       ports:
         - name: yb-tserver-ui