diff --git a/Documentation/yugabytedb-cluster-crd.md b/Documentation/yugabytedb-cluster-crd.md index 2b9b5469b..57e6e59c0 100644 --- a/Documentation/yugabytedb-cluster-crd.md +++ b/Documentation/yugabytedb-cluster-crd.md @@ -21,6 +21,14 @@ spec: master: # Replica count for Master. replicas: 3 + # optional. Default values for resource are as below + resource: + requests: + cpu: 2 + memory: 2Gi + limits: + cpu: 2 + memory: 2Gi # Mentioning network ports is optional. If some or all ports are not specified, then they will be defaulted to below-mentioned values, except for tserver-ui. network: ports: @@ -41,6 +49,14 @@ spec: tserver: # Replica count for TServer replicas: 3 + # optional. Default values for resource are as below + resource: + requests: + cpu: 2 + memory: 4Gi + limits: + cpu: 2 + memory: 4Gi # Mentioning network ports is optional. If some or all ports are not specified, then they will be defaulted to below-mentioned values, except for tserver-ui. # For tserver-ui a cluster ip service will be created if the yb-tserver-ui port is explicitly mentioned. If it is not specified, only StatefulSet & headless service will be created for TServer. TServer ClusterIP service creation will be skipped. Whereas for Master, all 3 kubernetes objects will always be created. network: @@ -78,6 +94,11 @@ Specify Master/TServer specific attributes under `master`/`tserver`. The valid a Specify replica count for `master` & `tserver` pods under `replicas` field. This is a **required** field. +### Resource + +Specify resource requests and limits for CPU & Memory. If provided, the given resource values will be used. If omitted the default CPU request & limit will be 2, whereas default memory request & limit will be 2Gi & 4Gi for Master & TServer, respectively. You may override these values for dev environments where the cluster doesn't have CPU/Memory as per the defaults. +**Though it is recommended to use the defaults for production-like deployments.** + ### Network `network` field accepts `NetworkSpec` to be specified which describes YugabyteDB network settings. This is an **optional** field. Default network settings will be used, if any or all of the acceptable values are absent. diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 1af03be34..7b6ddf0f1 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -21,6 +21,8 @@ ### YugabyteDB +- Master and TServer pods for YugabyteDB will have resources requests and limits specified as per YugabyteDB recommendations. This will help avoid the soft/hard memory limit issue. + ### Cassandra - Added [JMX Prometheus exporter](https://github.com/prometheus/jmx_exporter) support. diff --git a/cluster/examples/kubernetes/yugabytedb/cluster.yaml b/cluster/examples/kubernetes/yugabytedb/cluster.yaml index cdca129b8..3abc714fa 100644 --- a/cluster/examples/kubernetes/yugabytedb/cluster.yaml +++ b/cluster/examples/kubernetes/yugabytedb/cluster.yaml @@ -12,6 +12,13 @@ spec: master: # Replica count for Master. replicas: 3 + resource: + requests: + cpu: 2 + memory: 2Gi + limits: + cpu: 2 + memory: 2Gi # Mentioning network ports is optional. If some or all ports are not specified, then they will be defaulted to below-mentioned values, except for tserver-ui. network: ports: @@ -32,6 +39,13 @@ spec: tserver: # Replica count for TServer replicas: 3 + resource: + requests: + cpu: 2 + memory: 4Gi + limits: + cpu: 2 + memory: 4Gi # Mentioning network ports is optional. If some or all ports are not specified, then they will be defaulted to below-mentioned values, except for tserver-ui. # For tserver-ui a cluster ip service will be created if the yb-tserver-ui port is explicitly mentioned. If it is not specified, only StatefulSet & headless service will be created for TServer. TServer ClusterIP service creation will be skipped. Whereas for Master, all 3 kubernetes objects will always be created. network: diff --git a/pkg/apis/yugabytedb.rook.io/v1alpha1/types.go b/pkg/apis/yugabytedb.rook.io/v1alpha1/types.go index 7b965bb58..60dd6d77f 100644 --- a/pkg/apis/yugabytedb.rook.io/v1alpha1/types.go +++ b/pkg/apis/yugabytedb.rook.io/v1alpha1/types.go @@ -54,6 +54,7 @@ type YBClusterSpec struct { type ServerSpec struct { Replicas int32 `json:"replicas,omitempty"` + Resource v1.ResourceRequirements `json:"resource,omitempty"` Network NetworkSpec `json:"network,omitempty"` VolumeClaimTemplate v1.PersistentVolumeClaim `json:"volumeClaimTemplate,omitempty"` } diff --git a/pkg/apis/yugabytedb.rook.io/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/yugabytedb.rook.io/v1alpha1/zz_generated.deepcopy.go index 0ff76badb..247ddb8b5 100644 --- a/pkg/apis/yugabytedb.rook.io/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/yugabytedb.rook.io/v1alpha1/zz_generated.deepcopy.go @@ -65,6 +65,7 @@ func (in *PortSpec) DeepCopy() *PortSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServerSpec) DeepCopyInto(out *ServerSpec) { *out = *in + in.Resource.DeepCopyInto(&out.Resource) in.Network.DeepCopyInto(&out.Network) in.VolumeClaimTemplate.DeepCopyInto(&out.VolumeClaimTemplate) return diff --git a/pkg/operator/yugabytedb/controller.go b/pkg/operator/yugabytedb/controller.go index 4252f2e90..0a6c0f7b8 100644 --- a/pkg/operator/yugabytedb/controller.go +++ b/pkg/operator/yugabytedb/controller.go @@ -26,6 +26,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" @@ -72,6 +73,9 @@ const ( envPodName = "POD_NAME" envPodNameVal = "metadata.name" yugabyteDBImageName = "yugabytedb/yugabyte:2.0.10.0-b4" + podCPULimitDefault = "2" + masterMemLimitDefault = "2Gi" + tserverMemLimitDefault = "4Gi" ) var ClusterResource = k8sutil.CustomResource{ @@ -179,6 +183,35 @@ func validateClusterSpec(spec yugabytedbv1alpha1.YBClusterSpec) error { return fmt.Errorf("VolumeClaimTemplate unavailable in TServer spec.") } + validateResourceSpec(spec.Master.Resource, false) + validateResourceSpec(spec.TServer.Resource, true) + + return nil +} + +func validateResourceSpec(resourceSpec v1.ResourceRequirements, isTServer bool) error { + if (&resourceSpec.Requests == nil || len(resourceSpec.Requests) != 0) && (&resourceSpec.Limits == nil || len(resourceSpec.Limits) != 0) { + reqCPU, reqOk := resourceSpec.Requests[v1.ResourceCPU] + limCPU, limOk := resourceSpec.Limits[v1.ResourceCPU] + podName := "Master" + + if isTServer { + podName = "TServer" + } + + // q.Cmp(y Quantity) returns -1 if q < y + if reqOk && limOk && (&reqCPU).Cmp(limCPU) < 0 { + return fmt.Errorf("%s Requested CPU cannot be greater than CPU limit", podName) + } + + reqMem, reqOk := resourceSpec.Requests[v1.ResourceMemory] + limMem, limOk := resourceSpec.Limits[v1.ResourceMemory] + // q.Cmp(y Quantity) returns -1 if q < y + if reqOk && limOk && (&reqMem).Cmp(limMem) < 0 { + return fmt.Errorf("%s Requested Memory cannot be greater than Memory limit", podName) + } + } + return nil } @@ -328,7 +361,7 @@ func getPortsFromSpec(networkSpec yugabytedbv1alpha1.NetworkSpec) (clusterPort * return &ports, nil } -func createMasterContainerCommand(namespace, serviceName string, masterCompleteName string, grpcPort, replicas int32) []string { +func createMasterContainerCommand(namespace, serviceName string, masterCompleteName string, grpcPort, replicas int32, resources v1.ResourceRequirements) []string { command := []string{ "/home/yugabyte/bin/yb-master", fmt.Sprintf("--fs_data_dirs=%s", volumeMountPath), @@ -340,10 +373,17 @@ func createMasterContainerCommand(namespace, serviceName string, masterCompleteN fmt.Sprintf("--replication_factor=%d", replicas), "--logtostderr", } + + if &resources.Limits != nil { + if memLimit, ok := getMemoryLimitBytes(resources); ok { + command = append(command, fmt.Sprintf("--memory_limit_hard_bytes=%d", memLimit)) + } + } + return command } -func createTServerContainerCommand(namespace, serviceName, masterServiceName string, masterCompleteName string, masterGRPCPort, tserverGRPCPort, pgsqlPort, replicas int32) []string { +func createTServerContainerCommand(namespace, serviceName, masterServiceName string, masterCompleteName string, masterGRPCPort, tserverGRPCPort, pgsqlPort, replicas int32, resources v1.ResourceRequirements) []string { command := []string{ "/home/yugabyte/bin/yb-tserver", fmt.Sprintf("--fs_data_dirs=%s", volumeMountPath), @@ -355,9 +395,27 @@ func createTServerContainerCommand(namespace, serviceName, masterServiceName str "--enable_ysql=true", "--logtostderr", } + + if &resources.Limits != nil { + if memLimit, ok := getMemoryLimitBytes(resources); ok { + command = append(command, fmt.Sprintf("--memory_limit_hard_bytes=%d", memLimit)) + } + } + return command } +func getMemoryLimitBytes(resources v1.ResourceRequirements) (int64, bool) { + if memLimit, ok := resources.Limits[v1.ResourceMemory]; ok { + memBytes, _ := (&memLimit).AsInt64() + // Set DB mem limit at 85% of pod memory. 1024 * 85% ~= 870 + memBytesAdjusted := memBytes * 870 / 1024 + return memBytesAdjusted, true + } + + return int64(0), false +} + func getMasterAddresses(masterCompleteName string, serviceName string, namespace string, replicas int32, masterGRPCPort int32) string { masterAddrs := []string{} for i := int32(0); i < replicas; i++ { @@ -417,3 +475,38 @@ func createTServerContainerPortsList(clusterPortsSpec *clusterPorts) []v1.Contai func (c *cluster) addCRNameSuffix(str string) string { return fmt.Sprintf("%s-%s", str, c.name) } + +func getResourceSpec(resourceSpec v1.ResourceRequirements, isTServerStatefulset bool) v1.ResourceRequirements { + resources := resourceSpec + + if &resources == nil { + resources = v1.ResourceRequirements{ + Requests: make(map[v1.ResourceName]resource.Quantity), + Limits: make(map[v1.ResourceName]resource.Quantity), + } + } + + if len(resources.Requests) != 0 || len(resources.Limits) != 0 { + return resources + } + + memoryLimit := masterMemLimitDefault + + if isTServerStatefulset { + memoryLimit = tserverMemLimitDefault + } + + if &resources.Requests == nil || len(resources.Requests) == 0 { + resources.Requests = make(map[v1.ResourceName]resource.Quantity) + resources.Requests[v1.ResourceCPU] = resource.MustParse(podCPULimitDefault) + resources.Requests[v1.ResourceMemory] = resource.MustParse(memoryLimit) + } + + if &resources.Limits == nil || len(resources.Limits) == 0 { + resources.Limits = make(map[v1.ResourceName]resource.Quantity) + resources.Limits[v1.ResourceCPU] = resource.MustParse(podCPULimitDefault) + resources.Limits[v1.ResourceMemory] = resource.MustParse(memoryLimit) + } + + return resources +} diff --git a/pkg/operator/yugabytedb/controller_test.go b/pkg/operator/yugabytedb/controller_test.go index 9847eb783..da657e42b 100644 --- a/pkg/operator/yugabytedb/controller_test.go +++ b/pkg/operator/yugabytedb/controller_test.go @@ -260,36 +260,40 @@ func TestGetPortsFromSpec(t *testing.T) { func TestCreateMasterContainerCommand(t *testing.T) { replicationFactor := int32(3) + resources := v1.ResourceRequirements{} expectedCommand := getMasterContainerCommand(replicationFactor) - actualCommand := createMasterContainerCommand("default", masterNamePlural, masterName, int32(7100), replicationFactor) + actualCommand := createMasterContainerCommand("default", masterNamePlural, masterName, int32(7100), replicationFactor, resources) assert.Equal(t, expectedCommand, actualCommand) } func TestCreateTServerContainerCommand(t *testing.T) { replicationFactor := int32(3) + resources := v1.ResourceRequirements{} expectedCommand := getTserverContainerCommand(replicationFactor) - actualCommand := createTServerContainerCommand("default", tserverNamePlural, masterNamePlural, masterName, int32(7100), int32(9100), int32(5433), replicationFactor) + actualCommand := createTServerContainerCommand("default", tserverNamePlural, masterNamePlural, masterName, int32(7100), int32(9100), int32(5433), replicationFactor, resources) assert.Equal(t, expectedCommand, actualCommand) } func TestCreateMasterContainerCommandRF1(t *testing.T) { replicationFactor := int32(1) + resources := v1.ResourceRequirements{} expectedCommand := getMasterContainerCommand(replicationFactor) - actualCommand := createMasterContainerCommand("default", masterNamePlural, masterName, int32(7100), replicationFactor) + actualCommand := createMasterContainerCommand("default", masterNamePlural, masterName, int32(7100), replicationFactor, resources) assert.Equal(t, expectedCommand, actualCommand) } func TestCreateTServerContainerCommandRF1(t *testing.T) { replicationFactor := int32(1) + resources := v1.ResourceRequirements{} expectedCommand := getTserverContainerCommand(replicationFactor) - actualCommand := createTServerContainerCommand("default", tserverNamePlural, masterNamePlural, masterName, int32(7100), int32(9100), int32(5433), replicationFactor) + actualCommand := createTServerContainerCommand("default", tserverNamePlural, masterNamePlural, masterName, int32(7100), int32(9100), int32(5433), replicationFactor, resources) assert.Equal(t, expectedCommand, actualCommand) } @@ -402,6 +406,23 @@ func TestOnAdd(t *testing.T) { {Name: masterContainerRPCPortName, ContainerPort: masterRPCPortDefault}, } assert.Equal(t, expectedContainerPorts, container.Ports) + assert.NotNil(t, container.Resources) + assert.Equal(t, 2, len(container.Resources.Requests)) + assert.Equal(t, 2, len(container.Resources.Limits)) + + reqCPU, reqOk := container.Resources.Requests[v1.ResourceCPU] + limCPU, limOk := container.Resources.Limits[v1.ResourceCPU] + assert.True(t, reqOk) + assert.True(t, limOk) + assert.Equal(t, 0, (&reqCPU).Cmp(resource.MustParse(podCPULimitDefault))) + assert.Equal(t, 0, (&limCPU).Cmp(resource.MustParse(podCPULimitDefault))) + + reqMem, reqOk := container.Resources.Requests[v1.ResourceMemory] + limMem, limOk := container.Resources.Limits[v1.ResourceMemory] + assert.True(t, reqOk) + assert.True(t, reqOk) + assert.Equal(t, 0, (&reqMem).Cmp(resource.MustParse(masterMemLimitDefault))) + assert.Equal(t, 0, (&limMem).Cmp(resource.MustParse(masterMemLimitDefault))) volumeMountName := addCRNameSuffix(cluster.Spec.Master.VolumeClaimTemplate.Name) expectedVolumeMounts := []v1.VolumeMount{{Name: volumeMountName, MountPath: volumeMountPath}} @@ -440,6 +461,7 @@ func TestOnAdd(t *testing.T) { "--enable_ysql=true", "--replication_factor=3", "--logtostderr", + "--memory_limit_hard_bytes=1824522240", } assert.Equal(t, expectedCommand, container.Command) @@ -473,6 +495,23 @@ func TestOnAdd(t *testing.T) { {Name: postgresPortName, ContainerPort: tserverPostgresPortDefault}, } assert.Equal(t, expectedContainerPorts, container.Ports) + assert.NotNil(t, container.Resources) + assert.Equal(t, 2, len(container.Resources.Requests)) + assert.Equal(t, 2, len(container.Resources.Limits)) + + reqCPU, reqOk = container.Resources.Requests[v1.ResourceCPU] + limCPU, limOk = container.Resources.Limits[v1.ResourceCPU] + assert.True(t, reqOk) + assert.True(t, limOk) + assert.Equal(t, 0, (&reqCPU).Cmp(resource.MustParse(podCPULimitDefault))) + assert.Equal(t, 0, (&limCPU).Cmp(resource.MustParse(podCPULimitDefault))) + + reqMem, reqOk = container.Resources.Requests[v1.ResourceMemory] + limMem, limOk = container.Resources.Limits[v1.ResourceMemory] + assert.True(t, reqOk) + assert.True(t, reqOk) + assert.Equal(t, 0, (&reqMem).Cmp(resource.MustParse(tserverMemLimitDefault))) + assert.Equal(t, 0, (&limMem).Cmp(resource.MustParse(tserverMemLimitDefault))) volumeMountName = addCRNameSuffix(cluster.Spec.TServer.VolumeClaimTemplate.Name) expectedVolumeMounts = []v1.VolumeMount{{Name: volumeMountName, MountPath: volumeMountPath}} @@ -511,6 +550,7 @@ func TestOnAdd(t *testing.T) { fmt.Sprintf("--tserver_master_addrs=%s", getMasterAddresses(addCRNameSuffix(masterName), addCRNameSuffix(masterNamePlural), namespace, int32(3), int32(7100))), "--enable_ysql=true", "--logtostderr", + "--memory_limit_hard_bytes=3649044480", } assert.Equal(t, expectedCommand, container.Command) diff --git a/pkg/operator/yugabytedb/create_controller.go b/pkg/operator/yugabytedb/create_controller.go index 6bef151be..6169bad30 100644 --- a/pkg/operator/yugabytedb/create_controller.go +++ b/pkg/operator/yugabytedb/create_controller.go @@ -291,9 +291,10 @@ func createPodSpec(cluster *cluster, containerImage string, isTServerStatefulset } func createContainer(cluster *cluster, containerImage string, isTServerStatefulset bool, name, serviceName string) v1.Container { + resources := getResourceSpec(cluster.spec.Master.Resource, isTServerStatefulset) ports, _ := getPortsFromSpec(cluster.spec.Master.Network) masterCompleteName := cluster.addCRNameSuffix(masterName) - command := createMasterContainerCommand(cluster.namespace, serviceName, masterCompleteName, ports.masterPorts.rpc, cluster.spec.Master.Replicas) + command := createMasterContainerCommand(cluster.namespace, serviceName, masterCompleteName, ports.masterPorts.rpc, cluster.spec.Master.Replicas, resources) containerPorts := createMasterContainerPortsList(ports) volumeMountName := cluster.addCRNameSuffix(cluster.spec.Master.VolumeClaimTemplate.Name) @@ -302,7 +303,8 @@ func createContainer(cluster *cluster, containerImage string, isTServerStatefuls masterCompleteName := cluster.addCRNameSuffix(masterName) masterRPCPort := ports.masterPorts.rpc ports, _ = getPortsFromSpec(cluster.spec.TServer.Network) - command = createTServerContainerCommand(cluster.namespace, serviceName, masterServiceName, masterCompleteName, masterRPCPort, ports.tserverPorts.rpc, ports.tserverPorts.postgres, cluster.spec.TServer.Replicas) + resources = getResourceSpec(cluster.spec.TServer.Resource, isTServerStatefulset) + command = createTServerContainerCommand(cluster.namespace, serviceName, masterServiceName, masterCompleteName, masterRPCPort, ports.tserverPorts.rpc, ports.tserverPorts.postgres, cluster.spec.TServer.Replicas, resources) containerPorts = createTServerContainerPortsList(ports) volumeMountName = cluster.addCRNameSuffix(cluster.spec.TServer.VolumeClaimTemplate.Name) } @@ -333,8 +335,9 @@ func createContainer(cluster *cluster, containerImage string, isTServerStatefuls }, }, }, - Command: command, - Ports: containerPorts, + Resources: resources, + Command: command, + Ports: containerPorts, VolumeMounts: []v1.VolumeMount{ { Name: volumeMountName, diff --git a/pkg/operator/yugabytedb/update_controller.go b/pkg/operator/yugabytedb/update_controller.go index cda43be1d..6e0d8db52 100644 --- a/pkg/operator/yugabytedb/update_controller.go +++ b/pkg/operator/yugabytedb/update_controller.go @@ -196,7 +196,8 @@ func (c *ClusterController) updateStatefulSet(newCluster *cluster, isTServerStat vct := *newCluster.spec.Master.VolumeClaimTemplate.DeepCopy() vct.Name = newCluster.addCRNameSuffix(vct.Name) volumeClaimTemplates := []v1.PersistentVolumeClaim{vct} - command := createMasterContainerCommand(newCluster.namespace, masterServiceName, masterCompleteName, ports.masterPorts.rpc, newCluster.spec.Master.Replicas) + resources := getResourceSpec(newCluster.spec.Master.Resource, isTServerStatefulset) + command := createMasterContainerCommand(newCluster.namespace, masterServiceName, masterCompleteName, ports.masterPorts.rpc, newCluster.spec.Master.Replicas, resources) containerPorts := createMasterContainerPortsList(ports) if isTServerStatefulset { @@ -215,8 +216,9 @@ func (c *ClusterController) updateStatefulSet(newCluster *cluster, isTServerStat vct = *newCluster.spec.TServer.VolumeClaimTemplate.DeepCopy() vct.Name = newCluster.addCRNameSuffix(vct.Name) volumeClaimTemplates = []v1.PersistentVolumeClaim{vct} + resources = getResourceSpec(newCluster.spec.Master.Resource, isTServerStatefulset) command = createTServerContainerCommand(newCluster.namespace, tserverServiceName, masterServiceName, masterCompleteName, - masterRPCPort, ports.tserverPorts.rpc, ports.tserverPorts.postgres, newCluster.spec.TServer.Replicas) + masterRPCPort, ports.tserverPorts.rpc, ports.tserverPorts.postgres, newCluster.spec.TServer.Replicas, resources) containerPorts = createTServerContainerPortsList(ports) } @@ -228,6 +230,7 @@ func (c *ClusterController) updateStatefulSet(newCluster *cluster, isTServerStat sfs.Spec.Replicas = &replicas sfs.Spec.Template.Spec.Containers[0].Command = command + sfs.Spec.Template.Spec.Containers[0].Resources = resources sfs.Spec.Template.Spec.Containers[0].Ports = containerPorts sfs.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name = vct.Name sfs.Spec.VolumeClaimTemplates = volumeClaimTemplates diff --git a/tests/framework/installer/yugabytedb_manifests.go b/tests/framework/installer/yugabytedb_manifests.go index d8e5f32dd..9f6a6367f 100644 --- a/tests/framework/installer/yugabytedb_manifests.go +++ b/tests/framework/installer/yugabytedb_manifests.go @@ -148,6 +148,13 @@ metadata: spec: master: replicas: ` + strconv.Itoa(replicaCount) + ` + resource: + requests: + cpu: 300m + memory: 256Mi + limits: + cpu: 300m + memory: 256Mi network: ports: - name: yb-master-ui @@ -164,6 +171,13 @@ spec: storage: 10Mi tserver: replicas: ` + strconv.Itoa(replicaCount) + ` + resource: + requests: + cpu: 300m + memory: 256Mi + limits: + cpu: 300m + memory: 256Mi network: ports: - name: yb-tserver-ui