forked from rook/rook
subhamkrai
bfc878e33a
this commit reverts the changes done in controller file as previous changes failing in ci and those changes are related to linter so we can supress them. Signed-off-by: subhamkrai <srai@redhat.com>
518 lines
16 KiB
Go
518 lines
16 KiB
Go
/*
|
|
Copyright 2019 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package yugabytedb
|
|
|
|
import (
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
|
|
"github.com/pkg/errors"
|
|
rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1"
|
|
yugabytedbv1alpha1 "github.com/rook/rook/pkg/apis/yugabytedb.rook.io/v1alpha1"
|
|
"github.com/rook/rook/pkg/clusterd"
|
|
"github.com/rook/rook/pkg/operator/k8sutil"
|
|
v1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
"k8s.io/client-go/tools/cache"
|
|
)
|
|
|
|
const (
|
|
customResourceName = "ybcluster"
|
|
customResourceNamePlural = "ybclusters"
|
|
masterName = "yb-master"
|
|
masterNamePlural = "yb-masters"
|
|
tserverName = "yb-tserver"
|
|
tserverNamePlural = "yb-tservers"
|
|
masterUIServiceName = "yb-master-ui"
|
|
tserverUIServiceName = "yb-tserver-ui"
|
|
masterUIPortDefault = int32(7000)
|
|
masterUIPortName = "yb-master-ui"
|
|
masterRPCPortDefault = int32(7100)
|
|
masterRPCPortName = "yb-master-rpc"
|
|
tserverUIPortDefault = int32(9000)
|
|
tserverUIPortName = "yb-tserver-ui"
|
|
tserverRPCPortDefault = int32(9100)
|
|
tserverRPCPortName = "yb-tserver-rpc"
|
|
tserverCassandraPortDefault = int32(9042)
|
|
tserverCassandraPortName = "ycql"
|
|
tserverRedisPortDefault = int32(6379)
|
|
tserverRedisPortName = "yedis"
|
|
tserverPostgresPortDefault = int32(5433)
|
|
tserverPostgresPortName = "ysql"
|
|
masterContainerUIPortName = "master-ui"
|
|
masterContainerRPCPortName = "master-rpc"
|
|
tserverContainerUIPortName = "tserver-ui"
|
|
tserverContainerRPCPortName = "tserver-rpc"
|
|
uiPortName = "ui"
|
|
rpcPortName = "rpc-port"
|
|
cassandraPortName = "cassandra"
|
|
redisPortName = "redis"
|
|
postgresPortName = "postgres"
|
|
volumeMountPath = "/mnt/data0"
|
|
envGetHostsFrom = "GET_HOSTS_FROM"
|
|
envGetHostsFromVal = "dns"
|
|
envPodIP = "POD_IP"
|
|
envPodIPVal = "status.podIP"
|
|
envPodNameVal = "metadata.name"
|
|
yugabyteDBImageName = "yugabytedb/yugabyte:2.2.2.0-b15"
|
|
podCPULimitDefault = "2"
|
|
masterMemLimitDefault = "2Gi"
|
|
tserverMemLimitDefault = "4Gi"
|
|
)
|
|
|
|
var ClusterResource = k8sutil.CustomResource{
|
|
Name: customResourceName,
|
|
Plural: customResourceNamePlural,
|
|
Group: yugabytedbv1alpha1.CustomResourceGroup,
|
|
Version: yugabytedbv1alpha1.Version,
|
|
Kind: reflect.TypeOf(yugabytedbv1alpha1.YBCluster{}).Name(),
|
|
}
|
|
|
|
type ClusterController struct {
|
|
context *clusterd.Context
|
|
containerImage string
|
|
}
|
|
|
|
func NewClusterController(context *clusterd.Context, containerImage string) *ClusterController {
|
|
return &ClusterController{
|
|
context: context,
|
|
containerImage: containerImage,
|
|
}
|
|
}
|
|
|
|
type cluster struct {
|
|
context *clusterd.Context
|
|
name string
|
|
namespace string
|
|
spec yugabytedbv1alpha1.YBClusterSpec
|
|
annotations rookv1.Annotations
|
|
ownerRef metav1.OwnerReference
|
|
}
|
|
|
|
type clusterPorts struct {
|
|
masterPorts, tserverPorts serverPorts
|
|
}
|
|
|
|
type serverPorts struct {
|
|
ui, rpc, cassandra, redis, postgres int32
|
|
}
|
|
|
|
func NewCluster(c *yugabytedbv1alpha1.YBCluster, context *clusterd.Context) *cluster {
|
|
return &cluster{
|
|
context: context,
|
|
name: c.Name,
|
|
namespace: c.Namespace,
|
|
spec: c.Spec,
|
|
annotations: c.Spec.Annotations,
|
|
ownerRef: clusterOwnerRef(c.Name, string(c.UID)),
|
|
}
|
|
}
|
|
|
|
func clusterOwnerRef(name, clusterID string) metav1.OwnerReference {
|
|
blockOwner := true
|
|
return metav1.OwnerReference{
|
|
APIVersion: fmt.Sprintf("%s/%s", ClusterResource.Group, ClusterResource.Version),
|
|
Kind: ClusterResource.Kind,
|
|
Name: name,
|
|
UID: types.UID(clusterID),
|
|
BlockOwnerDeletion: &blockOwner,
|
|
}
|
|
}
|
|
|
|
func (c *ClusterController) StartWatch(namespace string, stopCh chan struct{}) {
|
|
resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{
|
|
AddFunc: c.OnAdd,
|
|
UpdateFunc: c.OnUpdate,
|
|
DeleteFunc: c.onDelete,
|
|
}
|
|
|
|
logger.Infof("start watching yugabytedb clusters in all namespaces")
|
|
go k8sutil.WatchCR(ClusterResource, namespace, resourceHandlerFuncs, c.context.RookClientset.YugabytedbV1alpha1().RESTClient(), &yugabytedbv1alpha1.YBCluster{}, stopCh)
|
|
}
|
|
|
|
func (c *ClusterController) onDelete(obj interface{}) {
|
|
cluster, ok := obj.(*yugabytedbv1alpha1.YBCluster)
|
|
if !ok {
|
|
return
|
|
}
|
|
cluster = cluster.DeepCopy()
|
|
logger.Infof("cluster %s deleted from namespace %s", cluster.Name, cluster.Namespace)
|
|
}
|
|
|
|
func validateClusterSpec(spec yugabytedbv1alpha1.YBClusterSpec) error {
|
|
|
|
if spec.Master.Replicas < 1 {
|
|
return fmt.Errorf("invalid Master replica count: %d. Must be at least 1", spec.Master.Replicas)
|
|
}
|
|
|
|
if spec.TServer.Replicas < 1 {
|
|
return fmt.Errorf("invalid TServer replica count: %d. Must be at least 1", spec.TServer.Replicas)
|
|
}
|
|
|
|
if _, err := getPortsFromSpec(spec.Master.Network); err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err := getPortsFromSpec(spec.TServer.Network); err != nil {
|
|
return err
|
|
}
|
|
|
|
if &spec.Master.VolumeClaimTemplate == nil { //nolint, ok to ignore this test
|
|
return fmt.Errorf("VolumeClaimTemplate unavailable in Master spec.")
|
|
}
|
|
|
|
if &spec.TServer.VolumeClaimTemplate == nil { //nolint, ok to ignore this test
|
|
return fmt.Errorf("VolumeClaimTemplate unavailable in TServer spec.")
|
|
}
|
|
|
|
if err := validateResourceSpec(spec.Master.Resource, false); err != nil {
|
|
return errors.Wrap(err, "failed to validate resource spec")
|
|
}
|
|
|
|
if err := validateResourceSpec(spec.TServer.Resource, true); err != nil {
|
|
return errors.Wrap(err, "failed to validate resource spec")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func validateResourceSpec(resourceSpec v1.ResourceRequirements, isTServer bool) error {
|
|
if (&resourceSpec.Requests == nil || len(resourceSpec.Requests) != 0) && (&resourceSpec.Limits == nil || len(resourceSpec.Limits) != 0) { //nolint, ok to ignore this test
|
|
reqCPU, reqOk := resourceSpec.Requests[v1.ResourceCPU]
|
|
limCPU, limOk := resourceSpec.Limits[v1.ResourceCPU]
|
|
podName := "Master"
|
|
|
|
if isTServer {
|
|
podName = "TServer"
|
|
}
|
|
|
|
// q.Cmp(y Quantity) returns -1 if q < y
|
|
if reqOk && limOk && (&reqCPU).Cmp(limCPU) < 0 {
|
|
return fmt.Errorf("%s Requested CPU cannot be greater than CPU limit", podName)
|
|
}
|
|
|
|
reqMem, reqOk := resourceSpec.Requests[v1.ResourceMemory]
|
|
limMem, limOk := resourceSpec.Limits[v1.ResourceMemory]
|
|
// q.Cmp(y Quantity) returns -1 if q < y
|
|
if reqOk && limOk && (&reqMem).Cmp(limMem) < 0 {
|
|
return fmt.Errorf("%s Requested Memory cannot be greater than Memory limit", podName)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func createAppLabels(label string) map[string]string {
|
|
return map[string]string{
|
|
k8sutil.AppAttr: label,
|
|
}
|
|
}
|
|
|
|
func createServicePorts(cluster *cluster, isTServerService bool) []v1.ServicePort {
|
|
var servicePorts []v1.ServicePort
|
|
|
|
if !isTServerService {
|
|
ports, _ := getPortsFromSpec(cluster.spec.Master.Network)
|
|
|
|
servicePorts = []v1.ServicePort{
|
|
{
|
|
Name: uiPortName,
|
|
Port: ports.masterPorts.ui,
|
|
TargetPort: intstr.FromInt(int(ports.masterPorts.ui)),
|
|
},
|
|
{
|
|
Name: rpcPortName,
|
|
Port: ports.masterPorts.rpc,
|
|
TargetPort: intstr.FromInt(int(ports.masterPorts.rpc)),
|
|
},
|
|
}
|
|
} else {
|
|
ports, _ := getPortsFromSpec(cluster.spec.TServer.Network)
|
|
|
|
tserverUIPort := ports.tserverPorts.ui
|
|
|
|
if tserverUIPort <= 0 {
|
|
tserverUIPort = tserverUIPortDefault
|
|
}
|
|
|
|
servicePorts = []v1.ServicePort{
|
|
{
|
|
Name: uiPortName,
|
|
Port: tserverUIPort,
|
|
TargetPort: intstr.FromInt(int(tserverUIPort)),
|
|
},
|
|
{
|
|
Name: rpcPortName,
|
|
Port: ports.tserverPorts.rpc,
|
|
TargetPort: intstr.FromInt(int(ports.tserverPorts.rpc)),
|
|
},
|
|
{
|
|
Name: cassandraPortName,
|
|
Port: ports.tserverPorts.cassandra,
|
|
TargetPort: intstr.FromInt(int(ports.tserverPorts.cassandra)),
|
|
},
|
|
{
|
|
Name: redisPortName,
|
|
Port: ports.tserverPorts.redis,
|
|
TargetPort: intstr.FromInt(int(ports.tserverPorts.redis)),
|
|
},
|
|
{
|
|
Name: postgresPortName,
|
|
Port: ports.tserverPorts.postgres,
|
|
TargetPort: intstr.FromInt(int(ports.tserverPorts.postgres)),
|
|
},
|
|
}
|
|
}
|
|
|
|
return servicePorts
|
|
}
|
|
|
|
func createUIServicePorts(ports *clusterPorts, isTServerService bool) []v1.ServicePort {
|
|
var servicePorts []v1.ServicePort
|
|
|
|
if !isTServerService {
|
|
servicePorts = []v1.ServicePort{
|
|
{
|
|
Name: uiPortName,
|
|
Port: ports.masterPorts.ui,
|
|
TargetPort: intstr.FromInt(int(ports.masterPorts.ui)),
|
|
},
|
|
}
|
|
} else {
|
|
if ports.tserverPorts.ui > 0 {
|
|
servicePorts = []v1.ServicePort{
|
|
{
|
|
Name: uiPortName,
|
|
Port: ports.tserverPorts.ui,
|
|
TargetPort: intstr.FromInt(int(ports.tserverPorts.ui)),
|
|
},
|
|
}
|
|
} else {
|
|
servicePorts = nil
|
|
}
|
|
}
|
|
|
|
return servicePorts
|
|
}
|
|
|
|
func getPortsFromSpec(networkSpec yugabytedbv1alpha1.NetworkSpec) (clusterPort *clusterPorts, err error) {
|
|
ports := clusterPorts{}
|
|
|
|
for _, p := range networkSpec.Ports {
|
|
switch p.Name {
|
|
case masterUIPortName:
|
|
ports.masterPorts.ui = p.Port
|
|
case masterRPCPortName:
|
|
ports.masterPorts.rpc = p.Port
|
|
case tserverUIPortName:
|
|
ports.tserverPorts.ui = p.Port
|
|
case tserverRPCPortName:
|
|
ports.tserverPorts.rpc = p.Port
|
|
case tserverCassandraPortName:
|
|
ports.tserverPorts.cassandra = p.Port
|
|
case tserverRedisPortName:
|
|
ports.tserverPorts.redis = p.Port
|
|
case tserverPostgresPortName:
|
|
ports.tserverPorts.postgres = p.Port
|
|
default:
|
|
return &clusterPorts{}, fmt.Errorf("Invalid port name: %s. Must be one of: [%s, %s, %s, %s, %s, %s, %s]", p.Name,
|
|
masterUIPortName, masterRPCPortName, tserverUIPortName, tserverRPCPortName, tserverCassandraPortName,
|
|
tserverRedisPortName, tserverPostgresPortName)
|
|
}
|
|
}
|
|
|
|
if ports.masterPorts.ui == 0 {
|
|
ports.masterPorts.ui = masterUIPortDefault
|
|
}
|
|
|
|
if ports.masterPorts.rpc == 0 {
|
|
ports.masterPorts.rpc = masterRPCPortDefault
|
|
}
|
|
|
|
if ports.tserverPorts.rpc == 0 {
|
|
ports.tserverPorts.rpc = tserverRPCPortDefault
|
|
}
|
|
|
|
if ports.tserverPorts.cassandra == 0 {
|
|
ports.tserverPorts.cassandra = tserverCassandraPortDefault
|
|
}
|
|
|
|
if ports.tserverPorts.redis == 0 {
|
|
ports.tserverPorts.redis = tserverRedisPortDefault
|
|
}
|
|
|
|
if ports.tserverPorts.postgres == 0 {
|
|
ports.tserverPorts.postgres = tserverPostgresPortDefault
|
|
}
|
|
|
|
return &ports, nil
|
|
}
|
|
|
|
func createMasterContainerCommand(namespace, serviceName string, masterCompleteName string, grpcPort, replicas int32, resources v1.ResourceRequirements) []string {
|
|
command := []string{
|
|
"/home/yugabyte/bin/yb-master",
|
|
fmt.Sprintf("--fs_data_dirs=%s", volumeMountPath),
|
|
fmt.Sprintf("--rpc_bind_addresses=$(POD_IP):%d", grpcPort),
|
|
fmt.Sprintf("--server_broadcast_addresses=$(POD_NAME).%s:%d", serviceName, grpcPort),
|
|
"--use_private_ip=never",
|
|
fmt.Sprintf("--master_addresses=%s", getMasterAddresses(masterCompleteName, serviceName, namespace, replicas, grpcPort)),
|
|
"--enable_ysql=true",
|
|
fmt.Sprintf("--replication_factor=%d", replicas),
|
|
"--logtostderr",
|
|
}
|
|
|
|
if &resources.Limits != nil { //nolint, ok to ignore this test
|
|
if memLimit, ok := getMemoryLimitBytes(resources); ok {
|
|
command = append(command, fmt.Sprintf("--memory_limit_hard_bytes=%d", memLimit))
|
|
}
|
|
}
|
|
|
|
return command
|
|
}
|
|
|
|
func createTServerContainerCommand(namespace, serviceName, masterServiceName string, masterCompleteName string, masterGRPCPort, tserverGRPCPort, pgsqlPort, replicas int32, resources v1.ResourceRequirements) []string {
|
|
command := []string{
|
|
"/home/yugabyte/bin/yb-tserver",
|
|
fmt.Sprintf("--fs_data_dirs=%s", volumeMountPath),
|
|
fmt.Sprintf("--rpc_bind_addresses=$(POD_IP):%d", tserverGRPCPort),
|
|
fmt.Sprintf("--server_broadcast_addresses=$(POD_NAME).%s:%d", serviceName, tserverGRPCPort),
|
|
fmt.Sprintf("--pgsql_proxy_bind_address=$(POD_IP):%d", pgsqlPort),
|
|
"--use_private_ip=never",
|
|
fmt.Sprintf("--tserver_master_addrs=%s", getMasterAddresses(masterCompleteName, masterServiceName, namespace, replicas, masterGRPCPort)),
|
|
"--enable_ysql=true",
|
|
"--logtostderr",
|
|
}
|
|
|
|
if &resources.Limits != nil { //nolint, ok to ignore this test
|
|
if memLimit, ok := getMemoryLimitBytes(resources); ok {
|
|
command = append(command, fmt.Sprintf("--memory_limit_hard_bytes=%d", memLimit))
|
|
}
|
|
}
|
|
|
|
return command
|
|
}
|
|
|
|
func getMemoryLimitBytes(resources v1.ResourceRequirements) (int64, bool) {
|
|
if memLimit, ok := resources.Limits[v1.ResourceMemory]; ok {
|
|
memBytes, _ := (&memLimit).AsInt64()
|
|
// Set DB mem limit at 85% of pod memory. 1024 * 85% ~= 870
|
|
memBytesAdjusted := memBytes * 870 / 1024
|
|
return memBytesAdjusted, true
|
|
}
|
|
|
|
return int64(0), false
|
|
}
|
|
|
|
func getMasterAddresses(masterCompleteName string, serviceName string, namespace string, replicas int32, masterGRPCPort int32) string {
|
|
masterAddrs := []string{}
|
|
for i := int32(0); i < replicas; i++ {
|
|
masterAddrs = append(masterAddrs, fmt.Sprintf("%s-%d.%s.%s.svc.cluster.local:%d", masterCompleteName, i, serviceName, namespace, masterGRPCPort))
|
|
}
|
|
return strings.Join(masterAddrs, ",")
|
|
}
|
|
|
|
func createMasterContainerPortsList(clusterPortsSpec *clusterPorts) []v1.ContainerPort {
|
|
ports := []v1.ContainerPort{
|
|
{
|
|
Name: masterContainerUIPortName,
|
|
ContainerPort: int32(clusterPortsSpec.masterPorts.ui),
|
|
},
|
|
{
|
|
Name: masterContainerRPCPortName,
|
|
ContainerPort: int32(clusterPortsSpec.masterPorts.rpc),
|
|
},
|
|
}
|
|
|
|
return ports
|
|
}
|
|
|
|
func createTServerContainerPortsList(clusterPortsSpec *clusterPorts) []v1.ContainerPort {
|
|
tserverUIPort := int32(clusterPortsSpec.tserverPorts.ui)
|
|
|
|
if tserverUIPort <= 0 {
|
|
tserverUIPort = tserverUIPortDefault
|
|
}
|
|
|
|
ports := []v1.ContainerPort{
|
|
{
|
|
Name: tserverContainerUIPortName,
|
|
ContainerPort: tserverUIPort,
|
|
},
|
|
{
|
|
Name: tserverContainerRPCPortName,
|
|
ContainerPort: int32(clusterPortsSpec.tserverPorts.rpc),
|
|
},
|
|
{
|
|
Name: cassandraPortName,
|
|
ContainerPort: int32(clusterPortsSpec.tserverPorts.cassandra),
|
|
},
|
|
{
|
|
Name: redisPortName,
|
|
ContainerPort: int32(clusterPortsSpec.tserverPorts.redis),
|
|
},
|
|
{
|
|
Name: postgresPortName,
|
|
ContainerPort: int32(clusterPortsSpec.tserverPorts.postgres),
|
|
},
|
|
}
|
|
|
|
return ports
|
|
}
|
|
|
|
func (c *cluster) addCRNameSuffix(str string) string {
|
|
return fmt.Sprintf("%s-%s", str, c.name)
|
|
}
|
|
|
|
func getResourceSpec(resourceSpec v1.ResourceRequirements, isTServerStatefulset bool) v1.ResourceRequirements {
|
|
resources := resourceSpec
|
|
|
|
if &resources == nil { //nolint, ok to ignore this test
|
|
resources = v1.ResourceRequirements{
|
|
Requests: make(map[v1.ResourceName]resource.Quantity),
|
|
Limits: make(map[v1.ResourceName]resource.Quantity),
|
|
}
|
|
}
|
|
|
|
if len(resources.Requests) != 0 || len(resources.Limits) != 0 {
|
|
return resources
|
|
}
|
|
|
|
memoryLimit := masterMemLimitDefault
|
|
|
|
if isTServerStatefulset {
|
|
memoryLimit = tserverMemLimitDefault
|
|
}
|
|
|
|
if &resources.Requests == nil || len(resources.Requests) == 0 { //nolint, ok to ignore this test
|
|
resources.Requests = make(map[v1.ResourceName]resource.Quantity)
|
|
resources.Requests[v1.ResourceCPU] = resource.MustParse(podCPULimitDefault)
|
|
resources.Requests[v1.ResourceMemory] = resource.MustParse(memoryLimit)
|
|
}
|
|
|
|
if &resources.Limits == nil || len(resources.Limits) == 0 { //nolint, ok to ignore this test
|
|
resources.Limits = make(map[v1.ResourceName]resource.Quantity)
|
|
resources.Limits[v1.ResourceCPU] = resource.MustParse(podCPULimitDefault)
|
|
resources.Limits[v1.ResourceMemory] = resource.MustParse(memoryLimit)
|
|
}
|
|
|
|
return resources
|
|
}
|