cassandra: controller-side implementation of scale down
Code handling the scale down on the side of the controller. Also, some new util functions. Signed-off-by: Yannis Zarkadas <yanniszarkadas@gmail.com>
This commit is contained in:
1 parent
624ff18211
commit
f5a41e9a7c
7 files changed
+338
-18
No files matched your search
@@ -87,9 +87,6 @@ func startOperator(cmd *cobra.Command, args []string) error {
|
||||
kubeInformerFactory.Apps().V1().StatefulSets(),
|
||||
kubeInformerFactory.Core().V1().Services(),
|
||||
kubeInformerFactory.Core().V1().Pods(),
|
||||
kubeInformerFactory.Core().V1().ServiceAccounts(),
|
||||
kubeInformerFactory.Rbac().V1().Roles(),
|
||||
kubeInformerFactory.Rbac().V1().RoleBindings(),
|
||||
)
|
||||
|
||||
// Create a channel to receive OS signals
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package controller
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
cassandrav1alpha1 "github.com/rook/rook/pkg/apis/cassandra.rook.io/v1alpha1"
|
||||
"github.com/rook/rook/pkg/operator/cassandra/controller/util"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
// cleanup deletes all resources remaining because of cluster scale downs
|
||||
func (cc *ClusterController) cleanup(c *cassandrav1alpha1.Cluster) error {
|
||||
|
||||
for _, r := range c.Spec.Datacenter.Racks {
|
||||
services, err := cc.serviceLister.Services(c.Namespace).List(util.RackSelector(r, c))
|
||||
if err != nil {
|
||||
return fmt.Errorf("error listing member services: %s", err.Error())
|
||||
}
|
||||
// Get rack status. If it doesn't exist, the rack isn't yet created.
|
||||
stsName := util.StatefulSetNameForRack(r, c)
|
||||
sts, err := cc.statefulSetLister.StatefulSets(c.Namespace).Get(stsName)
|
||||
if apierrors.IsNotFound(err) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting statefulset %s: %s", stsName, err.Error())
|
||||
}
|
||||
memberCount := *sts.Spec.Replicas
|
||||
memberServiceCount := int32(len(services))
|
||||
// If there are more services than members, some services need to be cleaned up
|
||||
if memberServiceCount > memberCount {
|
||||
maxIndex := memberCount - 1
|
||||
for _, svc := range services {
|
||||
svcIndex, err := util.IndexFromName(svc.Name)
|
||||
if err != nil {
|
||||
logger.Errorf("Unexpected error while parsing index from name %s : %s", svc.Name, err.Error())
|
||||
continue
|
||||
}
|
||||
if svcIndex > maxIndex {
|
||||
err := cc.cleanupMemberResources(svc.Name, r, c)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error cleaning up member resources: %s", err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.Infof("%s/%s - Successfully cleaned up cluster.", c.Namespace, c.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupMemberResources deletes all resources associated with a given member.
|
||||
// Currently those are :
|
||||
// - A PVC
|
||||
// - A ClusterIP Service
|
||||
func (cc *ClusterController) cleanupMemberResources(memberName string, r cassandrav1alpha1.RackSpec, c *cassandrav1alpha1.Cluster) error {
|
||||
|
||||
logger.Infof("%s/%s - Cleaning up resources for member %s", c.Namespace, c.Name, memberName)
|
||||
// Delete PVC
|
||||
if len(r.Storage.VolumeClaimTemplates) > 0 {
|
||||
// PVC naming convention for StatefulSets is <volumeClaimTemplate.Name>-<pod.Name>
|
||||
pvcName := fmt.Sprintf("%s-%s", r.Storage.VolumeClaimTemplates[0].Name, memberName)
|
||||
err := cc.kubeClient.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(pvcName, &metav1.DeleteOptions{})
|
||||
if err != nil && !apierrors.IsNotFound(err) {
|
||||
return fmt.Errorf("error deleting pvc %s: %s", pvcName, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// Delete Member Service
|
||||
err := cc.kubeClient.CoreV1().Services(c.Namespace).Delete(memberName, &metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error deleting member service %s: %s", memberName, err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -19,6 +19,7 @@ package controller
|
||||
import (
|
||||
"fmt"
|
||||
cassandrav1alpha1 "github.com/rook/rook/pkg/apis/cassandra.rook.io/v1alpha1"
|
||||
"github.com/rook/rook/pkg/operator/cassandra/constants"
|
||||
"github.com/rook/rook/pkg/operator/cassandra/controller/util"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
@@ -55,6 +56,30 @@ func (cc *ClusterController) updateStatus(c *cassandrav1alpha1.Cluster) error {
|
||||
// Update ReadyMembers
|
||||
status.ReadyMembers = sts.Status.ReadyReplicas
|
||||
|
||||
// Update Scaling Down condition
|
||||
services, err := util.GerMemberServicesForRack(rack, c, cc.serviceLister)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error trying to get Pods for rack %s", rack.Name)
|
||||
}
|
||||
for _, svc := range services {
|
||||
// Check if there is a decommission in progress
|
||||
if _, ok := svc.Labels[constants.DecommissionLabel]; ok {
|
||||
// Add MemberLeaving Condition to rack status
|
||||
status.Conditions = append(status.Conditions, cassandrav1alpha1.RackCondition{
|
||||
Type: cassandrav1alpha1.RackConditionTypeMemberLeaving,
|
||||
Status: cassandrav1alpha1.ConditionTrue,
|
||||
})
|
||||
// Sanity check. Only the last member should be decommissioning.
|
||||
index, err := util.IndexFromName(svc.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if index != status.Members-1 {
|
||||
return fmt.Errorf("only last member of each rack should be decommissioning, but %d-th member of %s found decommissioning while rack had %d members", index, rack.Name, status.Members)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update Status for Rack
|
||||
clusterStatus.Racks[rack.Name] = status
|
||||
}
|
||||
@@ -77,6 +102,15 @@ func (cc *ClusterController) syncCluster(c *cassandrav1alpha1.Cluster) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Check if there is a scale-down in progress
|
||||
for _, rack := range c.Spec.Datacenter.Racks {
|
||||
if util.IsRackConditionTrue(c.Status.Racks[rack.Name], cassandrav1alpha1.RackConditionTypeMemberLeaving) {
|
||||
// Resume scale down
|
||||
err := cc.scaleDownRack(rack, c)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Check that all racks are ready before taking any action
|
||||
for _, rack := range c.Spec.Datacenter.Racks {
|
||||
rackStatus := c.Status.Racks[rack.Name]
|
||||
@@ -86,6 +120,15 @@ func (cc *ClusterController) syncCluster(c *cassandrav1alpha1.Cluster) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Check if any rack needs to scale down
|
||||
for _, rack := range c.Spec.Datacenter.Racks {
|
||||
if rack.Members < c.Status.Racks[rack.Name].Members {
|
||||
// scale down
|
||||
err := cc.scaleDownRack(rack, c)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Check if any rack needs to scale up
|
||||
for _, rack := range c.Spec.Datacenter.Racks {
|
||||
|
||||
@@ -154,3 +197,70 @@ func (cc *ClusterController) scaleUpRack(r cassandrav1alpha1.RackSpec, c *cassan
|
||||
return err
|
||||
|
||||
}
|
||||
|
||||
// scaleDownRack handles scaling down for an existing Cassandra Rack.
|
||||
// Calling this action implies all members of the Rack are Ready.
|
||||
func (cc *ClusterController) scaleDownRack(r cassandrav1alpha1.RackSpec, c *cassandrav1alpha1.Cluster) error {
|
||||
|
||||
logger.Infof("Scaling down rack %s", r.Name)
|
||||
|
||||
// Get the current actual number of Members
|
||||
members := c.Status.Racks[r.Name].Members
|
||||
|
||||
// Find the member to decommission
|
||||
memberName := fmt.Sprintf("%s-%d", util.StatefulSetNameForRack(r, c), members-1)
|
||||
logger.Infof("Member of interest: %s", memberName)
|
||||
memberService, err := cc.serviceLister.Services(c.Namespace).Get(memberName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error trying to get Member Service %s: %s", memberName, err.Error())
|
||||
}
|
||||
|
||||
// Check if there was a scale down in progress that has completed.
|
||||
if memberService.Labels[constants.DecommissionLabel] == constants.LabelValueTrue {
|
||||
|
||||
logger.Infof("Found decommissioned member: %s", memberName)
|
||||
|
||||
// Get rack's statefulset
|
||||
stsName := util.StatefulSetNameForRack(r, c)
|
||||
sts, err := cc.statefulSetLister.StatefulSets(c.Namespace).Get(stsName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error trying to get StatefulSet %s", stsName)
|
||||
}
|
||||
// Scale the statefulset
|
||||
err = util.ScaleStatefulSet(sts, -1, cc.kubeClient)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error trying to scale down StatefulSet %s", stsName)
|
||||
}
|
||||
// Cleanup is done on each sync loop, no need to do anything else here
|
||||
|
||||
cc.recorder.Event(
|
||||
c,
|
||||
corev1.EventTypeNormal,
|
||||
SuccessSynced,
|
||||
fmt.Sprintf(MessageRackScaledDown, r.Name, members-1),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Infof("Checking for scale down. Desired: %d. Actual: %d", r.Members, c.Status.Racks[r.Name].Members)
|
||||
// Then, check if there is a requested scale down.
|
||||
if r.Members < c.Status.Racks[r.Name].Members {
|
||||
|
||||
logger.Infof("Scale down requested, member %s will decommission", memberName)
|
||||
// Record the intent to decommission the member
|
||||
old := memberService.DeepCopy()
|
||||
memberService.Labels[constants.DecommissionLabel] = constants.LabelValueFalse
|
||||
if err := util.PatchService(old, memberService, cc.kubeClient); err != nil {
|
||||
return fmt.Errorf("error patching member service %s: %s", memberName, err.Error())
|
||||
}
|
||||
|
||||
cc.recorder.Event(
|
||||
c,
|
||||
corev1.EventTypeNormal,
|
||||
SuccessSynced,
|
||||
fmt.Sprintf(MessageRackScaleDownInProgress, r.Name, members-1),
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -34,7 +34,6 @@ import (
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
appsinformers "k8s.io/client-go/informers/apps/v1"
|
||||
coreinformers "k8s.io/client-go/informers/core/v1"
|
||||
rbacinformers "k8s.io/client-go/informers/rbac/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/kubernetes/scheme"
|
||||
typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
|
||||
@@ -88,10 +87,6 @@ func New(
|
||||
statefulSetInformer appsinformers.StatefulSetInformer,
|
||||
serviceInformer coreinformers.ServiceInformer,
|
||||
podInformer coreinformers.PodInformer,
|
||||
serviceAccountInformer coreinformers.ServiceAccountInformer,
|
||||
roleInformer rbacinformers.RoleInformer,
|
||||
roleBindingInformer rbacinformers.RoleBindingInformer,
|
||||
|
||||
) *ClusterController {
|
||||
|
||||
// Add sample-controller types to the default Kubernetes Scheme so Events can be
|
||||
@@ -139,9 +134,12 @@ func New(
|
||||
}
|
||||
cc.enqueueCluster(newCluster)
|
||||
},
|
||||
DeleteFunc: func(obj interface{}) {
|
||||
// TODO: handle deletion
|
||||
},
|
||||
//Deletion handling:
|
||||
// Atm, the only thing left behind will be the state, ie
|
||||
// the PVCs that the StatefulSets don't erase.
|
||||
// This behaviour may actually be preferrable to deleting them,
|
||||
// since it ensures that no data will be lost if someone accidentally
|
||||
// deletes the cluster.
|
||||
})
|
||||
|
||||
statefulSetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
|
||||
|
||||
@@ -18,6 +18,7 @@ package controller
|
||||
|
||||
import (
|
||||
cassandrav1alpha1 "github.com/rook/rook/pkg/apis/cassandra.rook.io/v1alpha1"
|
||||
"github.com/rook/rook/pkg/operator/cassandra/controller/util"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
@@ -30,13 +31,16 @@ const (
|
||||
// existing.
|
||||
ErrSyncFailed = "ErrSyncFailed"
|
||||
|
||||
MessageRackCreated = "Rack %s created"
|
||||
MessageRackScaledUp = "Rack %s scaled up to %d members"
|
||||
MessageRackCreated = "Rack %s created"
|
||||
MessageRackScaledUp = "Rack %s scaled up to %d members"
|
||||
MessageRackScaleDownInProgress = "Rack %s scaling down to %d members"
|
||||
MessageRackScaledDown = "Rack %s scaled down to %d members"
|
||||
|
||||
// Messages to display when experiencing an error.
|
||||
MessageHeadlessServiceSyncFailed = "Failed to sync Headless Service for cluster"
|
||||
MessageMemberServicesSyncFailed = "Failed to sync MemberServices for cluster"
|
||||
MessageUpdateStatusFailed = "Failed to update status for cluster"
|
||||
MessageCleanupFailed = "Failed to clean up cluster resources"
|
||||
MessageClusterSyncFailed = "Failed to sync cluster"
|
||||
)
|
||||
|
||||
@@ -44,6 +48,25 @@ const (
|
||||
// NOTE: the Cluster Object is a DeepCopy. Modify at will.
|
||||
func (cc *ClusterController) Sync(c *cassandrav1alpha1.Cluster) error {
|
||||
|
||||
// Before syncing, ensure that all StatefulSets are up-to-date
|
||||
stale, err := util.StatefulSetStatusesStale(c, cc.statefulSetLister)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if stale {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Cleanup Cluster resources
|
||||
if err := cc.cleanup(c); err != nil {
|
||||
cc.recorder.Event(
|
||||
c,
|
||||
corev1.EventTypeWarning,
|
||||
ErrSyncFailed,
|
||||
MessageCleanupFailed,
|
||||
)
|
||||
}
|
||||
|
||||
// Sync Headless Service for Cluster
|
||||
if err := cc.syncClusterHeadlessService(c); err != nil {
|
||||
cc.recorder.Event(
|
||||
|
||||
@@ -27,9 +27,9 @@ import (
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
// PatchPod patches the old Pod so that it matches the
|
||||
// new Pod.
|
||||
func PatchPod(old, new *corev1.Pod, kubeClient kubernetes.Interface) error {
|
||||
// PatchService patches the old Service so that it matches the
|
||||
// new Service.
|
||||
func PatchService(old, new *corev1.Service, kubeClient kubernetes.Interface) error {
|
||||
|
||||
oldJSON, err := json.Marshal(old)
|
||||
if err != nil {
|
||||
@@ -41,12 +41,12 @@ func PatchPod(old, new *corev1.Pod, kubeClient kubernetes.Interface) error {
|
||||
return err
|
||||
}
|
||||
|
||||
patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldJSON, newJSON, corev1.Pod{})
|
||||
patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldJSON, newJSON, corev1.Service{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = kubeClient.CoreV1().Pods(old.Namespace).Patch(old.Name, types.StrategicMergePatchType, patchBytes)
|
||||
_, err = kubeClient.CoreV1().Services(old.Namespace).Patch(old.Name, types.StrategicMergePatchType, patchBytes)
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,10 @@ import (
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/selection"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
appslisters "k8s.io/client-go/listers/apps/v1"
|
||||
corelisters "k8s.io/client-go/listers/core/v1"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// GetPodsForCluster returns the existing Pods for
|
||||
@@ -44,6 +47,29 @@ func GetPodsForCluster(cluster *cassandrav1alpha1.Cluster, podLister corelisters
|
||||
|
||||
}
|
||||
|
||||
// GetMemberServicesForRack returns the member services for the given rack.
|
||||
func GerMemberServicesForRack(
|
||||
r cassandrav1alpha1.RackSpec,
|
||||
c *cassandrav1alpha1.Cluster,
|
||||
serviceLister corelisters.ServiceLister,
|
||||
) ([]*corev1.Service, error) {
|
||||
|
||||
sel := RackSelector(r, c)
|
||||
return serviceLister.Services(c.Namespace).List(sel)
|
||||
}
|
||||
|
||||
// GetPodsForRack returns the created Pods for the given rack.
|
||||
func GetPodsForRack(
|
||||
r cassandrav1alpha1.RackSpec,
|
||||
c *cassandrav1alpha1.Cluster,
|
||||
podLister corelisters.PodLister,
|
||||
) ([]*corev1.Pod, error) {
|
||||
|
||||
sel := RackSelector(r, c)
|
||||
return podLister.Pods(c.Namespace).List(sel)
|
||||
|
||||
}
|
||||
|
||||
// VerifyOwner checks if the owner Object is the controller
|
||||
// of the obj Object and returns an error if it isn't.
|
||||
func VerifyOwner(obj, owner metav1.Object) error {
|
||||
@@ -70,6 +96,49 @@ func NewControllerRef(c *cassandrav1alpha1.Cluster) metav1.OwnerReference {
|
||||
})
|
||||
}
|
||||
|
||||
// RefFromString is a helper function that takes a string
|
||||
// and outputs a reference to that string.
|
||||
// Useful for initializing a string pointer from a literal.
|
||||
func RefFromString(s string) *string {
|
||||
return &s
|
||||
}
|
||||
|
||||
// RefFromInt is a helper function that takes a int
|
||||
// and outputs a reference to that int.
|
||||
// Useful for initializing an int pointer from a literal.
|
||||
func RefFromInt(i int32) *int32 {
|
||||
return &i
|
||||
}
|
||||
|
||||
// IndexFromName attempts to get the index from a name using the
|
||||
// naming convention <name>-<index>.
|
||||
func IndexFromName(n string) (int32, error) {
|
||||
|
||||
// index := svc.Name[strings.LastIndex(svc.Name, "-") + 1 : len(svc.Name)]
|
||||
delimIndex := strings.LastIndex(n, "-")
|
||||
if delimIndex == -1 {
|
||||
return -1, fmt.Errorf("couldn't get index from name %s", n)
|
||||
}
|
||||
|
||||
index, err := strconv.Atoi(n[delimIndex+1:])
|
||||
if err != nil {
|
||||
return -1, fmt.Errorf("couldn't get index from name %s", n)
|
||||
}
|
||||
|
||||
return int32(index), nil
|
||||
}
|
||||
|
||||
// isPodUnschedulable iterates a Pod's Status.Conditions to find out
|
||||
// if it has been deemed unschedulable
|
||||
func IsPodUnschedulable(pod *corev1.Pod) bool {
|
||||
for _, v := range pod.Status.Conditions {
|
||||
if v.Reason == corev1.PodReasonUnschedulable {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// RefFromInt32 is a helper function that takes a int32
|
||||
// and outputs a reference to that int.
|
||||
func RefFromInt32(i int32) *int32 {
|
||||
@@ -109,3 +178,35 @@ func ScaleStatefulSet(sts *appsv1.StatefulSet, amount int32, kubeClient kubernet
|
||||
err := PatchStatefulSet(sts, updatedSts, kubeClient)
|
||||
return err
|
||||
}
|
||||
|
||||
// IsRackConditionTrue checks a rack's status for the presense of a condition type
|
||||
// and checks if it is true.
|
||||
func IsRackConditionTrue(rackStatus *cassandrav1alpha1.RackStatus, condType cassandrav1alpha1.RackConditionType) bool {
|
||||
for _, cond := range rackStatus.Conditions {
|
||||
if cond.Type == cassandrav1alpha1.RackConditionTypeMemberLeaving && cond.Status == cassandrav1alpha1.ConditionTrue {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// StatefulSetStatusesStale checks if the StatefulSet Objects of a Cluster
|
||||
// have been observed by the StatefulSet controller.
|
||||
// If they haven't, their status might be stale, so it's better to wait
|
||||
// and process them later.
|
||||
func StatefulSetStatusesStale(c *cassandrav1alpha1.Cluster, statefulSetLister appslisters.StatefulSetLister) (bool, error) {
|
||||
// Before proceeding, ensure all the Statefulset Statuses are valid
|
||||
for _, r := range c.Spec.Datacenter.Racks {
|
||||
if _, ok := c.Status.Racks[r.Name]; !ok {
|
||||
continue
|
||||
}
|
||||
sts, err := statefulSetLister.StatefulSets(c.Namespace).Get(StatefulSetNameForRack(r, c))
|
||||
if err != nil {
|
||||
return true, fmt.Errorf("error getting statefulset: %s", err.Error())
|
||||
}
|
||||
if sts.Generation != sts.Status.ObservedGeneration {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
Reference in new issue
Block a user