cassandra: controller-side implementation of scale down

Code handling the scale down on the side of the controller.
Also, some new util functions.

Signed-off-by: Yannis Zarkadas <yanniszarkadas@gmail.com>
This commit is contained in:
Yannis Zarkadas committed 2018-12-07 13:46:45 +02:00
1 parent 624ff18211
commit f5a41e9a7c
7 files changed
+338 -18

No files matched your search

-3
View File
@@ -87,9 +87,6 @@ func startOperator(cmd *cobra.Command, args []string) error {
kubeInformerFactory.Apps().V1().StatefulSets(),
kubeInformerFactory.Core().V1().Services(),
kubeInformerFactory.Core().V1().Pods(),
kubeInformerFactory.Core().V1().ServiceAccounts(),
kubeInformerFactory.Rbac().V1().Roles(),
kubeInformerFactory.Rbac().V1().RoleBindings(),
)
// Create a channel to receive OS signals
@@ -0,0 +1,91 @@
/*
Copyright 2018 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controller
import (
"fmt"
cassandrav1alpha1 "github.com/rook/rook/pkg/apis/cassandra.rook.io/v1alpha1"
"github.com/rook/rook/pkg/operator/cassandra/controller/util"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// cleanup deletes all resources remaining because of cluster scale downs
func (cc *ClusterController) cleanup(c *cassandrav1alpha1.Cluster) error {
for _, r := range c.Spec.Datacenter.Racks {
services, err := cc.serviceLister.Services(c.Namespace).List(util.RackSelector(r, c))
if err != nil {
return fmt.Errorf("error listing member services: %s", err.Error())
}
// Get rack status. If it doesn't exist, the rack isn't yet created.
stsName := util.StatefulSetNameForRack(r, c)
sts, err := cc.statefulSetLister.StatefulSets(c.Namespace).Get(stsName)
if apierrors.IsNotFound(err) {
continue
}
if err != nil {
return fmt.Errorf("error getting statefulset %s: %s", stsName, err.Error())
}
memberCount := *sts.Spec.Replicas
memberServiceCount := int32(len(services))
// If there are more services than members, some services need to be cleaned up
if memberServiceCount > memberCount {
maxIndex := memberCount - 1
for _, svc := range services {
svcIndex, err := util.IndexFromName(svc.Name)
if err != nil {
logger.Errorf("Unexpected error while parsing index from name %s : %s", svc.Name, err.Error())
continue
}
if svcIndex > maxIndex {
err := cc.cleanupMemberResources(svc.Name, r, c)
if err != nil {
return fmt.Errorf("error cleaning up member resources: %s", err.Error())
}
}
}
}
}
logger.Infof("%s/%s - Successfully cleaned up cluster.", c.Namespace, c.Name)
return nil
}
// cleanupMemberResources deletes all resources associated with a given member.
// Currently those are :
// - A PVC
// - A ClusterIP Service
func (cc *ClusterController) cleanupMemberResources(memberName string, r cassandrav1alpha1.RackSpec, c *cassandrav1alpha1.Cluster) error {
logger.Infof("%s/%s - Cleaning up resources for member %s", c.Namespace, c.Name, memberName)
// Delete PVC
if len(r.Storage.VolumeClaimTemplates) > 0 {
// PVC naming convention for StatefulSets is <volumeClaimTemplate.Name>-<pod.Name>
pvcName := fmt.Sprintf("%s-%s", r.Storage.VolumeClaimTemplates[0].Name, memberName)
err := cc.kubeClient.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(pvcName, &metav1.DeleteOptions{})
if err != nil && !apierrors.IsNotFound(err) {
return fmt.Errorf("error deleting pvc %s: %s", pvcName, err.Error())
}
}
// Delete Member Service
err := cc.kubeClient.CoreV1().Services(c.Namespace).Delete(memberName, &metav1.DeleteOptions{})
if err != nil {
return fmt.Errorf("error deleting member service %s: %s", memberName, err.Error())
}
return nil
}
@@ -19,6 +19,7 @@ package controller
import (
"fmt"
cassandrav1alpha1 "github.com/rook/rook/pkg/apis/cassandra.rook.io/v1alpha1"
"github.com/rook/rook/pkg/operator/cassandra/constants"
"github.com/rook/rook/pkg/operator/cassandra/controller/util"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -55,6 +56,30 @@ func (cc *ClusterController) updateStatus(c *cassandrav1alpha1.Cluster) error {
// Update ReadyMembers
status.ReadyMembers = sts.Status.ReadyReplicas
// Update Scaling Down condition
services, err := util.GerMemberServicesForRack(rack, c, cc.serviceLister)
if err != nil {
return fmt.Errorf("error trying to get Pods for rack %s", rack.Name)
}
for _, svc := range services {
// Check if there is a decommission in progress
if _, ok := svc.Labels[constants.DecommissionLabel]; ok {
// Add MemberLeaving Condition to rack status
status.Conditions = append(status.Conditions, cassandrav1alpha1.RackCondition{
Type: cassandrav1alpha1.RackConditionTypeMemberLeaving,
Status: cassandrav1alpha1.ConditionTrue,
})
// Sanity check. Only the last member should be decommissioning.
index, err := util.IndexFromName(svc.Name)
if err != nil {
return err
}
if index != status.Members-1 {
return fmt.Errorf("only last member of each rack should be decommissioning, but %d-th member of %s found decommissioning while rack had %d members", index, rack.Name, status.Members)
}
}
}
// Update Status for Rack
clusterStatus.Racks[rack.Name] = status
}
@@ -77,6 +102,15 @@ func (cc *ClusterController) syncCluster(c *cassandrav1alpha1.Cluster) error {
}
}
// Check if there is a scale-down in progress
for _, rack := range c.Spec.Datacenter.Racks {
if util.IsRackConditionTrue(c.Status.Racks[rack.Name], cassandrav1alpha1.RackConditionTypeMemberLeaving) {
// Resume scale down
err := cc.scaleDownRack(rack, c)
return err
}
}
// Check that all racks are ready before taking any action
for _, rack := range c.Spec.Datacenter.Racks {
rackStatus := c.Status.Racks[rack.Name]
@@ -86,6 +120,15 @@ func (cc *ClusterController) syncCluster(c *cassandrav1alpha1.Cluster) error {
}
}
// Check if any rack needs to scale down
for _, rack := range c.Spec.Datacenter.Racks {
if rack.Members < c.Status.Racks[rack.Name].Members {
// scale down
err := cc.scaleDownRack(rack, c)
return err
}
}
// Check if any rack needs to scale up
for _, rack := range c.Spec.Datacenter.Racks {
@@ -154,3 +197,70 @@ func (cc *ClusterController) scaleUpRack(r cassandrav1alpha1.RackSpec, c *cassan
return err
}
// scaleDownRack handles scaling down for an existing Cassandra Rack.
// Calling this action implies all members of the Rack are Ready.
func (cc *ClusterController) scaleDownRack(r cassandrav1alpha1.RackSpec, c *cassandrav1alpha1.Cluster) error {
logger.Infof("Scaling down rack %s", r.Name)
// Get the current actual number of Members
members := c.Status.Racks[r.Name].Members
// Find the member to decommission
memberName := fmt.Sprintf("%s-%d", util.StatefulSetNameForRack(r, c), members-1)
logger.Infof("Member of interest: %s", memberName)
memberService, err := cc.serviceLister.Services(c.Namespace).Get(memberName)
if err != nil {
return fmt.Errorf("error trying to get Member Service %s: %s", memberName, err.Error())
}
// Check if there was a scale down in progress that has completed.
if memberService.Labels[constants.DecommissionLabel] == constants.LabelValueTrue {
logger.Infof("Found decommissioned member: %s", memberName)
// Get rack's statefulset
stsName := util.StatefulSetNameForRack(r, c)
sts, err := cc.statefulSetLister.StatefulSets(c.Namespace).Get(stsName)
if err != nil {
return fmt.Errorf("error trying to get StatefulSet %s", stsName)
}
// Scale the statefulset
err = util.ScaleStatefulSet(sts, -1, cc.kubeClient)
if err != nil {
return fmt.Errorf("error trying to scale down StatefulSet %s", stsName)
}
// Cleanup is done on each sync loop, no need to do anything else here
cc.recorder.Event(
c,
corev1.EventTypeNormal,
SuccessSynced,
fmt.Sprintf(MessageRackScaledDown, r.Name, members-1),
)
return nil
}
logger.Infof("Checking for scale down. Desired: %d. Actual: %d", r.Members, c.Status.Racks[r.Name].Members)
// Then, check if there is a requested scale down.
if r.Members < c.Status.Racks[r.Name].Members {
logger.Infof("Scale down requested, member %s will decommission", memberName)
// Record the intent to decommission the member
old := memberService.DeepCopy()
memberService.Labels[constants.DecommissionLabel] = constants.LabelValueFalse
if err := util.PatchService(old, memberService, cc.kubeClient); err != nil {
return fmt.Errorf("error patching member service %s: %s", memberName, err.Error())
}
cc.recorder.Event(
c,
corev1.EventTypeNormal,
SuccessSynced,
fmt.Sprintf(MessageRackScaleDownInProgress, r.Name, members-1),
)
}
return nil
}
@@ -34,7 +34,6 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
appsinformers "k8s.io/client-go/informers/apps/v1"
coreinformers "k8s.io/client-go/informers/core/v1"
rbacinformers "k8s.io/client-go/informers/rbac/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -88,10 +87,6 @@ func New(
statefulSetInformer appsinformers.StatefulSetInformer,
serviceInformer coreinformers.ServiceInformer,
podInformer coreinformers.PodInformer,
serviceAccountInformer coreinformers.ServiceAccountInformer,
roleInformer rbacinformers.RoleInformer,
roleBindingInformer rbacinformers.RoleBindingInformer,
) *ClusterController {
// Add sample-controller types to the default Kubernetes Scheme so Events can be
@@ -139,9 +134,12 @@ func New(
}
cc.enqueueCluster(newCluster)
},
DeleteFunc: func(obj interface{}) {
// TODO: handle deletion
},
//Deletion handling:
// Atm, the only thing left behind will be the state, ie
// the PVCs that the StatefulSets don't erase.
// This behaviour may actually be preferrable to deleting them,
// since it ensures that no data will be lost if someone accidentally
// deletes the cluster.
})
statefulSetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+25 -2
View File
@@ -18,6 +18,7 @@ package controller
import (
cassandrav1alpha1 "github.com/rook/rook/pkg/apis/cassandra.rook.io/v1alpha1"
"github.com/rook/rook/pkg/operator/cassandra/controller/util"
corev1 "k8s.io/api/core/v1"
)
@@ -30,13 +31,16 @@ const (
// existing.
ErrSyncFailed = "ErrSyncFailed"
MessageRackCreated = "Rack %s created"
MessageRackScaledUp = "Rack %s scaled up to %d members"
MessageRackCreated = "Rack %s created"
MessageRackScaledUp = "Rack %s scaled up to %d members"
MessageRackScaleDownInProgress = "Rack %s scaling down to %d members"
MessageRackScaledDown = "Rack %s scaled down to %d members"
// Messages to display when experiencing an error.
MessageHeadlessServiceSyncFailed = "Failed to sync Headless Service for cluster"
MessageMemberServicesSyncFailed = "Failed to sync MemberServices for cluster"
MessageUpdateStatusFailed = "Failed to update status for cluster"
MessageCleanupFailed = "Failed to clean up cluster resources"
MessageClusterSyncFailed = "Failed to sync cluster"
)
@@ -44,6 +48,25 @@ const (
// NOTE: the Cluster Object is a DeepCopy. Modify at will.
func (cc *ClusterController) Sync(c *cassandrav1alpha1.Cluster) error {
// Before syncing, ensure that all StatefulSets are up-to-date
stale, err := util.StatefulSetStatusesStale(c, cc.statefulSetLister)
if err != nil {
return err
}
if stale {
return nil
}
// Cleanup Cluster resources
if err := cc.cleanup(c); err != nil {
cc.recorder.Event(
c,
corev1.EventTypeWarning,
ErrSyncFailed,
MessageCleanupFailed,
)
}
// Sync Headless Service for Cluster
if err := cc.syncClusterHeadlessService(c); err != nil {
cc.recorder.Event(
@@ -27,9 +27,9 @@ import (
"k8s.io/client-go/kubernetes"
)
// PatchPod patches the old Pod so that it matches the
// new Pod.
func PatchPod(old, new *corev1.Pod, kubeClient kubernetes.Interface) error {
// PatchService patches the old Service so that it matches the
// new Service.
func PatchService(old, new *corev1.Service, kubeClient kubernetes.Interface) error {
oldJSON, err := json.Marshal(old)
if err != nil {
@@ -41,12 +41,12 @@ func PatchPod(old, new *corev1.Pod, kubeClient kubernetes.Interface) error {
return err
}
patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldJSON, newJSON, corev1.Pod{})
patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldJSON, newJSON, corev1.Service{})
if err != nil {
return err
}
_, err = kubeClient.CoreV1().Pods(old.Namespace).Patch(old.Name, types.StrategicMergePatchType, patchBytes)
_, err = kubeClient.CoreV1().Services(old.Namespace).Patch(old.Name, types.StrategicMergePatchType, patchBytes)
return err
}
@@ -28,7 +28,10 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/selection"
"k8s.io/client-go/kubernetes"
appslisters "k8s.io/client-go/listers/apps/v1"
corelisters "k8s.io/client-go/listers/core/v1"
"strconv"
"strings"
)
// GetPodsForCluster returns the existing Pods for
@@ -44,6 +47,29 @@ func GetPodsForCluster(cluster *cassandrav1alpha1.Cluster, podLister corelisters
}
// GetMemberServicesForRack returns the member services for the given rack.
func GerMemberServicesForRack(
r cassandrav1alpha1.RackSpec,
c *cassandrav1alpha1.Cluster,
serviceLister corelisters.ServiceLister,
) ([]*corev1.Service, error) {
sel := RackSelector(r, c)
return serviceLister.Services(c.Namespace).List(sel)
}
// GetPodsForRack returns the created Pods for the given rack.
func GetPodsForRack(
r cassandrav1alpha1.RackSpec,
c *cassandrav1alpha1.Cluster,
podLister corelisters.PodLister,
) ([]*corev1.Pod, error) {
sel := RackSelector(r, c)
return podLister.Pods(c.Namespace).List(sel)
}
// VerifyOwner checks if the owner Object is the controller
// of the obj Object and returns an error if it isn't.
func VerifyOwner(obj, owner metav1.Object) error {
@@ -70,6 +96,49 @@ func NewControllerRef(c *cassandrav1alpha1.Cluster) metav1.OwnerReference {
})
}
// RefFromString is a helper function that takes a string
// and outputs a reference to that string.
// Useful for initializing a string pointer from a literal.
func RefFromString(s string) *string {
return &s
}
// RefFromInt is a helper function that takes a int
// and outputs a reference to that int.
// Useful for initializing an int pointer from a literal.
func RefFromInt(i int32) *int32 {
return &i
}
// IndexFromName attempts to get the index from a name using the
// naming convention <name>-<index>.
func IndexFromName(n string) (int32, error) {
// index := svc.Name[strings.LastIndex(svc.Name, "-") + 1 : len(svc.Name)]
delimIndex := strings.LastIndex(n, "-")
if delimIndex == -1 {
return -1, fmt.Errorf("couldn't get index from name %s", n)
}
index, err := strconv.Atoi(n[delimIndex+1:])
if err != nil {
return -1, fmt.Errorf("couldn't get index from name %s", n)
}
return int32(index), nil
}
// isPodUnschedulable iterates a Pod's Status.Conditions to find out
// if it has been deemed unschedulable
func IsPodUnschedulable(pod *corev1.Pod) bool {
for _, v := range pod.Status.Conditions {
if v.Reason == corev1.PodReasonUnschedulable {
return true
}
}
return false
}
// RefFromInt32 is a helper function that takes a int32
// and outputs a reference to that int.
func RefFromInt32(i int32) *int32 {
@@ -109,3 +178,35 @@ func ScaleStatefulSet(sts *appsv1.StatefulSet, amount int32, kubeClient kubernet
err := PatchStatefulSet(sts, updatedSts, kubeClient)
return err
}
// IsRackConditionTrue checks a rack's status for the presense of a condition type
// and checks if it is true.
func IsRackConditionTrue(rackStatus *cassandrav1alpha1.RackStatus, condType cassandrav1alpha1.RackConditionType) bool {
for _, cond := range rackStatus.Conditions {
if cond.Type == cassandrav1alpha1.RackConditionTypeMemberLeaving && cond.Status == cassandrav1alpha1.ConditionTrue {
return true
}
}
return false
}
// StatefulSetStatusesStale checks if the StatefulSet Objects of a Cluster
// have been observed by the StatefulSet controller.
// If they haven't, their status might be stale, so it's better to wait
// and process them later.
func StatefulSetStatusesStale(c *cassandrav1alpha1.Cluster, statefulSetLister appslisters.StatefulSetLister) (bool, error) {
// Before proceeding, ensure all the Statefulset Statuses are valid
for _, r := range c.Spec.Datacenter.Racks {
if _, ok := c.Status.Racks[r.Name]; !ok {
continue
}
sts, err := statefulSetLister.StatefulSets(c.Namespace).Get(StatefulSetNameForRack(r, c))
if err != nil {
return true, fmt.Errorf("error getting statefulset: %s", err.Error())
}
if sts.Generation != sts.Status.ObservedGeneration {
return true, nil
}
}
return false, nil
}