From f5a41e9a7cbe3ba09bc5028ccbc83ef171d1eae0 Mon Sep 17 00:00:00 2001 From: Yannis Zarkadas Date: Tue, 4 Dec 2018 02:06:00 +0200 Subject: [PATCH] cassandra: controller-side implementation of scale down Code handling the scale down on the side of the controller. Also, some new util functions. Signed-off-by: Yannis Zarkadas --- cmd/rook/cassandra/operator.go | 3 - pkg/operator/cassandra/controller/cleanup.go | 91 +++++++++++++++ pkg/operator/cassandra/controller/cluster.go | 110 ++++++++++++++++++ .../cassandra/controller/controller.go | 14 +-- pkg/operator/cassandra/controller/sync.go | 27 ++++- .../cassandra/controller/util/patch.go | 10 +- .../cassandra/controller/util/util.go | 101 ++++++++++++++++ 7 files changed, 338 insertions(+), 18 deletions(-) create mode 100644 pkg/operator/cassandra/controller/cleanup.go diff --git a/cmd/rook/cassandra/operator.go b/cmd/rook/cassandra/operator.go index 3022000ee..759811772 100644 --- a/cmd/rook/cassandra/operator.go +++ b/cmd/rook/cassandra/operator.go @@ -87,9 +87,6 @@ func startOperator(cmd *cobra.Command, args []string) error { kubeInformerFactory.Apps().V1().StatefulSets(), kubeInformerFactory.Core().V1().Services(), kubeInformerFactory.Core().V1().Pods(), - kubeInformerFactory.Core().V1().ServiceAccounts(), - kubeInformerFactory.Rbac().V1().Roles(), - kubeInformerFactory.Rbac().V1().RoleBindings(), ) // Create a channel to receive OS signals diff --git a/pkg/operator/cassandra/controller/cleanup.go b/pkg/operator/cassandra/controller/cleanup.go new file mode 100644 index 000000000..1b3b1013e --- /dev/null +++ b/pkg/operator/cassandra/controller/cleanup.go @@ -0,0 +1,91 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +    http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "fmt" + cassandrav1alpha1 "github.com/rook/rook/pkg/apis/cassandra.rook.io/v1alpha1" + "github.com/rook/rook/pkg/operator/cassandra/controller/util" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// cleanup deletes all resources remaining because of cluster scale downs +func (cc *ClusterController) cleanup(c *cassandrav1alpha1.Cluster) error { + + for _, r := range c.Spec.Datacenter.Racks { + services, err := cc.serviceLister.Services(c.Namespace).List(util.RackSelector(r, c)) + if err != nil { + return fmt.Errorf("error listing member services: %s", err.Error()) + } + // Get rack status. If it doesn't exist, the rack isn't yet created. + stsName := util.StatefulSetNameForRack(r, c) + sts, err := cc.statefulSetLister.StatefulSets(c.Namespace).Get(stsName) + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return fmt.Errorf("error getting statefulset %s: %s", stsName, err.Error()) + } + memberCount := *sts.Spec.Replicas + memberServiceCount := int32(len(services)) + // If there are more services than members, some services need to be cleaned up + if memberServiceCount > memberCount { + maxIndex := memberCount - 1 + for _, svc := range services { + svcIndex, err := util.IndexFromName(svc.Name) + if err != nil { + logger.Errorf("Unexpected error while parsing index from name %s : %s", svc.Name, err.Error()) + continue + } + if svcIndex > maxIndex { + err := cc.cleanupMemberResources(svc.Name, r, c) + if err != nil { + return fmt.Errorf("error cleaning up member resources: %s", err.Error()) + } + } + } + } + } + logger.Infof("%s/%s - Successfully cleaned up cluster.", c.Namespace, c.Name) + return nil +} + +// cleanupMemberResources deletes all resources associated with a given member. +// Currently those are : +// - A PVC +// - A ClusterIP Service +func (cc *ClusterController) cleanupMemberResources(memberName string, r cassandrav1alpha1.RackSpec, c *cassandrav1alpha1.Cluster) error { + + logger.Infof("%s/%s - Cleaning up resources for member %s", c.Namespace, c.Name, memberName) + // Delete PVC + if len(r.Storage.VolumeClaimTemplates) > 0 { + // PVC naming convention for StatefulSets is - + pvcName := fmt.Sprintf("%s-%s", r.Storage.VolumeClaimTemplates[0].Name, memberName) + err := cc.kubeClient.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(pvcName, &metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("error deleting pvc %s: %s", pvcName, err.Error()) + } + } + + // Delete Member Service + err := cc.kubeClient.CoreV1().Services(c.Namespace).Delete(memberName, &metav1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("error deleting member service %s: %s", memberName, err.Error()) + } + return nil +} diff --git a/pkg/operator/cassandra/controller/cluster.go b/pkg/operator/cassandra/controller/cluster.go index a9b3085c6..6eaa0da71 100644 --- a/pkg/operator/cassandra/controller/cluster.go +++ b/pkg/operator/cassandra/controller/cluster.go @@ -19,6 +19,7 @@ package controller import ( "fmt" cassandrav1alpha1 "github.com/rook/rook/pkg/apis/cassandra.rook.io/v1alpha1" + "github.com/rook/rook/pkg/operator/cassandra/constants" "github.com/rook/rook/pkg/operator/cassandra/controller/util" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -55,6 +56,30 @@ func (cc *ClusterController) updateStatus(c *cassandrav1alpha1.Cluster) error { // Update ReadyMembers status.ReadyMembers = sts.Status.ReadyReplicas + // Update Scaling Down condition + services, err := util.GerMemberServicesForRack(rack, c, cc.serviceLister) + if err != nil { + return fmt.Errorf("error trying to get Pods for rack %s", rack.Name) + } + for _, svc := range services { + // Check if there is a decommission in progress + if _, ok := svc.Labels[constants.DecommissionLabel]; ok { + // Add MemberLeaving Condition to rack status + status.Conditions = append(status.Conditions, cassandrav1alpha1.RackCondition{ + Type: cassandrav1alpha1.RackConditionTypeMemberLeaving, + Status: cassandrav1alpha1.ConditionTrue, + }) + // Sanity check. Only the last member should be decommissioning. + index, err := util.IndexFromName(svc.Name) + if err != nil { + return err + } + if index != status.Members-1 { + return fmt.Errorf("only last member of each rack should be decommissioning, but %d-th member of %s found decommissioning while rack had %d members", index, rack.Name, status.Members) + } + } + } + // Update Status for Rack clusterStatus.Racks[rack.Name] = status } @@ -77,6 +102,15 @@ func (cc *ClusterController) syncCluster(c *cassandrav1alpha1.Cluster) error { } } + // Check if there is a scale-down in progress + for _, rack := range c.Spec.Datacenter.Racks { + if util.IsRackConditionTrue(c.Status.Racks[rack.Name], cassandrav1alpha1.RackConditionTypeMemberLeaving) { + // Resume scale down + err := cc.scaleDownRack(rack, c) + return err + } + } + // Check that all racks are ready before taking any action for _, rack := range c.Spec.Datacenter.Racks { rackStatus := c.Status.Racks[rack.Name] @@ -86,6 +120,15 @@ func (cc *ClusterController) syncCluster(c *cassandrav1alpha1.Cluster) error { } } + // Check if any rack needs to scale down + for _, rack := range c.Spec.Datacenter.Racks { + if rack.Members < c.Status.Racks[rack.Name].Members { + // scale down + err := cc.scaleDownRack(rack, c) + return err + } + } + // Check if any rack needs to scale up for _, rack := range c.Spec.Datacenter.Racks { @@ -154,3 +197,70 @@ func (cc *ClusterController) scaleUpRack(r cassandrav1alpha1.RackSpec, c *cassan return err } + +// scaleDownRack handles scaling down for an existing Cassandra Rack. +// Calling this action implies all members of the Rack are Ready. +func (cc *ClusterController) scaleDownRack(r cassandrav1alpha1.RackSpec, c *cassandrav1alpha1.Cluster) error { + + logger.Infof("Scaling down rack %s", r.Name) + + // Get the current actual number of Members + members := c.Status.Racks[r.Name].Members + + // Find the member to decommission + memberName := fmt.Sprintf("%s-%d", util.StatefulSetNameForRack(r, c), members-1) + logger.Infof("Member of interest: %s", memberName) + memberService, err := cc.serviceLister.Services(c.Namespace).Get(memberName) + if err != nil { + return fmt.Errorf("error trying to get Member Service %s: %s", memberName, err.Error()) + } + + // Check if there was a scale down in progress that has completed. + if memberService.Labels[constants.DecommissionLabel] == constants.LabelValueTrue { + + logger.Infof("Found decommissioned member: %s", memberName) + + // Get rack's statefulset + stsName := util.StatefulSetNameForRack(r, c) + sts, err := cc.statefulSetLister.StatefulSets(c.Namespace).Get(stsName) + if err != nil { + return fmt.Errorf("error trying to get StatefulSet %s", stsName) + } + // Scale the statefulset + err = util.ScaleStatefulSet(sts, -1, cc.kubeClient) + if err != nil { + return fmt.Errorf("error trying to scale down StatefulSet %s", stsName) + } + // Cleanup is done on each sync loop, no need to do anything else here + + cc.recorder.Event( + c, + corev1.EventTypeNormal, + SuccessSynced, + fmt.Sprintf(MessageRackScaledDown, r.Name, members-1), + ) + return nil + } + + logger.Infof("Checking for scale down. Desired: %d. Actual: %d", r.Members, c.Status.Racks[r.Name].Members) + // Then, check if there is a requested scale down. + if r.Members < c.Status.Racks[r.Name].Members { + + logger.Infof("Scale down requested, member %s will decommission", memberName) + // Record the intent to decommission the member + old := memberService.DeepCopy() + memberService.Labels[constants.DecommissionLabel] = constants.LabelValueFalse + if err := util.PatchService(old, memberService, cc.kubeClient); err != nil { + return fmt.Errorf("error patching member service %s: %s", memberName, err.Error()) + } + + cc.recorder.Event( + c, + corev1.EventTypeNormal, + SuccessSynced, + fmt.Sprintf(MessageRackScaleDownInProgress, r.Name, members-1), + ) + } + + return nil +} diff --git a/pkg/operator/cassandra/controller/controller.go b/pkg/operator/cassandra/controller/controller.go index ef979abea..e82d17044 100644 --- a/pkg/operator/cassandra/controller/controller.go +++ b/pkg/operator/cassandra/controller/controller.go @@ -34,7 +34,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" appsinformers "k8s.io/client-go/informers/apps/v1" coreinformers "k8s.io/client-go/informers/core/v1" - rbacinformers "k8s.io/client-go/informers/rbac/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" @@ -88,10 +87,6 @@ func New( statefulSetInformer appsinformers.StatefulSetInformer, serviceInformer coreinformers.ServiceInformer, podInformer coreinformers.PodInformer, - serviceAccountInformer coreinformers.ServiceAccountInformer, - roleInformer rbacinformers.RoleInformer, - roleBindingInformer rbacinformers.RoleBindingInformer, - ) *ClusterController { // Add sample-controller types to the default Kubernetes Scheme so Events can be @@ -139,9 +134,12 @@ func New( } cc.enqueueCluster(newCluster) }, - DeleteFunc: func(obj interface{}) { - // TODO: handle deletion - }, + //Deletion handling: + // Atm, the only thing left behind will be the state, ie + // the PVCs that the StatefulSets don't erase. + // This behaviour may actually be preferrable to deleting them, + // since it ensures that no data will be lost if someone accidentally + // deletes the cluster. }) statefulSetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ diff --git a/pkg/operator/cassandra/controller/sync.go b/pkg/operator/cassandra/controller/sync.go index c8bf053a8..48d5b737f 100644 --- a/pkg/operator/cassandra/controller/sync.go +++ b/pkg/operator/cassandra/controller/sync.go @@ -18,6 +18,7 @@ package controller import ( cassandrav1alpha1 "github.com/rook/rook/pkg/apis/cassandra.rook.io/v1alpha1" + "github.com/rook/rook/pkg/operator/cassandra/controller/util" corev1 "k8s.io/api/core/v1" ) @@ -30,13 +31,16 @@ const ( // existing. ErrSyncFailed = "ErrSyncFailed" - MessageRackCreated = "Rack %s created" - MessageRackScaledUp = "Rack %s scaled up to %d members" + MessageRackCreated = "Rack %s created" + MessageRackScaledUp = "Rack %s scaled up to %d members" + MessageRackScaleDownInProgress = "Rack %s scaling down to %d members" + MessageRackScaledDown = "Rack %s scaled down to %d members" // Messages to display when experiencing an error. MessageHeadlessServiceSyncFailed = "Failed to sync Headless Service for cluster" MessageMemberServicesSyncFailed = "Failed to sync MemberServices for cluster" MessageUpdateStatusFailed = "Failed to update status for cluster" + MessageCleanupFailed = "Failed to clean up cluster resources" MessageClusterSyncFailed = "Failed to sync cluster" ) @@ -44,6 +48,25 @@ const ( // NOTE: the Cluster Object is a DeepCopy. Modify at will. func (cc *ClusterController) Sync(c *cassandrav1alpha1.Cluster) error { + // Before syncing, ensure that all StatefulSets are up-to-date + stale, err := util.StatefulSetStatusesStale(c, cc.statefulSetLister) + if err != nil { + return err + } + if stale { + return nil + } + + // Cleanup Cluster resources + if err := cc.cleanup(c); err != nil { + cc.recorder.Event( + c, + corev1.EventTypeWarning, + ErrSyncFailed, + MessageCleanupFailed, + ) + } + // Sync Headless Service for Cluster if err := cc.syncClusterHeadlessService(c); err != nil { cc.recorder.Event( diff --git a/pkg/operator/cassandra/controller/util/patch.go b/pkg/operator/cassandra/controller/util/patch.go index 6e11f231b..8e24d3ea9 100644 --- a/pkg/operator/cassandra/controller/util/patch.go +++ b/pkg/operator/cassandra/controller/util/patch.go @@ -27,9 +27,9 @@ import ( "k8s.io/client-go/kubernetes" ) -// PatchPod patches the old Pod so that it matches the -// new Pod. -func PatchPod(old, new *corev1.Pod, kubeClient kubernetes.Interface) error { +// PatchService patches the old Service so that it matches the +// new Service. +func PatchService(old, new *corev1.Service, kubeClient kubernetes.Interface) error { oldJSON, err := json.Marshal(old) if err != nil { @@ -41,12 +41,12 @@ func PatchPod(old, new *corev1.Pod, kubeClient kubernetes.Interface) error { return err } - patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldJSON, newJSON, corev1.Pod{}) + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldJSON, newJSON, corev1.Service{}) if err != nil { return err } - _, err = kubeClient.CoreV1().Pods(old.Namespace).Patch(old.Name, types.StrategicMergePatchType, patchBytes) + _, err = kubeClient.CoreV1().Services(old.Namespace).Patch(old.Name, types.StrategicMergePatchType, patchBytes) return err } diff --git a/pkg/operator/cassandra/controller/util/util.go b/pkg/operator/cassandra/controller/util/util.go index 2cb46d8b8..20cb36dff 100644 --- a/pkg/operator/cassandra/controller/util/util.go +++ b/pkg/operator/cassandra/controller/util/util.go @@ -28,7 +28,10 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/selection" "k8s.io/client-go/kubernetes" + appslisters "k8s.io/client-go/listers/apps/v1" corelisters "k8s.io/client-go/listers/core/v1" + "strconv" + "strings" ) // GetPodsForCluster returns the existing Pods for @@ -44,6 +47,29 @@ func GetPodsForCluster(cluster *cassandrav1alpha1.Cluster, podLister corelisters } +// GetMemberServicesForRack returns the member services for the given rack. +func GerMemberServicesForRack( + r cassandrav1alpha1.RackSpec, + c *cassandrav1alpha1.Cluster, + serviceLister corelisters.ServiceLister, +) ([]*corev1.Service, error) { + + sel := RackSelector(r, c) + return serviceLister.Services(c.Namespace).List(sel) +} + +// GetPodsForRack returns the created Pods for the given rack. +func GetPodsForRack( + r cassandrav1alpha1.RackSpec, + c *cassandrav1alpha1.Cluster, + podLister corelisters.PodLister, +) ([]*corev1.Pod, error) { + + sel := RackSelector(r, c) + return podLister.Pods(c.Namespace).List(sel) + +} + // VerifyOwner checks if the owner Object is the controller // of the obj Object and returns an error if it isn't. func VerifyOwner(obj, owner metav1.Object) error { @@ -70,6 +96,49 @@ func NewControllerRef(c *cassandrav1alpha1.Cluster) metav1.OwnerReference { }) } +// RefFromString is a helper function that takes a string +// and outputs a reference to that string. +// Useful for initializing a string pointer from a literal. +func RefFromString(s string) *string { + return &s +} + +// RefFromInt is a helper function that takes a int +// and outputs a reference to that int. +// Useful for initializing an int pointer from a literal. +func RefFromInt(i int32) *int32 { + return &i +} + +// IndexFromName attempts to get the index from a name using the +// naming convention -. +func IndexFromName(n string) (int32, error) { + + // index := svc.Name[strings.LastIndex(svc.Name, "-") + 1 : len(svc.Name)] + delimIndex := strings.LastIndex(n, "-") + if delimIndex == -1 { + return -1, fmt.Errorf("couldn't get index from name %s", n) + } + + index, err := strconv.Atoi(n[delimIndex+1:]) + if err != nil { + return -1, fmt.Errorf("couldn't get index from name %s", n) + } + + return int32(index), nil +} + +// isPodUnschedulable iterates a Pod's Status.Conditions to find out +// if it has been deemed unschedulable +func IsPodUnschedulable(pod *corev1.Pod) bool { + for _, v := range pod.Status.Conditions { + if v.Reason == corev1.PodReasonUnschedulable { + return true + } + } + return false +} + // RefFromInt32 is a helper function that takes a int32 // and outputs a reference to that int. func RefFromInt32(i int32) *int32 { @@ -109,3 +178,35 @@ func ScaleStatefulSet(sts *appsv1.StatefulSet, amount int32, kubeClient kubernet err := PatchStatefulSet(sts, updatedSts, kubeClient) return err } + +// IsRackConditionTrue checks a rack's status for the presense of a condition type +// and checks if it is true. +func IsRackConditionTrue(rackStatus *cassandrav1alpha1.RackStatus, condType cassandrav1alpha1.RackConditionType) bool { + for _, cond := range rackStatus.Conditions { + if cond.Type == cassandrav1alpha1.RackConditionTypeMemberLeaving && cond.Status == cassandrav1alpha1.ConditionTrue { + return true + } + } + return false +} + +// StatefulSetStatusesStale checks if the StatefulSet Objects of a Cluster +// have been observed by the StatefulSet controller. +// If they haven't, their status might be stale, so it's better to wait +// and process them later. +func StatefulSetStatusesStale(c *cassandrav1alpha1.Cluster, statefulSetLister appslisters.StatefulSetLister) (bool, error) { + // Before proceeding, ensure all the Statefulset Statuses are valid + for _, r := range c.Spec.Datacenter.Racks { + if _, ok := c.Status.Racks[r.Name]; !ok { + continue + } + sts, err := statefulSetLister.StatefulSets(c.Namespace).Get(StatefulSetNameForRack(r, c)) + if err != nil { + return true, fmt.Errorf("error getting statefulset: %s", err.Error()) + } + if sts.Generation != sts.Status.ObservedGeneration { + return true, nil + } + } + return false, nil +}