cassandra: sidecar-side implementation of scale down

The sidecar now has a control loop, which watches its ClusterIP
Service for the `cassandra.rook.io/decommissioned` label.
If it finds it, then it decommissions and updates the label to
value "true".

Signed-off-by: Yannis Zarkadas <yanniszarkadas@gmail.com>
This commit is contained in:
Yannis Zarkadas committed 2018-12-07 13:46:46 +02:00
1 parent f5a41e9a7c
commit a9178ccb59
3 files changed
+209 -15

No files matched your search

+22
View File
@@ -23,7 +23,10 @@ import (
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/flags"
"github.com/spf13/cobra"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/server"
kubeinformers "k8s.io/client-go/informers"
"k8s.io/client-go/informers/internalinterfaces"
"os"
)
@@ -58,11 +61,29 @@ func startSidecar(cmd *cobra.Command, args []string) error {
rook.TerminateFatal(fmt.Errorf("cannot detect the pod namespace. Please provide it using the downward API in the manifest file"))
}
// This func will make our informer only watch resources with the name of our member
tweakListOptionsFunc := internalinterfaces.TweakListOptionsFunc(
func(options *metav1.ListOptions) {
options.FieldSelector = fmt.Sprintf("metadata.name=%s", podName)
},
)
// kubeInformerFactory watches resources with:
// namespace: podNamespace
// name: podName
kubeInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(
kubeClient,
resyncPeriod,
kubeinformers.WithNamespace(podNamespace),
kubeinformers.WithTweakListOptions(tweakListOptionsFunc),
)
mc, err := sidecar.New(
podName,
podNamespace,
kubeClient,
rookClient,
kubeInformerFactory.Core().V1().Services(),
)
if err != nil {
@@ -72,6 +93,7 @@ func startSidecar(cmd *cobra.Command, args []string) error {
// Create a channel to receive OS signals
stopCh := server.SetupSignalHandler()
go kubeInformerFactory.Start(stopCh)
// Start the controller loop
logger.Infof("Starting rook sidecar for Cassandra.")
+134 -15
View File
@@ -19,18 +19,25 @@ package sidecar
import (
"fmt"
"github.com/coreos/pkg/capnslog"
"github.com/davecgh/go-spew/spew"
cassandrav1alpha1 "github.com/rook/rook/pkg/apis/cassandra.rook.io/v1alpha1"
rookClientset "github.com/rook/rook/pkg/client/clientset/versioned"
"github.com/rook/rook/pkg/operator/cassandra/constants"
"github.com/yanniszark/go-nodetool/nodetool"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
coreinformers "k8s.io/client-go/informers/core/v1"
"k8s.io/client-go/kubernetes"
corelisters "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"
"net/url"
"os"
"os/exec"
"reflect"
"time"
)
@@ -42,9 +49,11 @@ type MemberController struct {
cluster, datacenter, rack string
mode cassandrav1alpha1.ClusterMode
// Clients to handle Kubernetes Objects
kubeClient kubernetes.Interface
rookClient rookClientset.Interface
// Clients and listers to handle Kubernetes Objects
kubeClient kubernetes.Interface
rookClient rookClientset.Interface
serviceLister corelisters.ServiceLister
serviceListerSynced cache.InformerSynced
nodetool *nodetool.Nodetool
queue workqueue.RateLimitingInterface
@@ -56,6 +65,7 @@ func New(
name, namespace string,
kubeClient kubernetes.Interface,
rookClient rookClientset.Interface,
serviceInformer coreinformers.ServiceInformer,
) (*MemberController, error) {
logger := capnslog.NewPackageLogger("github.com/rook/rook", "sidecar")
@@ -95,20 +105,51 @@ func New(
}
m := &MemberController{
name: name,
namespace: namespace,
ip: memberService.Spec.ClusterIP,
cluster: pod.Labels[constants.ClusterNameLabel],
datacenter: pod.Labels[constants.DatacenterNameLabel],
rack: pod.Labels[constants.RackNameLabel],
mode: cluster.Spec.Mode,
kubeClient: kubeClient,
rookClient: rookClient,
nodetool: nodetool,
queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
logger: logger,
name: name,
namespace: namespace,
ip: memberService.Spec.ClusterIP,
cluster: pod.Labels[constants.ClusterNameLabel],
datacenter: pod.Labels[constants.DatacenterNameLabel],
rack: pod.Labels[constants.RackNameLabel],
mode: cluster.Spec.Mode,
kubeClient: kubeClient,
rookClient: rookClient,
serviceLister: serviceInformer.Lister(),
serviceListerSynced: serviceInformer.Informer().HasSynced,
nodetool: nodetool,
queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
logger: logger,
}
serviceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
svc := obj.(*corev1.Service)
if svc.Name != m.name {
logger.Errorf("Lister returned unexpected service %s", svc.Name)
return
}
m.enqueueMemberService(svc)
},
UpdateFunc: func(old, new interface{}) {
oldService := old.(*corev1.Service)
newService := new.(*corev1.Service)
if oldService.ResourceVersion == newService.ResourceVersion {
return
}
if reflect.DeepEqual(oldService.Labels, newService.Labels) {
return
}
logger.Infof("New event for my MemberService %s", newService.Name)
m.enqueueMemberService(newService)
},
DeleteFunc: func(obj interface{}) {
svc := obj.(*corev1.Service)
if svc.Name == m.name {
logger.Errorf("Unexpected deletion of MemberService %s", svc.Name)
}
},
})
return m, nil
}
@@ -117,16 +158,84 @@ func (m *MemberController) Run(threadiness int, stopCh <-chan struct{}) error {
defer runtime.HandleCrash()
if ok := cache.WaitForCacheSync(stopCh, m.serviceListerSynced); !ok {
return fmt.Errorf("failed to wait for caches to sync")
}
if err := m.onStartup(); err != nil {
return fmt.Errorf("error on startup: %s", err.Error())
}
m.logger.Infof("Main event loop")
go wait.Until(m.runWorker, time.Second, stopCh)
<-stopCh
m.logger.Info("Shutting down sidecar.")
return nil
}
func (m *MemberController) runWorker() {
for m.processNextWorkItem() {
}
}
func (m *MemberController) processNextWorkItem() bool {
obj, shutdown := m.queue.Get()
if shutdown {
return false
}
err := func(obj interface{}) error {
defer m.queue.Done(obj)
key, ok := obj.(string)
if !ok {
m.queue.Forget(obj)
runtime.HandleError(fmt.Errorf("expected string in queue but got %#v", obj))
}
if err := m.syncHandler(key); err != nil {
m.queue.AddRateLimited(key)
return fmt.Errorf("error syncing '%s', requeueing: %s", key, err.Error())
}
m.queue.Forget(obj)
m.logger.Infof("Successfully synced '%s'", key)
return nil
}(obj)
if err != nil {
runtime.HandleError(err)
return true
}
return true
}
func (m *MemberController) syncHandler(key string) error {
// Convert the namespace/name string into a distinct namespace and name.
namespace, name, err := cache.SplitMetaNamespaceKey(key)
if err != nil {
runtime.HandleError(fmt.Errorf("invalid resource key: %s", key))
return nil
}
// Get the Cluster resource with this namespace/name
svc, err := m.serviceLister.Services(namespace).Get(name)
if err != nil {
// The Cluster resource may no longer exist, in which case we stop processing.
if apierrors.IsNotFound(err) {
runtime.HandleError(fmt.Errorf("member service '%s' in work queue no longer exists", key))
return nil
}
return fmt.Errorf("unexpected error while getting member service object: %s", err)
}
m.logger.Infof("handling member service object: %+v", spew.Sdump(svc))
err = m.Sync(svc)
return err
}
// onStartup is executed before the MemberController starts
// its sync loop.
func (m *MemberController) onStartup() error {
@@ -157,3 +266,13 @@ func (m *MemberController) onStartup() error {
return nil
}
func (m *MemberController) enqueueMemberService(obj metav1.Object) {
var key string
var err error
if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil {
runtime.HandleError(err)
return
}
m.queue.AddRateLimited(key)
}
+53
View File
@@ -0,0 +1,53 @@
/*
Copyright 2018 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sidecar
import (
"fmt"
"github.com/rook/rook/pkg/operator/cassandra/constants"
"github.com/rook/rook/pkg/operator/cassandra/controller/util"
"github.com/yanniszark/go-nodetool/nodetool"
"k8s.io/api/core/v1"
)
func (m *MemberController) Sync(memberService *v1.Service) error {
// Check if member must decommission
if decommission, ok := memberService.Labels[constants.DecommissionLabel]; ok {
// Check if member has already decommissioned
if decommission == constants.LabelValueTrue {
return nil
}
// Else, decommission member
if err := m.nodetool.Decommission(); err != nil {
m.logger.Errorf("Error during decommission: %s", err.Error())
}
// Confirm memberService has been decommissioned
if opMode, err := m.nodetool.OperationMode(); err != nil || opMode != nodetool.NodeOperationModeDecommissioned {
return fmt.Errorf("error during decommission, operation mode: %s, error: %v", opMode, err)
}
// Update Label
old := memberService.DeepCopy()
memberService.Labels[constants.DecommissionLabel] = constants.LabelValueTrue
if err := util.PatchService(old, memberService, m.kubeClient); err != nil {
return fmt.Errorf("error patching MemberService, %s", err.Error())
}
}
return nil
}