From a9178ccb5966fd9f74864ebca42ca910023aff1e Mon Sep 17 00:00:00 2001 From: Yannis Zarkadas Date: Tue, 4 Dec 2018 02:15:02 +0200 Subject: [PATCH] cassandra: sidecar-side implementation of scale down The sidecar now has a control loop, which watches its ClusterIP Service for the `cassandra.rook.io/decommissioned` label. If it finds it, then it decommissions and updates the label to value "true". Signed-off-by: Yannis Zarkadas --- cmd/rook/cassandra/sidecar.go | 22 ++++ pkg/operator/cassandra/sidecar/sidecar.go | 149 +++++++++++++++++++--- pkg/operator/cassandra/sidecar/sync.go | 53 ++++++++ 3 files changed, 209 insertions(+), 15 deletions(-) create mode 100644 pkg/operator/cassandra/sidecar/sync.go diff --git a/cmd/rook/cassandra/sidecar.go b/cmd/rook/cassandra/sidecar.go index 697262d95..bb43afb53 100644 --- a/cmd/rook/cassandra/sidecar.go +++ b/cmd/rook/cassandra/sidecar.go @@ -23,7 +23,10 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/flags" "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/server" + kubeinformers "k8s.io/client-go/informers" + "k8s.io/client-go/informers/internalinterfaces" "os" ) @@ -58,11 +61,29 @@ func startSidecar(cmd *cobra.Command, args []string) error { rook.TerminateFatal(fmt.Errorf("cannot detect the pod namespace. Please provide it using the downward API in the manifest file")) } + // This func will make our informer only watch resources with the name of our member + tweakListOptionsFunc := internalinterfaces.TweakListOptionsFunc( + func(options *metav1.ListOptions) { + options.FieldSelector = fmt.Sprintf("metadata.name=%s", podName) + }, + ) + + // kubeInformerFactory watches resources with: + // namespace: podNamespace + // name: podName + kubeInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions( + kubeClient, + resyncPeriod, + kubeinformers.WithNamespace(podNamespace), + kubeinformers.WithTweakListOptions(tweakListOptionsFunc), + ) + mc, err := sidecar.New( podName, podNamespace, kubeClient, rookClient, + kubeInformerFactory.Core().V1().Services(), ) if err != nil { @@ -72,6 +93,7 @@ func startSidecar(cmd *cobra.Command, args []string) error { // Create a channel to receive OS signals stopCh := server.SetupSignalHandler() + go kubeInformerFactory.Start(stopCh) // Start the controller loop logger.Infof("Starting rook sidecar for Cassandra.") diff --git a/pkg/operator/cassandra/sidecar/sidecar.go b/pkg/operator/cassandra/sidecar/sidecar.go index f5102d4c7..609f84d3a 100644 --- a/pkg/operator/cassandra/sidecar/sidecar.go +++ b/pkg/operator/cassandra/sidecar/sidecar.go @@ -19,18 +19,25 @@ package sidecar import ( "fmt" "github.com/coreos/pkg/capnslog" + "github.com/davecgh/go-spew/spew" cassandrav1alpha1 "github.com/rook/rook/pkg/apis/cassandra.rook.io/v1alpha1" rookClientset "github.com/rook/rook/pkg/client/clientset/versioned" "github.com/rook/rook/pkg/operator/cassandra/constants" "github.com/yanniszark/go-nodetool/nodetool" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + coreinformers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" "net/url" "os" "os/exec" + "reflect" "time" ) @@ -42,9 +49,11 @@ type MemberController struct { cluster, datacenter, rack string mode cassandrav1alpha1.ClusterMode - // Clients to handle Kubernetes Objects - kubeClient kubernetes.Interface - rookClient rookClientset.Interface + // Clients and listers to handle Kubernetes Objects + kubeClient kubernetes.Interface + rookClient rookClientset.Interface + serviceLister corelisters.ServiceLister + serviceListerSynced cache.InformerSynced nodetool *nodetool.Nodetool queue workqueue.RateLimitingInterface @@ -56,6 +65,7 @@ func New( name, namespace string, kubeClient kubernetes.Interface, rookClient rookClientset.Interface, + serviceInformer coreinformers.ServiceInformer, ) (*MemberController, error) { logger := capnslog.NewPackageLogger("github.com/rook/rook", "sidecar") @@ -95,20 +105,51 @@ func New( } m := &MemberController{ - name: name, - namespace: namespace, - ip: memberService.Spec.ClusterIP, - cluster: pod.Labels[constants.ClusterNameLabel], - datacenter: pod.Labels[constants.DatacenterNameLabel], - rack: pod.Labels[constants.RackNameLabel], - mode: cluster.Spec.Mode, - kubeClient: kubeClient, - rookClient: rookClient, - nodetool: nodetool, - queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), - logger: logger, + name: name, + namespace: namespace, + ip: memberService.Spec.ClusterIP, + cluster: pod.Labels[constants.ClusterNameLabel], + datacenter: pod.Labels[constants.DatacenterNameLabel], + rack: pod.Labels[constants.RackNameLabel], + mode: cluster.Spec.Mode, + kubeClient: kubeClient, + rookClient: rookClient, + serviceLister: serviceInformer.Lister(), + serviceListerSynced: serviceInformer.Informer().HasSynced, + nodetool: nodetool, + queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), + logger: logger, } + serviceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + svc := obj.(*corev1.Service) + if svc.Name != m.name { + logger.Errorf("Lister returned unexpected service %s", svc.Name) + return + } + m.enqueueMemberService(svc) + }, + UpdateFunc: func(old, new interface{}) { + oldService := old.(*corev1.Service) + newService := new.(*corev1.Service) + if oldService.ResourceVersion == newService.ResourceVersion { + return + } + if reflect.DeepEqual(oldService.Labels, newService.Labels) { + return + } + logger.Infof("New event for my MemberService %s", newService.Name) + m.enqueueMemberService(newService) + }, + DeleteFunc: func(obj interface{}) { + svc := obj.(*corev1.Service) + if svc.Name == m.name { + logger.Errorf("Unexpected deletion of MemberService %s", svc.Name) + } + }, + }) + return m, nil } @@ -117,16 +158,84 @@ func (m *MemberController) Run(threadiness int, stopCh <-chan struct{}) error { defer runtime.HandleCrash() + if ok := cache.WaitForCacheSync(stopCh, m.serviceListerSynced); !ok { + return fmt.Errorf("failed to wait for caches to sync") + } + if err := m.onStartup(); err != nil { return fmt.Errorf("error on startup: %s", err.Error()) } + m.logger.Infof("Main event loop") + go wait.Until(m.runWorker, time.Second, stopCh) + <-stopCh m.logger.Info("Shutting down sidecar.") return nil } +func (m *MemberController) runWorker() { + for m.processNextWorkItem() { + } +} + +func (m *MemberController) processNextWorkItem() bool { + obj, shutdown := m.queue.Get() + + if shutdown { + return false + } + + err := func(obj interface{}) error { + defer m.queue.Done(obj) + key, ok := obj.(string) + if !ok { + m.queue.Forget(obj) + runtime.HandleError(fmt.Errorf("expected string in queue but got %#v", obj)) + } + if err := m.syncHandler(key); err != nil { + m.queue.AddRateLimited(key) + return fmt.Errorf("error syncing '%s', requeueing: %s", key, err.Error()) + } + m.queue.Forget(obj) + m.logger.Infof("Successfully synced '%s'", key) + return nil + }(obj) + + if err != nil { + runtime.HandleError(err) + return true + } + + return true +} + +func (m *MemberController) syncHandler(key string) error { + // Convert the namespace/name string into a distinct namespace and name. + namespace, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + runtime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + // Get the Cluster resource with this namespace/name + svc, err := m.serviceLister.Services(namespace).Get(name) + if err != nil { + // The Cluster resource may no longer exist, in which case we stop processing. + if apierrors.IsNotFound(err) { + runtime.HandleError(fmt.Errorf("member service '%s' in work queue no longer exists", key)) + return nil + } + return fmt.Errorf("unexpected error while getting member service object: %s", err) + } + + m.logger.Infof("handling member service object: %+v", spew.Sdump(svc)) + err = m.Sync(svc) + + return err +} + // onStartup is executed before the MemberController starts // its sync loop. func (m *MemberController) onStartup() error { @@ -157,3 +266,13 @@ func (m *MemberController) onStartup() error { return nil } + +func (m *MemberController) enqueueMemberService(obj metav1.Object) { + var key string + var err error + if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { + runtime.HandleError(err) + return + } + m.queue.AddRateLimited(key) +} diff --git a/pkg/operator/cassandra/sidecar/sync.go b/pkg/operator/cassandra/sidecar/sync.go new file mode 100644 index 000000000..4e5b91bfe --- /dev/null +++ b/pkg/operator/cassandra/sidecar/sync.go @@ -0,0 +1,53 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +    http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sidecar + +import ( + "fmt" + "github.com/rook/rook/pkg/operator/cassandra/constants" + "github.com/rook/rook/pkg/operator/cassandra/controller/util" + "github.com/yanniszark/go-nodetool/nodetool" + "k8s.io/api/core/v1" +) + +func (m *MemberController) Sync(memberService *v1.Service) error { + + // Check if member must decommission + if decommission, ok := memberService.Labels[constants.DecommissionLabel]; ok { + // Check if member has already decommissioned + if decommission == constants.LabelValueTrue { + return nil + } + // Else, decommission member + if err := m.nodetool.Decommission(); err != nil { + m.logger.Errorf("Error during decommission: %s", err.Error()) + } + // Confirm memberService has been decommissioned + if opMode, err := m.nodetool.OperationMode(); err != nil || opMode != nodetool.NodeOperationModeDecommissioned { + return fmt.Errorf("error during decommission, operation mode: %s, error: %v", opMode, err) + } + // Update Label + old := memberService.DeepCopy() + memberService.Labels[constants.DecommissionLabel] = constants.LabelValueTrue + if err := util.PatchService(old, memberService, m.kubeClient); err != nil { + return fmt.Errorf("error patching MemberService, %s", err.Error()) + } + + } + + return nil +}