cassandra: sidecar-side implementation of scale down
The sidecar now has a control loop, which watches its ClusterIP Service for the `cassandra.rook.io/decommissioned` label. If it finds it, then it decommissions and updates the label to value "true". Signed-off-by: Yannis Zarkadas <yanniszarkadas@gmail.com>
This commit is contained in:
1 parent
f5a41e9a7c
commit
a9178ccb59
3 files changed
+209
-15
No files matched your search
@@ -23,7 +23,10 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/flags"
|
||||
"github.com/spf13/cobra"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apiserver/pkg/server"
|
||||
kubeinformers "k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/informers/internalinterfaces"
|
||||
"os"
|
||||
)
|
||||
|
||||
@@ -58,11 +61,29 @@ func startSidecar(cmd *cobra.Command, args []string) error {
|
||||
rook.TerminateFatal(fmt.Errorf("cannot detect the pod namespace. Please provide it using the downward API in the manifest file"))
|
||||
}
|
||||
|
||||
// This func will make our informer only watch resources with the name of our member
|
||||
tweakListOptionsFunc := internalinterfaces.TweakListOptionsFunc(
|
||||
func(options *metav1.ListOptions) {
|
||||
options.FieldSelector = fmt.Sprintf("metadata.name=%s", podName)
|
||||
},
|
||||
)
|
||||
|
||||
// kubeInformerFactory watches resources with:
|
||||
// namespace: podNamespace
|
||||
// name: podName
|
||||
kubeInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(
|
||||
kubeClient,
|
||||
resyncPeriod,
|
||||
kubeinformers.WithNamespace(podNamespace),
|
||||
kubeinformers.WithTweakListOptions(tweakListOptionsFunc),
|
||||
)
|
||||
|
||||
mc, err := sidecar.New(
|
||||
podName,
|
||||
podNamespace,
|
||||
kubeClient,
|
||||
rookClient,
|
||||
kubeInformerFactory.Core().V1().Services(),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
@@ -72,6 +93,7 @@ func startSidecar(cmd *cobra.Command, args []string) error {
|
||||
|
||||
// Create a channel to receive OS signals
|
||||
stopCh := server.SetupSignalHandler()
|
||||
go kubeInformerFactory.Start(stopCh)
|
||||
|
||||
// Start the controller loop
|
||||
logger.Infof("Starting rook sidecar for Cassandra.")
|
||||
|
||||
@@ -19,18 +19,25 @@ package sidecar
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
cassandrav1alpha1 "github.com/rook/rook/pkg/apis/cassandra.rook.io/v1alpha1"
|
||||
rookClientset "github.com/rook/rook/pkg/client/clientset/versioned"
|
||||
"github.com/rook/rook/pkg/operator/cassandra/constants"
|
||||
"github.com/yanniszark/go-nodetool/nodetool"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
coreinformers "k8s.io/client-go/informers/core/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
corelisters "k8s.io/client-go/listers/core/v1"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
"k8s.io/client-go/util/workqueue"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"reflect"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -42,9 +49,11 @@ type MemberController struct {
|
||||
cluster, datacenter, rack string
|
||||
mode cassandrav1alpha1.ClusterMode
|
||||
|
||||
// Clients to handle Kubernetes Objects
|
||||
kubeClient kubernetes.Interface
|
||||
rookClient rookClientset.Interface
|
||||
// Clients and listers to handle Kubernetes Objects
|
||||
kubeClient kubernetes.Interface
|
||||
rookClient rookClientset.Interface
|
||||
serviceLister corelisters.ServiceLister
|
||||
serviceListerSynced cache.InformerSynced
|
||||
|
||||
nodetool *nodetool.Nodetool
|
||||
queue workqueue.RateLimitingInterface
|
||||
@@ -56,6 +65,7 @@ func New(
|
||||
name, namespace string,
|
||||
kubeClient kubernetes.Interface,
|
||||
rookClient rookClientset.Interface,
|
||||
serviceInformer coreinformers.ServiceInformer,
|
||||
) (*MemberController, error) {
|
||||
|
||||
logger := capnslog.NewPackageLogger("github.com/rook/rook", "sidecar")
|
||||
@@ -95,20 +105,51 @@ func New(
|
||||
}
|
||||
|
||||
m := &MemberController{
|
||||
name: name,
|
||||
namespace: namespace,
|
||||
ip: memberService.Spec.ClusterIP,
|
||||
cluster: pod.Labels[constants.ClusterNameLabel],
|
||||
datacenter: pod.Labels[constants.DatacenterNameLabel],
|
||||
rack: pod.Labels[constants.RackNameLabel],
|
||||
mode: cluster.Spec.Mode,
|
||||
kubeClient: kubeClient,
|
||||
rookClient: rookClient,
|
||||
nodetool: nodetool,
|
||||
queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
|
||||
logger: logger,
|
||||
name: name,
|
||||
namespace: namespace,
|
||||
ip: memberService.Spec.ClusterIP,
|
||||
cluster: pod.Labels[constants.ClusterNameLabel],
|
||||
datacenter: pod.Labels[constants.DatacenterNameLabel],
|
||||
rack: pod.Labels[constants.RackNameLabel],
|
||||
mode: cluster.Spec.Mode,
|
||||
kubeClient: kubeClient,
|
||||
rookClient: rookClient,
|
||||
serviceLister: serviceInformer.Lister(),
|
||||
serviceListerSynced: serviceInformer.Informer().HasSynced,
|
||||
nodetool: nodetool,
|
||||
queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
|
||||
logger: logger,
|
||||
}
|
||||
|
||||
serviceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) {
|
||||
svc := obj.(*corev1.Service)
|
||||
if svc.Name != m.name {
|
||||
logger.Errorf("Lister returned unexpected service %s", svc.Name)
|
||||
return
|
||||
}
|
||||
m.enqueueMemberService(svc)
|
||||
},
|
||||
UpdateFunc: func(old, new interface{}) {
|
||||
oldService := old.(*corev1.Service)
|
||||
newService := new.(*corev1.Service)
|
||||
if oldService.ResourceVersion == newService.ResourceVersion {
|
||||
return
|
||||
}
|
||||
if reflect.DeepEqual(oldService.Labels, newService.Labels) {
|
||||
return
|
||||
}
|
||||
logger.Infof("New event for my MemberService %s", newService.Name)
|
||||
m.enqueueMemberService(newService)
|
||||
},
|
||||
DeleteFunc: func(obj interface{}) {
|
||||
svc := obj.(*corev1.Service)
|
||||
if svc.Name == m.name {
|
||||
logger.Errorf("Unexpected deletion of MemberService %s", svc.Name)
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
@@ -117,16 +158,84 @@ func (m *MemberController) Run(threadiness int, stopCh <-chan struct{}) error {
|
||||
|
||||
defer runtime.HandleCrash()
|
||||
|
||||
if ok := cache.WaitForCacheSync(stopCh, m.serviceListerSynced); !ok {
|
||||
return fmt.Errorf("failed to wait for caches to sync")
|
||||
}
|
||||
|
||||
if err := m.onStartup(); err != nil {
|
||||
return fmt.Errorf("error on startup: %s", err.Error())
|
||||
}
|
||||
|
||||
m.logger.Infof("Main event loop")
|
||||
go wait.Until(m.runWorker, time.Second, stopCh)
|
||||
|
||||
<-stopCh
|
||||
m.logger.Info("Shutting down sidecar.")
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func (m *MemberController) runWorker() {
|
||||
for m.processNextWorkItem() {
|
||||
}
|
||||
}
|
||||
|
||||
func (m *MemberController) processNextWorkItem() bool {
|
||||
obj, shutdown := m.queue.Get()
|
||||
|
||||
if shutdown {
|
||||
return false
|
||||
}
|
||||
|
||||
err := func(obj interface{}) error {
|
||||
defer m.queue.Done(obj)
|
||||
key, ok := obj.(string)
|
||||
if !ok {
|
||||
m.queue.Forget(obj)
|
||||
runtime.HandleError(fmt.Errorf("expected string in queue but got %#v", obj))
|
||||
}
|
||||
if err := m.syncHandler(key); err != nil {
|
||||
m.queue.AddRateLimited(key)
|
||||
return fmt.Errorf("error syncing '%s', requeueing: %s", key, err.Error())
|
||||
}
|
||||
m.queue.Forget(obj)
|
||||
m.logger.Infof("Successfully synced '%s'", key)
|
||||
return nil
|
||||
}(obj)
|
||||
|
||||
if err != nil {
|
||||
runtime.HandleError(err)
|
||||
return true
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (m *MemberController) syncHandler(key string) error {
|
||||
// Convert the namespace/name string into a distinct namespace and name.
|
||||
namespace, name, err := cache.SplitMetaNamespaceKey(key)
|
||||
if err != nil {
|
||||
runtime.HandleError(fmt.Errorf("invalid resource key: %s", key))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get the Cluster resource with this namespace/name
|
||||
svc, err := m.serviceLister.Services(namespace).Get(name)
|
||||
if err != nil {
|
||||
// The Cluster resource may no longer exist, in which case we stop processing.
|
||||
if apierrors.IsNotFound(err) {
|
||||
runtime.HandleError(fmt.Errorf("member service '%s' in work queue no longer exists", key))
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("unexpected error while getting member service object: %s", err)
|
||||
}
|
||||
|
||||
m.logger.Infof("handling member service object: %+v", spew.Sdump(svc))
|
||||
err = m.Sync(svc)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// onStartup is executed before the MemberController starts
|
||||
// its sync loop.
|
||||
func (m *MemberController) onStartup() error {
|
||||
@@ -157,3 +266,13 @@ func (m *MemberController) onStartup() error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MemberController) enqueueMemberService(obj metav1.Object) {
|
||||
var key string
|
||||
var err error
|
||||
if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil {
|
||||
runtime.HandleError(err)
|
||||
return
|
||||
}
|
||||
m.queue.AddRateLimited(key)
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package sidecar
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/rook/rook/pkg/operator/cassandra/constants"
|
||||
"github.com/rook/rook/pkg/operator/cassandra/controller/util"
|
||||
"github.com/yanniszark/go-nodetool/nodetool"
|
||||
"k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
func (m *MemberController) Sync(memberService *v1.Service) error {
|
||||
|
||||
// Check if member must decommission
|
||||
if decommission, ok := memberService.Labels[constants.DecommissionLabel]; ok {
|
||||
// Check if member has already decommissioned
|
||||
if decommission == constants.LabelValueTrue {
|
||||
return nil
|
||||
}
|
||||
// Else, decommission member
|
||||
if err := m.nodetool.Decommission(); err != nil {
|
||||
m.logger.Errorf("Error during decommission: %s", err.Error())
|
||||
}
|
||||
// Confirm memberService has been decommissioned
|
||||
if opMode, err := m.nodetool.OperationMode(); err != nil || opMode != nodetool.NodeOperationModeDecommissioned {
|
||||
return fmt.Errorf("error during decommission, operation mode: %s, error: %v", opMode, err)
|
||||
}
|
||||
// Update Label
|
||||
old := memberService.DeepCopy()
|
||||
memberService.Labels[constants.DecommissionLabel] = constants.LabelValueTrue
|
||||
if err := util.PatchService(old, memberService, m.kubeClient); err != nil {
|
||||
return fmt.Errorf("error patching MemberService, %s", err.Error())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user