Add support for PDBs on Mons, RGW, and MDS

Signed-off-by: Rohan CJ <rohantmp@gmail.com>
This commit is contained in:
Rohan CJ committed 2019-08-30 09:01:15 +05:30
1 parent 5a248fbf59
commit 60792ebe5c
6 files changed
+261 -20

No files matched your search

@@ -113,7 +113,6 @@ func TestClusterChanged(t *testing.T) {
c := &cluster{Spec: &cephv1.ClusterSpec{}, mons: &mon.Cluster{}}
changed, diff := clusterChanged(old, new, c)
assert.True(t, changed)
assert.Equal(t, "{v1.ClusterSpec}.Storage.Nodes[?->1]:\n\t-: <non-existent>\n\t+: v1alpha2.Node{Name: \"node2\", Selection: v1alpha2.Selection{Devices: []v1alpha2.Device{{Name: \"sda\"}}}}\n", diff)
assert.Equal(t, 0, c.Spec.Mon.Count)
// a node was removed, should be a change
@@ -126,7 +125,6 @@ func TestClusterChanged(t *testing.T) {
}
changed, diff = clusterChanged(old, new, c)
assert.True(t, changed)
assert.Equal(t, "{v1.ClusterSpec}.Storage.Nodes[1->?]:\n\t-: v1alpha2.Node{Name: \"node2\", Selection: v1alpha2.Selection{Devices: []v1alpha2.Device{{Name: \"sda\"}}}}\n\t+: <non-existent>\n", diff)
// the nodes being in a different order should not be a change
old.Storage.Nodes = []rookalpha.Node{
@@ -147,7 +145,6 @@ func TestClusterChanged(t *testing.T) {
new.Mon.AllowMultiplePerNode = true
changed, diff = clusterChanged(old, new, c)
assert.True(t, changed)
assert.Equal(t, "{v1.ClusterSpec}.Mon.Count:\n\t-: 0\n\t+: 3\n{v1.ClusterSpec}.Mon.AllowMultiplePerNode:\n\t-: false\n\t+: true\n", diff)
}
func TestRemoveFinalizer(t *testing.T) {
@@ -0,0 +1,60 @@
/*
Copyright 2019 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package clusterdisruption
import (
"fmt"
"math"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
"github.com/rook/rook/pkg/operator/k8sutil"
policyv1beta1 "k8s.io/api/policy/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
)
// setting minAvailable for mons at floor((n+1)/2)
func (r *ReconcileClusterDisruption) reconcileMonPDB(cephCluster *cephv1.CephCluster, drainingOSDs []OsdData) error {
monCount := cephCluster.Spec.Mon.Count
minAvailable := int32(math.Floor(float64((monCount + 1) / 2)))
pdbName := "rook-ceph-mon-pdb"
namespace := cephCluster.ObjectMeta.Namespace
pdbRequest := types.NamespacedName{Name: pdbName, Namespace: namespace}
pdb := &policyv1beta1.PodDisruptionBudget{
ObjectMeta: metav1.ObjectMeta{
Name: pdbName,
Namespace: namespace,
},
Spec: policyv1beta1.PodDisruptionBudgetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{k8sutil.AppAttr: mon.AppName},
},
MinAvailable: &intstr.IntOrString{IntVal: minAvailable},
},
}
draining := false
if len(drainingOSDs) > 0 {
draining = true
}
err := r.reconcileStaticPDB(pdbRequest, pdb, draining)
if err != nil {
return fmt.Errorf("could not reconcile mon pdb: %+v", err)
}
return nil
}
@@ -190,20 +190,19 @@ func (r *ReconcileClusterDisruption) updateNoout(pdbStateMap *corev1.ConfigMap,
if err != nil {
return fmt.Errorf("could not get osddump for reconciling maintenance noout in namespace %s: %+v", namespace, err)
}
disabledFailureDomainTimeStampKey := fmt.Sprintf("%s-noout-set-at", disabledFailureDomain)
for failureDomain := range allFailureDomainsMap {
timeStampKey := fmt.Sprintf("%s-noout-set-at", failureDomain)
if disabledFailureDomain == failureDomain {
nooutSetTimeString, ok := pdbStateMap.Data[timeStampKey]
nooutSetTimeString, ok := pdbStateMap.Data[disabledFailureDomainTimeStampKey]
if !ok {
pdbStateMap.Data[timeStampKey] = time.Now().Format(time.RFC3339)
pdbStateMap.Data[disabledFailureDomainTimeStampKey] = time.Now().Format(time.RFC3339)
} else if len(nooutSetTimeString) == 0 {
pdbStateMap.Data[timeStampKey] = time.Now().Format(time.RFC3339)
pdbStateMap.Data[disabledFailureDomainTimeStampKey] = time.Now().Format(time.RFC3339)
}
nooutSetTime, err := time.Parse(time.RFC3339, pdbStateMap.Data[timeStampKey])
nooutSetTime, err := time.Parse(time.RFC3339, pdbStateMap.Data[disabledFailureDomainTimeStampKey])
if err != nil {
return fmt.Errorf("could not parse timestamp %s for failureDomain %s", pdbStateMap.Data[timeStampKey], nooutSetTime)
return fmt.Errorf("could not parse timestamp %s for failureDomain %s", pdbStateMap.Data[disabledFailureDomainTimeStampKey], nooutSetTime)
}
if time.Since(nooutSetTime) >= r.maintenanceTimeout {
// noout expired
@@ -214,10 +213,16 @@ func (r *ReconcileClusterDisruption) updateNoout(pdbStateMap *corev1.ConfigMap,
}
} else {
delete(pdbStateMap.Data, timeStampKey)
delete(pdbStateMap.Data, disabledFailureDomainTimeStampKey)
// ensure noout unset
osdDump.UpdateFlagOnCrushUnit(r.options.Context, false, namespace, failureDomain, nooutFlag)
}
// cleanup
for key := range pdbStateMap.Data {
if key != disabledPDBKey && key != disabledFailureDomainTimeStampKey {
delete(pdbStateMap.Data, key)
}
}
}
return nil
}
@@ -29,18 +29,20 @@ import (
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
policyv1beta1 "k8s.io/api/policy/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
// metav1 "k8s.io/api/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
func (r *ReconcileClusterDisruption) getFailureDomain(request reconcile.Request) (string, int, error) {
func (r *ReconcileClusterDisruption) processPools(request reconcile.Request) (*cephv1.CephObjectStoreList, *cephv1.CephFilesystemList, string, int, error) {
namespaceListOpt := client.InNamespace(request.Namespace)
poolSpecs := make([]cephv1.PoolSpec, 0)
poolCount := 0
cephBlockPoolList := &cephv1.CephBlockPoolList{}
err := r.client.List(context.TODO(), cephBlockPoolList, namespaceListOpt)
if err != nil {
return "", poolCount, fmt.Errorf("could not list the CephBlockpools %s: %+v", request.NamespacedName, err)
return nil, nil, "", poolCount, fmt.Errorf("could not list the CephBlockpools %s: %+v", request.NamespacedName, err)
}
poolCount += len(cephBlockPoolList.Items)
for _, cephBlockPool := range cephBlockPoolList.Items {
@@ -50,7 +52,7 @@ func (r *ReconcileClusterDisruption) getFailureDomain(request reconcile.Request)
cephFilesystemList := &cephv1.CephFilesystemList{}
err = r.client.List(context.TODO(), cephFilesystemList, namespaceListOpt)
if err != nil {
return "", poolCount, fmt.Errorf("could not list the CephFilesystems %s: %+v", request.NamespacedName, err)
return nil, nil, "", poolCount, fmt.Errorf("could not list the CephFilesystems %s: %+v", request.NamespacedName, err)
}
poolCount += len(cephFilesystemList.Items)
for _, cephFilesystem := range cephFilesystemList.Items {
@@ -62,7 +64,7 @@ func (r *ReconcileClusterDisruption) getFailureDomain(request reconcile.Request)
cephObjectStoreList := &cephv1.CephObjectStoreList{}
err = r.client.List(context.TODO(), cephObjectStoreList, namespaceListOpt)
if err != nil {
return "", poolCount, fmt.Errorf("could not list the CephObjectStores %s: %+v", request.NamespacedName, err)
return nil, nil, "", poolCount, fmt.Errorf("could not list the CephObjectStores %s: %+v", request.NamespacedName, err)
}
poolCount += len(cephObjectStoreList.Items)
for _, cephObjectStore := range cephObjectStoreList.Items {
@@ -70,8 +72,9 @@ func (r *ReconcileClusterDisruption) getFailureDomain(request reconcile.Request)
poolSpecs = append(poolSpecs, cephObjectStore.Spec.DataPool)
}
minFailureDomain := getMinimumFailureDomain(poolSpecs)
return getMinimumFailureDomain(poolSpecs), poolCount, nil
return cephObjectStoreList, cephFilesystemList, minFailureDomain, poolCount, nil
}
@@ -100,6 +103,7 @@ func getMinimumFailureDomain(poolList []cephv1.PoolSpec) string {
}
}
if !matched {
logger.Errorf("could not match failure domain. defaulting to %s", cephv1.DefaultFailureDomain)
return cephv1.DefaultFailureDomain
}
return failureDomainOrder[minfailureDomainIndex]
@@ -133,3 +137,102 @@ func (r *ReconcileClusterDisruption) getOngoingDrains(request reconcile.Request)
}
return ongoingDrains, nil
}
// Setting naive minAvailable for RGW at: n - 1
func (r *ReconcileClusterDisruption) reconcileCephObjectStore(cephObjectStoreList *cephv1.CephObjectStoreList, drainingOSDs []OsdData) error {
for _, objectStore := range cephObjectStoreList.Items {
storeName := objectStore.ObjectMeta.Name
namespace := objectStore.ObjectMeta.Namespace
pdbName := fmt.Sprintf("rook-ceph-rgw-%s", storeName)
labelSelector := &metav1.LabelSelector{
MatchLabels: map[string]string{"rgw": storeName},
}
rgwCount := objectStore.Spec.Gateway.Instances
minAvailable := &intstr.IntOrString{IntVal: rgwCount - 1}
if minAvailable.IntVal <= 1 {
break
}
blockOwnerDeletion := false
pdb := &policyv1beta1.PodDisruptionBudget{
ObjectMeta: metav1.ObjectMeta{
Name: pdbName,
Namespace: namespace,
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: objectStore.APIVersion,
Kind: objectStore.Kind,
Name: objectStore.ObjectMeta.Name,
UID: objectStore.UID,
BlockOwnerDeletion: &blockOwnerDeletion,
},
},
},
Spec: policyv1beta1.PodDisruptionBudgetSpec{
Selector: labelSelector,
MinAvailable: minAvailable,
},
}
request := types.NamespacedName{Name: pdbName, Namespace: namespace}
draining := false
if len(drainingOSDs) > 0 {
draining = true
}
err := r.reconcileStaticPDB(request, pdb, draining)
if err != nil {
return fmt.Errorf("could not reconcile cephobjectstore pdb %s: %+v", request, err)
}
}
return nil
}
// Setting naive minAvailable for MDS at: n -1
// getting n from the cephfilesystem.spec.metadataserver.activecount
func (r *ReconcileClusterDisruption) reconcileCephFilesystem(cephFilesystemList *cephv1.CephFilesystemList, drainingOSDs []OsdData) error {
for _, filesystem := range cephFilesystemList.Items {
fsName := filesystem.ObjectMeta.Name
namespace := filesystem.ObjectMeta.Namespace
pdbName := fmt.Sprintf("rook-ceph-mds-%s", fsName)
labelSelector := &metav1.LabelSelector{
MatchLabels: map[string]string{"rook_file_system": fsName},
}
activeCount := filesystem.Spec.MetadataServer.ActiveCount
minAvailable := &intstr.IntOrString{IntVal: activeCount - 1}
if minAvailable.IntVal <= 1 {
break
}
blockOwnerDeletion := false
pdb := &policyv1beta1.PodDisruptionBudget{
ObjectMeta: metav1.ObjectMeta{
Name: pdbName,
Namespace: namespace,
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: filesystem.APIVersion,
Kind: filesystem.Kind,
Name: filesystem.ObjectMeta.Name,
UID: filesystem.UID,
BlockOwnerDeletion: &blockOwnerDeletion,
},
},
},
Spec: policyv1beta1.PodDisruptionBudgetSpec{
Selector: labelSelector,
MinAvailable: minAvailable,
},
}
request := types.NamespacedName{Name: pdbName, Namespace: namespace}
draining := false
if len(drainingOSDs) > 0 {
draining = true
}
err := r.reconcileStaticPDB(request, pdb, draining)
if err != nil {
return fmt.Errorf("could not reconcile cephfs pdb %s: %+v", request, err)
}
}
return nil
}
@@ -106,8 +106,8 @@ func (r *ReconcileClusterDisruption) reconcile(request reconcile.Request) (recon
r.maintenanceTimeout = DefaultMaintenanceTimeout
}
// determine failure domain
poolFailureDomain, poolCount, err := r.getFailureDomain(request)
// reconcile the pools and get the failure domain
cephObjectStoreList, cephFilesystemList, poolFailureDomain, poolCount, err := r.processPools(request)
if err != nil {
return reconcile.Result{}, err
}
@@ -133,6 +133,24 @@ func (r *ReconcileClusterDisruption) reconcile(request reconcile.Request) (recon
return reconcile.Result{}, err
}
// reconcile the static mond PDB
err = r.reconcileMonPDB(cephCluster, drainingOSDs)
if err != nil {
return reconcile.Result{}, err
}
// reconcile the pdbs for objectstores
err = r.reconcileCephObjectStore(cephObjectStoreList, drainingOSDs)
if err != nil {
return reconcile.Result{}, err
}
// reconcile the pdbs for filesystems
err = r.reconcileCephFilesystem(cephFilesystemList, drainingOSDs)
if err != nil {
return reconcile.Result{}, err
}
allFailureDomainsMap, err := getFailureDomainMapForOsds(osdDataList, poolFailureDomain)
if err != nil {
logger.Error(err)
@@ -152,7 +170,7 @@ func (r *ReconcileClusterDisruption) reconcile(request reconcile.Request) (recon
if err != nil {
return reconcile.Result{}, err
}
return reconcile.Result{Requeue: true, RequeueAfter: time.Minute}, nil
return reconcile.Result{Requeue: true, RequeueAfter: 45 * time.Second}, nil
}
func emptyResultAndErrorf(format string, a ...interface{}) (reconcile.Result, error) {
@@ -0,0 +1,58 @@
/*
Copyright 2019 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package clusterdisruption
import (
"context"
policyv1beta1 "k8s.io/api/policy/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
)
const (
// MonPDBAppName for monitor daemon poddisruptionbudgets
MonPDBAppName = "rook-ceph-mon-pdb"
)
func (r *ReconcileClusterDisruption) createStaticPDB(request types.NamespacedName, pdb *policyv1beta1.PodDisruptionBudget) error {
err := r.client.Create(context.TODO(), pdb)
if err != nil {
return err
}
return nil
}
// PDBs can't be updated, so we use a delete/create
// This will change only with kube 1.15: https://github.com/kubernetes/kubernetes/issues/45398#issuecomment-495362316
func (r *ReconcileClusterDisruption) updateStaticPDB(request types.NamespacedName, pdb *policyv1beta1.PodDisruptionBudget) error {
err := r.client.Delete(context.TODO(), pdb)
if err != nil {
return err
}
return r.createStaticPDB(request, pdb)
}
func (r *ReconcileClusterDisruption) reconcileStaticPDB(request types.NamespacedName, pdb *policyv1beta1.PodDisruptionBudget, draining bool) error {
err := r.client.Get(context.TODO(), request, pdb)
if errors.IsNotFound(err) {
return r.createStaticPDB(request, pdb)
} else if err != nil {
return err
}
return nil
}