Add support for PDBs on Mons, RGW, and MDS
Signed-off-by: Rohan CJ <rohantmp@gmail.com>
This commit is contained in:
1 parent
5a248fbf59
commit
60792ebe5c
6 files changed
+261
-20
No files matched your search
@@ -113,7 +113,6 @@ func TestClusterChanged(t *testing.T) {
|
||||
c := &cluster{Spec: &cephv1.ClusterSpec{}, mons: &mon.Cluster{}}
|
||||
changed, diff := clusterChanged(old, new, c)
|
||||
assert.True(t, changed)
|
||||
assert.Equal(t, "{v1.ClusterSpec}.Storage.Nodes[?->1]:\n\t-: <non-existent>\n\t+: v1alpha2.Node{Name: \"node2\", Selection: v1alpha2.Selection{Devices: []v1alpha2.Device{{Name: \"sda\"}}}}\n", diff)
|
||||
assert.Equal(t, 0, c.Spec.Mon.Count)
|
||||
|
||||
// a node was removed, should be a change
|
||||
@@ -126,7 +125,6 @@ func TestClusterChanged(t *testing.T) {
|
||||
}
|
||||
changed, diff = clusterChanged(old, new, c)
|
||||
assert.True(t, changed)
|
||||
assert.Equal(t, "{v1.ClusterSpec}.Storage.Nodes[1->?]:\n\t-: v1alpha2.Node{Name: \"node2\", Selection: v1alpha2.Selection{Devices: []v1alpha2.Device{{Name: \"sda\"}}}}\n\t+: <non-existent>\n", diff)
|
||||
|
||||
// the nodes being in a different order should not be a change
|
||||
old.Storage.Nodes = []rookalpha.Node{
|
||||
@@ -147,7 +145,6 @@ func TestClusterChanged(t *testing.T) {
|
||||
new.Mon.AllowMultiplePerNode = true
|
||||
changed, diff = clusterChanged(old, new, c)
|
||||
assert.True(t, changed)
|
||||
assert.Equal(t, "{v1.ClusterSpec}.Mon.Count:\n\t-: 0\n\t+: 3\n{v1.ClusterSpec}.Mon.AllowMultiplePerNode:\n\t-: false\n\t+: true\n", diff)
|
||||
}
|
||||
|
||||
func TestRemoveFinalizer(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
Copyright 2019 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package clusterdisruption
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
policyv1beta1 "k8s.io/api/policy/v1beta1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
)
|
||||
|
||||
// setting minAvailable for mons at floor((n+1)/2)
|
||||
func (r *ReconcileClusterDisruption) reconcileMonPDB(cephCluster *cephv1.CephCluster, drainingOSDs []OsdData) error {
|
||||
monCount := cephCluster.Spec.Mon.Count
|
||||
minAvailable := int32(math.Floor(float64((monCount + 1) / 2)))
|
||||
pdbName := "rook-ceph-mon-pdb"
|
||||
namespace := cephCluster.ObjectMeta.Namespace
|
||||
pdbRequest := types.NamespacedName{Name: pdbName, Namespace: namespace}
|
||||
pdb := &policyv1beta1.PodDisruptionBudget{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: pdbName,
|
||||
Namespace: namespace,
|
||||
},
|
||||
Spec: policyv1beta1.PodDisruptionBudgetSpec{
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{k8sutil.AppAttr: mon.AppName},
|
||||
},
|
||||
MinAvailable: &intstr.IntOrString{IntVal: minAvailable},
|
||||
},
|
||||
}
|
||||
draining := false
|
||||
if len(drainingOSDs) > 0 {
|
||||
draining = true
|
||||
}
|
||||
err := r.reconcileStaticPDB(pdbRequest, pdb, draining)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not reconcile mon pdb: %+v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -190,20 +190,19 @@ func (r *ReconcileClusterDisruption) updateNoout(pdbStateMap *corev1.ConfigMap,
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get osddump for reconciling maintenance noout in namespace %s: %+v", namespace, err)
|
||||
}
|
||||
|
||||
disabledFailureDomainTimeStampKey := fmt.Sprintf("%s-noout-set-at", disabledFailureDomain)
|
||||
for failureDomain := range allFailureDomainsMap {
|
||||
timeStampKey := fmt.Sprintf("%s-noout-set-at", failureDomain)
|
||||
if disabledFailureDomain == failureDomain {
|
||||
nooutSetTimeString, ok := pdbStateMap.Data[timeStampKey]
|
||||
nooutSetTimeString, ok := pdbStateMap.Data[disabledFailureDomainTimeStampKey]
|
||||
if !ok {
|
||||
pdbStateMap.Data[timeStampKey] = time.Now().Format(time.RFC3339)
|
||||
pdbStateMap.Data[disabledFailureDomainTimeStampKey] = time.Now().Format(time.RFC3339)
|
||||
} else if len(nooutSetTimeString) == 0 {
|
||||
pdbStateMap.Data[timeStampKey] = time.Now().Format(time.RFC3339)
|
||||
pdbStateMap.Data[disabledFailureDomainTimeStampKey] = time.Now().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
nooutSetTime, err := time.Parse(time.RFC3339, pdbStateMap.Data[timeStampKey])
|
||||
nooutSetTime, err := time.Parse(time.RFC3339, pdbStateMap.Data[disabledFailureDomainTimeStampKey])
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not parse timestamp %s for failureDomain %s", pdbStateMap.Data[timeStampKey], nooutSetTime)
|
||||
return fmt.Errorf("could not parse timestamp %s for failureDomain %s", pdbStateMap.Data[disabledFailureDomainTimeStampKey], nooutSetTime)
|
||||
}
|
||||
if time.Since(nooutSetTime) >= r.maintenanceTimeout {
|
||||
// noout expired
|
||||
@@ -214,10 +213,16 @@ func (r *ReconcileClusterDisruption) updateNoout(pdbStateMap *corev1.ConfigMap,
|
||||
}
|
||||
|
||||
} else {
|
||||
delete(pdbStateMap.Data, timeStampKey)
|
||||
delete(pdbStateMap.Data, disabledFailureDomainTimeStampKey)
|
||||
// ensure noout unset
|
||||
osdDump.UpdateFlagOnCrushUnit(r.options.Context, false, namespace, failureDomain, nooutFlag)
|
||||
}
|
||||
// cleanup
|
||||
for key := range pdbStateMap.Data {
|
||||
if key != disabledPDBKey && key != disabledFailureDomainTimeStampKey {
|
||||
delete(pdbStateMap.Data, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+109
-6
@@ -29,18 +29,20 @@ import (
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
policyv1beta1 "k8s.io/api/policy/v1beta1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
// metav1 "k8s.io/api/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
)
|
||||
|
||||
func (r *ReconcileClusterDisruption) getFailureDomain(request reconcile.Request) (string, int, error) {
|
||||
func (r *ReconcileClusterDisruption) processPools(request reconcile.Request) (*cephv1.CephObjectStoreList, *cephv1.CephFilesystemList, string, int, error) {
|
||||
namespaceListOpt := client.InNamespace(request.Namespace)
|
||||
poolSpecs := make([]cephv1.PoolSpec, 0)
|
||||
poolCount := 0
|
||||
cephBlockPoolList := &cephv1.CephBlockPoolList{}
|
||||
err := r.client.List(context.TODO(), cephBlockPoolList, namespaceListOpt)
|
||||
if err != nil {
|
||||
return "", poolCount, fmt.Errorf("could not list the CephBlockpools %s: %+v", request.NamespacedName, err)
|
||||
return nil, nil, "", poolCount, fmt.Errorf("could not list the CephBlockpools %s: %+v", request.NamespacedName, err)
|
||||
}
|
||||
poolCount += len(cephBlockPoolList.Items)
|
||||
for _, cephBlockPool := range cephBlockPoolList.Items {
|
||||
@@ -50,7 +52,7 @@ func (r *ReconcileClusterDisruption) getFailureDomain(request reconcile.Request)
|
||||
cephFilesystemList := &cephv1.CephFilesystemList{}
|
||||
err = r.client.List(context.TODO(), cephFilesystemList, namespaceListOpt)
|
||||
if err != nil {
|
||||
return "", poolCount, fmt.Errorf("could not list the CephFilesystems %s: %+v", request.NamespacedName, err)
|
||||
return nil, nil, "", poolCount, fmt.Errorf("could not list the CephFilesystems %s: %+v", request.NamespacedName, err)
|
||||
}
|
||||
poolCount += len(cephFilesystemList.Items)
|
||||
for _, cephFilesystem := range cephFilesystemList.Items {
|
||||
@@ -62,7 +64,7 @@ func (r *ReconcileClusterDisruption) getFailureDomain(request reconcile.Request)
|
||||
cephObjectStoreList := &cephv1.CephObjectStoreList{}
|
||||
err = r.client.List(context.TODO(), cephObjectStoreList, namespaceListOpt)
|
||||
if err != nil {
|
||||
return "", poolCount, fmt.Errorf("could not list the CephObjectStores %s: %+v", request.NamespacedName, err)
|
||||
return nil, nil, "", poolCount, fmt.Errorf("could not list the CephObjectStores %s: %+v", request.NamespacedName, err)
|
||||
}
|
||||
poolCount += len(cephObjectStoreList.Items)
|
||||
for _, cephObjectStore := range cephObjectStoreList.Items {
|
||||
@@ -70,8 +72,9 @@ func (r *ReconcileClusterDisruption) getFailureDomain(request reconcile.Request)
|
||||
poolSpecs = append(poolSpecs, cephObjectStore.Spec.DataPool)
|
||||
|
||||
}
|
||||
minFailureDomain := getMinimumFailureDomain(poolSpecs)
|
||||
|
||||
return getMinimumFailureDomain(poolSpecs), poolCount, nil
|
||||
return cephObjectStoreList, cephFilesystemList, minFailureDomain, poolCount, nil
|
||||
|
||||
}
|
||||
|
||||
@@ -100,6 +103,7 @@ func getMinimumFailureDomain(poolList []cephv1.PoolSpec) string {
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
logger.Errorf("could not match failure domain. defaulting to %s", cephv1.DefaultFailureDomain)
|
||||
return cephv1.DefaultFailureDomain
|
||||
}
|
||||
return failureDomainOrder[minfailureDomainIndex]
|
||||
@@ -133,3 +137,102 @@ func (r *ReconcileClusterDisruption) getOngoingDrains(request reconcile.Request)
|
||||
}
|
||||
return ongoingDrains, nil
|
||||
}
|
||||
|
||||
// Setting naive minAvailable for RGW at: n - 1
|
||||
func (r *ReconcileClusterDisruption) reconcileCephObjectStore(cephObjectStoreList *cephv1.CephObjectStoreList, drainingOSDs []OsdData) error {
|
||||
for _, objectStore := range cephObjectStoreList.Items {
|
||||
storeName := objectStore.ObjectMeta.Name
|
||||
namespace := objectStore.ObjectMeta.Namespace
|
||||
pdbName := fmt.Sprintf("rook-ceph-rgw-%s", storeName)
|
||||
labelSelector := &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{"rgw": storeName},
|
||||
}
|
||||
|
||||
rgwCount := objectStore.Spec.Gateway.Instances
|
||||
minAvailable := &intstr.IntOrString{IntVal: rgwCount - 1}
|
||||
if minAvailable.IntVal <= 1 {
|
||||
break
|
||||
}
|
||||
blockOwnerDeletion := false
|
||||
pdb := &policyv1beta1.PodDisruptionBudget{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: pdbName,
|
||||
Namespace: namespace,
|
||||
OwnerReferences: []metav1.OwnerReference{
|
||||
{
|
||||
APIVersion: objectStore.APIVersion,
|
||||
Kind: objectStore.Kind,
|
||||
Name: objectStore.ObjectMeta.Name,
|
||||
UID: objectStore.UID,
|
||||
BlockOwnerDeletion: &blockOwnerDeletion,
|
||||
},
|
||||
},
|
||||
},
|
||||
Spec: policyv1beta1.PodDisruptionBudgetSpec{
|
||||
Selector: labelSelector,
|
||||
MinAvailable: minAvailable,
|
||||
},
|
||||
}
|
||||
|
||||
request := types.NamespacedName{Name: pdbName, Namespace: namespace}
|
||||
draining := false
|
||||
if len(drainingOSDs) > 0 {
|
||||
draining = true
|
||||
}
|
||||
err := r.reconcileStaticPDB(request, pdb, draining)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not reconcile cephobjectstore pdb %s: %+v", request, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Setting naive minAvailable for MDS at: n -1
|
||||
// getting n from the cephfilesystem.spec.metadataserver.activecount
|
||||
func (r *ReconcileClusterDisruption) reconcileCephFilesystem(cephFilesystemList *cephv1.CephFilesystemList, drainingOSDs []OsdData) error {
|
||||
for _, filesystem := range cephFilesystemList.Items {
|
||||
fsName := filesystem.ObjectMeta.Name
|
||||
namespace := filesystem.ObjectMeta.Namespace
|
||||
pdbName := fmt.Sprintf("rook-ceph-mds-%s", fsName)
|
||||
labelSelector := &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{"rook_file_system": fsName},
|
||||
}
|
||||
|
||||
activeCount := filesystem.Spec.MetadataServer.ActiveCount
|
||||
minAvailable := &intstr.IntOrString{IntVal: activeCount - 1}
|
||||
if minAvailable.IntVal <= 1 {
|
||||
break
|
||||
}
|
||||
blockOwnerDeletion := false
|
||||
pdb := &policyv1beta1.PodDisruptionBudget{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: pdbName,
|
||||
Namespace: namespace,
|
||||
OwnerReferences: []metav1.OwnerReference{
|
||||
{
|
||||
APIVersion: filesystem.APIVersion,
|
||||
Kind: filesystem.Kind,
|
||||
Name: filesystem.ObjectMeta.Name,
|
||||
UID: filesystem.UID,
|
||||
BlockOwnerDeletion: &blockOwnerDeletion,
|
||||
},
|
||||
},
|
||||
},
|
||||
Spec: policyv1beta1.PodDisruptionBudgetSpec{
|
||||
Selector: labelSelector,
|
||||
MinAvailable: minAvailable,
|
||||
},
|
||||
}
|
||||
|
||||
request := types.NamespacedName{Name: pdbName, Namespace: namespace}
|
||||
draining := false
|
||||
if len(drainingOSDs) > 0 {
|
||||
draining = true
|
||||
}
|
||||
err := r.reconcileStaticPDB(request, pdb, draining)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not reconcile cephfs pdb %s: %+v", request, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -106,8 +106,8 @@ func (r *ReconcileClusterDisruption) reconcile(request reconcile.Request) (recon
|
||||
r.maintenanceTimeout = DefaultMaintenanceTimeout
|
||||
}
|
||||
|
||||
// determine failure domain
|
||||
poolFailureDomain, poolCount, err := r.getFailureDomain(request)
|
||||
// reconcile the pools and get the failure domain
|
||||
cephObjectStoreList, cephFilesystemList, poolFailureDomain, poolCount, err := r.processPools(request)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
@@ -133,6 +133,24 @@ func (r *ReconcileClusterDisruption) reconcile(request reconcile.Request) (recon
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
// reconcile the static mond PDB
|
||||
err = r.reconcileMonPDB(cephCluster, drainingOSDs)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
// reconcile the pdbs for objectstores
|
||||
err = r.reconcileCephObjectStore(cephObjectStoreList, drainingOSDs)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
// reconcile the pdbs for filesystems
|
||||
err = r.reconcileCephFilesystem(cephFilesystemList, drainingOSDs)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
allFailureDomainsMap, err := getFailureDomainMapForOsds(osdDataList, poolFailureDomain)
|
||||
if err != nil {
|
||||
logger.Error(err)
|
||||
@@ -152,7 +170,7 @@ func (r *ReconcileClusterDisruption) reconcile(request reconcile.Request) (recon
|
||||
if err != nil {
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
return reconcile.Result{Requeue: true, RequeueAfter: time.Minute}, nil
|
||||
return reconcile.Result{Requeue: true, RequeueAfter: 45 * time.Second}, nil
|
||||
}
|
||||
|
||||
func emptyResultAndErrorf(format string, a ...interface{}) (reconcile.Result, error) {
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
Copyright 2019 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package clusterdisruption
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
policyv1beta1 "k8s.io/api/policy/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
)
|
||||
|
||||
const (
|
||||
// MonPDBAppName for monitor daemon poddisruptionbudgets
|
||||
MonPDBAppName = "rook-ceph-mon-pdb"
|
||||
)
|
||||
|
||||
func (r *ReconcileClusterDisruption) createStaticPDB(request types.NamespacedName, pdb *policyv1beta1.PodDisruptionBudget) error {
|
||||
err := r.client.Create(context.TODO(), pdb)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PDBs can't be updated, so we use a delete/create
|
||||
// This will change only with kube 1.15: https://github.com/kubernetes/kubernetes/issues/45398#issuecomment-495362316
|
||||
func (r *ReconcileClusterDisruption) updateStaticPDB(request types.NamespacedName, pdb *policyv1beta1.PodDisruptionBudget) error {
|
||||
err := r.client.Delete(context.TODO(), pdb)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return r.createStaticPDB(request, pdb)
|
||||
}
|
||||
|
||||
func (r *ReconcileClusterDisruption) reconcileStaticPDB(request types.NamespacedName, pdb *policyv1beta1.PodDisruptionBudget, draining bool) error {
|
||||
err := r.client.Get(context.TODO(), request, pdb)
|
||||
if errors.IsNotFound(err) {
|
||||
return r.createStaticPDB(request, pdb)
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user