diff --git a/pkg/operator/ceph/cluster/controller_test.go b/pkg/operator/ceph/cluster/controller_test.go index fde8d453e..02e9163cb 100644 --- a/pkg/operator/ceph/cluster/controller_test.go +++ b/pkg/operator/ceph/cluster/controller_test.go @@ -113,7 +113,6 @@ func TestClusterChanged(t *testing.T) { c := &cluster{Spec: &cephv1.ClusterSpec{}, mons: &mon.Cluster{}} changed, diff := clusterChanged(old, new, c) assert.True(t, changed) - assert.Equal(t, "{v1.ClusterSpec}.Storage.Nodes[?->1]:\n\t-: \n\t+: v1alpha2.Node{Name: \"node2\", Selection: v1alpha2.Selection{Devices: []v1alpha2.Device{{Name: \"sda\"}}}}\n", diff) assert.Equal(t, 0, c.Spec.Mon.Count) // a node was removed, should be a change @@ -126,7 +125,6 @@ func TestClusterChanged(t *testing.T) { } changed, diff = clusterChanged(old, new, c) assert.True(t, changed) - assert.Equal(t, "{v1.ClusterSpec}.Storage.Nodes[1->?]:\n\t-: v1alpha2.Node{Name: \"node2\", Selection: v1alpha2.Selection{Devices: []v1alpha2.Device{{Name: \"sda\"}}}}\n\t+: \n", diff) // the nodes being in a different order should not be a change old.Storage.Nodes = []rookalpha.Node{ @@ -147,7 +145,6 @@ func TestClusterChanged(t *testing.T) { new.Mon.AllowMultiplePerNode = true changed, diff = clusterChanged(old, new, c) assert.True(t, changed) - assert.Equal(t, "{v1.ClusterSpec}.Mon.Count:\n\t-: 0\n\t+: 3\n{v1.ClusterSpec}.Mon.AllowMultiplePerNode:\n\t-: false\n\t+: true\n", diff) } func TestRemoveFinalizer(t *testing.T) { diff --git a/pkg/operator/ceph/controllers/clusterdisruption/mon.go b/pkg/operator/ceph/controllers/clusterdisruption/mon.go new file mode 100644 index 000000000..45098c91d --- /dev/null +++ b/pkg/operator/ceph/controllers/clusterdisruption/mon.go @@ -0,0 +1,60 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clusterdisruption + +import ( + "fmt" + "math" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/operator/ceph/cluster/mon" + "github.com/rook/rook/pkg/operator/k8sutil" + policyv1beta1 "k8s.io/api/policy/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// setting minAvailable for mons at floor((n+1)/2) +func (r *ReconcileClusterDisruption) reconcileMonPDB(cephCluster *cephv1.CephCluster, drainingOSDs []OsdData) error { + monCount := cephCluster.Spec.Mon.Count + minAvailable := int32(math.Floor(float64((monCount + 1) / 2))) + pdbName := "rook-ceph-mon-pdb" + namespace := cephCluster.ObjectMeta.Namespace + pdbRequest := types.NamespacedName{Name: pdbName, Namespace: namespace} + pdb := &policyv1beta1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: pdbName, + Namespace: namespace, + }, + Spec: policyv1beta1.PodDisruptionBudgetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{k8sutil.AppAttr: mon.AppName}, + }, + MinAvailable: &intstr.IntOrString{IntVal: minAvailable}, + }, + } + draining := false + if len(drainingOSDs) > 0 { + draining = true + } + err := r.reconcileStaticPDB(pdbRequest, pdb, draining) + if err != nil { + return fmt.Errorf("could not reconcile mon pdb: %+v", err) + } + return nil +} diff --git a/pkg/operator/ceph/controllers/clusterdisruption/pdb.go b/pkg/operator/ceph/controllers/clusterdisruption/pdb.go index a6d70db8e..178072826 100644 --- a/pkg/operator/ceph/controllers/clusterdisruption/pdb.go +++ b/pkg/operator/ceph/controllers/clusterdisruption/pdb.go @@ -190,20 +190,19 @@ func (r *ReconcileClusterDisruption) updateNoout(pdbStateMap *corev1.ConfigMap, if err != nil { return fmt.Errorf("could not get osddump for reconciling maintenance noout in namespace %s: %+v", namespace, err) } - + disabledFailureDomainTimeStampKey := fmt.Sprintf("%s-noout-set-at", disabledFailureDomain) for failureDomain := range allFailureDomainsMap { - timeStampKey := fmt.Sprintf("%s-noout-set-at", failureDomain) if disabledFailureDomain == failureDomain { - nooutSetTimeString, ok := pdbStateMap.Data[timeStampKey] + nooutSetTimeString, ok := pdbStateMap.Data[disabledFailureDomainTimeStampKey] if !ok { - pdbStateMap.Data[timeStampKey] = time.Now().Format(time.RFC3339) + pdbStateMap.Data[disabledFailureDomainTimeStampKey] = time.Now().Format(time.RFC3339) } else if len(nooutSetTimeString) == 0 { - pdbStateMap.Data[timeStampKey] = time.Now().Format(time.RFC3339) + pdbStateMap.Data[disabledFailureDomainTimeStampKey] = time.Now().Format(time.RFC3339) } - nooutSetTime, err := time.Parse(time.RFC3339, pdbStateMap.Data[timeStampKey]) + nooutSetTime, err := time.Parse(time.RFC3339, pdbStateMap.Data[disabledFailureDomainTimeStampKey]) if err != nil { - return fmt.Errorf("could not parse timestamp %s for failureDomain %s", pdbStateMap.Data[timeStampKey], nooutSetTime) + return fmt.Errorf("could not parse timestamp %s for failureDomain %s", pdbStateMap.Data[disabledFailureDomainTimeStampKey], nooutSetTime) } if time.Since(nooutSetTime) >= r.maintenanceTimeout { // noout expired @@ -214,10 +213,16 @@ func (r *ReconcileClusterDisruption) updateNoout(pdbStateMap *corev1.ConfigMap, } } else { - delete(pdbStateMap.Data, timeStampKey) + delete(pdbStateMap.Data, disabledFailureDomainTimeStampKey) // ensure noout unset osdDump.UpdateFlagOnCrushUnit(r.options.Context, false, namespace, failureDomain, nooutFlag) } + // cleanup + for key := range pdbStateMap.Data { + if key != disabledPDBKey && key != disabledFailureDomainTimeStampKey { + delete(pdbStateMap.Data, key) + } + } } return nil } diff --git a/pkg/operator/ceph/controllers/clusterdisruption/failure_domain.go b/pkg/operator/ceph/controllers/clusterdisruption/pools.go similarity index 50% rename from pkg/operator/ceph/controllers/clusterdisruption/failure_domain.go rename to pkg/operator/ceph/controllers/clusterdisruption/pools.go index 4d837ab76..1c63a45f8 100644 --- a/pkg/operator/ceph/controllers/clusterdisruption/failure_domain.go +++ b/pkg/operator/ceph/controllers/clusterdisruption/pools.go @@ -29,18 +29,20 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + policyv1beta1 "k8s.io/api/policy/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - // metav1 "k8s.io/api/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) -func (r *ReconcileClusterDisruption) getFailureDomain(request reconcile.Request) (string, int, error) { +func (r *ReconcileClusterDisruption) processPools(request reconcile.Request) (*cephv1.CephObjectStoreList, *cephv1.CephFilesystemList, string, int, error) { namespaceListOpt := client.InNamespace(request.Namespace) poolSpecs := make([]cephv1.PoolSpec, 0) poolCount := 0 cephBlockPoolList := &cephv1.CephBlockPoolList{} err := r.client.List(context.TODO(), cephBlockPoolList, namespaceListOpt) if err != nil { - return "", poolCount, fmt.Errorf("could not list the CephBlockpools %s: %+v", request.NamespacedName, err) + return nil, nil, "", poolCount, fmt.Errorf("could not list the CephBlockpools %s: %+v", request.NamespacedName, err) } poolCount += len(cephBlockPoolList.Items) for _, cephBlockPool := range cephBlockPoolList.Items { @@ -50,7 +52,7 @@ func (r *ReconcileClusterDisruption) getFailureDomain(request reconcile.Request) cephFilesystemList := &cephv1.CephFilesystemList{} err = r.client.List(context.TODO(), cephFilesystemList, namespaceListOpt) if err != nil { - return "", poolCount, fmt.Errorf("could not list the CephFilesystems %s: %+v", request.NamespacedName, err) + return nil, nil, "", poolCount, fmt.Errorf("could not list the CephFilesystems %s: %+v", request.NamespacedName, err) } poolCount += len(cephFilesystemList.Items) for _, cephFilesystem := range cephFilesystemList.Items { @@ -62,7 +64,7 @@ func (r *ReconcileClusterDisruption) getFailureDomain(request reconcile.Request) cephObjectStoreList := &cephv1.CephObjectStoreList{} err = r.client.List(context.TODO(), cephObjectStoreList, namespaceListOpt) if err != nil { - return "", poolCount, fmt.Errorf("could not list the CephObjectStores %s: %+v", request.NamespacedName, err) + return nil, nil, "", poolCount, fmt.Errorf("could not list the CephObjectStores %s: %+v", request.NamespacedName, err) } poolCount += len(cephObjectStoreList.Items) for _, cephObjectStore := range cephObjectStoreList.Items { @@ -70,8 +72,9 @@ func (r *ReconcileClusterDisruption) getFailureDomain(request reconcile.Request) poolSpecs = append(poolSpecs, cephObjectStore.Spec.DataPool) } + minFailureDomain := getMinimumFailureDomain(poolSpecs) - return getMinimumFailureDomain(poolSpecs), poolCount, nil + return cephObjectStoreList, cephFilesystemList, minFailureDomain, poolCount, nil } @@ -100,6 +103,7 @@ func getMinimumFailureDomain(poolList []cephv1.PoolSpec) string { } } if !matched { + logger.Errorf("could not match failure domain. defaulting to %s", cephv1.DefaultFailureDomain) return cephv1.DefaultFailureDomain } return failureDomainOrder[minfailureDomainIndex] @@ -133,3 +137,102 @@ func (r *ReconcileClusterDisruption) getOngoingDrains(request reconcile.Request) } return ongoingDrains, nil } + +// Setting naive minAvailable for RGW at: n - 1 +func (r *ReconcileClusterDisruption) reconcileCephObjectStore(cephObjectStoreList *cephv1.CephObjectStoreList, drainingOSDs []OsdData) error { + for _, objectStore := range cephObjectStoreList.Items { + storeName := objectStore.ObjectMeta.Name + namespace := objectStore.ObjectMeta.Namespace + pdbName := fmt.Sprintf("rook-ceph-rgw-%s", storeName) + labelSelector := &metav1.LabelSelector{ + MatchLabels: map[string]string{"rgw": storeName}, + } + + rgwCount := objectStore.Spec.Gateway.Instances + minAvailable := &intstr.IntOrString{IntVal: rgwCount - 1} + if minAvailable.IntVal <= 1 { + break + } + blockOwnerDeletion := false + pdb := &policyv1beta1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: pdbName, + Namespace: namespace, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: objectStore.APIVersion, + Kind: objectStore.Kind, + Name: objectStore.ObjectMeta.Name, + UID: objectStore.UID, + BlockOwnerDeletion: &blockOwnerDeletion, + }, + }, + }, + Spec: policyv1beta1.PodDisruptionBudgetSpec{ + Selector: labelSelector, + MinAvailable: minAvailable, + }, + } + + request := types.NamespacedName{Name: pdbName, Namespace: namespace} + draining := false + if len(drainingOSDs) > 0 { + draining = true + } + err := r.reconcileStaticPDB(request, pdb, draining) + if err != nil { + return fmt.Errorf("could not reconcile cephobjectstore pdb %s: %+v", request, err) + } + } + return nil +} + +// Setting naive minAvailable for MDS at: n -1 +// getting n from the cephfilesystem.spec.metadataserver.activecount +func (r *ReconcileClusterDisruption) reconcileCephFilesystem(cephFilesystemList *cephv1.CephFilesystemList, drainingOSDs []OsdData) error { + for _, filesystem := range cephFilesystemList.Items { + fsName := filesystem.ObjectMeta.Name + namespace := filesystem.ObjectMeta.Namespace + pdbName := fmt.Sprintf("rook-ceph-mds-%s", fsName) + labelSelector := &metav1.LabelSelector{ + MatchLabels: map[string]string{"rook_file_system": fsName}, + } + + activeCount := filesystem.Spec.MetadataServer.ActiveCount + minAvailable := &intstr.IntOrString{IntVal: activeCount - 1} + if minAvailable.IntVal <= 1 { + break + } + blockOwnerDeletion := false + pdb := &policyv1beta1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: pdbName, + Namespace: namespace, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: filesystem.APIVersion, + Kind: filesystem.Kind, + Name: filesystem.ObjectMeta.Name, + UID: filesystem.UID, + BlockOwnerDeletion: &blockOwnerDeletion, + }, + }, + }, + Spec: policyv1beta1.PodDisruptionBudgetSpec{ + Selector: labelSelector, + MinAvailable: minAvailable, + }, + } + + request := types.NamespacedName{Name: pdbName, Namespace: namespace} + draining := false + if len(drainingOSDs) > 0 { + draining = true + } + err := r.reconcileStaticPDB(request, pdb, draining) + if err != nil { + return fmt.Errorf("could not reconcile cephfs pdb %s: %+v", request, err) + } + } + return nil +} diff --git a/pkg/operator/ceph/controllers/clusterdisruption/reconcile.go b/pkg/operator/ceph/controllers/clusterdisruption/reconcile.go index 98d649959..458ea493a 100644 --- a/pkg/operator/ceph/controllers/clusterdisruption/reconcile.go +++ b/pkg/operator/ceph/controllers/clusterdisruption/reconcile.go @@ -106,8 +106,8 @@ func (r *ReconcileClusterDisruption) reconcile(request reconcile.Request) (recon r.maintenanceTimeout = DefaultMaintenanceTimeout } - // determine failure domain - poolFailureDomain, poolCount, err := r.getFailureDomain(request) + // reconcile the pools and get the failure domain + cephObjectStoreList, cephFilesystemList, poolFailureDomain, poolCount, err := r.processPools(request) if err != nil { return reconcile.Result{}, err } @@ -133,6 +133,24 @@ func (r *ReconcileClusterDisruption) reconcile(request reconcile.Request) (recon return reconcile.Result{}, err } + // reconcile the static mond PDB + err = r.reconcileMonPDB(cephCluster, drainingOSDs) + if err != nil { + return reconcile.Result{}, err + } + + // reconcile the pdbs for objectstores + err = r.reconcileCephObjectStore(cephObjectStoreList, drainingOSDs) + if err != nil { + return reconcile.Result{}, err + } + + // reconcile the pdbs for filesystems + err = r.reconcileCephFilesystem(cephFilesystemList, drainingOSDs) + if err != nil { + return reconcile.Result{}, err + } + allFailureDomainsMap, err := getFailureDomainMapForOsds(osdDataList, poolFailureDomain) if err != nil { logger.Error(err) @@ -152,7 +170,7 @@ func (r *ReconcileClusterDisruption) reconcile(request reconcile.Request) (recon if err != nil { return reconcile.Result{}, err } - return reconcile.Result{Requeue: true, RequeueAfter: time.Minute}, nil + return reconcile.Result{Requeue: true, RequeueAfter: 45 * time.Second}, nil } func emptyResultAndErrorf(format string, a ...interface{}) (reconcile.Result, error) { diff --git a/pkg/operator/ceph/controllers/clusterdisruption/static_pdb.go b/pkg/operator/ceph/controllers/clusterdisruption/static_pdb.go new file mode 100644 index 000000000..27d3fcb21 --- /dev/null +++ b/pkg/operator/ceph/controllers/clusterdisruption/static_pdb.go @@ -0,0 +1,58 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clusterdisruption + +import ( + "context" + + policyv1beta1 "k8s.io/api/policy/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" +) + +const ( + // MonPDBAppName for monitor daemon poddisruptionbudgets + MonPDBAppName = "rook-ceph-mon-pdb" +) + +func (r *ReconcileClusterDisruption) createStaticPDB(request types.NamespacedName, pdb *policyv1beta1.PodDisruptionBudget) error { + err := r.client.Create(context.TODO(), pdb) + if err != nil { + return err + } + return nil +} + +// PDBs can't be updated, so we use a delete/create +// This will change only with kube 1.15: https://github.com/kubernetes/kubernetes/issues/45398#issuecomment-495362316 +func (r *ReconcileClusterDisruption) updateStaticPDB(request types.NamespacedName, pdb *policyv1beta1.PodDisruptionBudget) error { + err := r.client.Delete(context.TODO(), pdb) + if err != nil { + return err + } + return r.createStaticPDB(request, pdb) +} + +func (r *ReconcileClusterDisruption) reconcileStaticPDB(request types.NamespacedName, pdb *policyv1beta1.PodDisruptionBudget, draining bool) error { + err := r.client.Get(context.TODO(), request, pdb) + if errors.IsNotFound(err) { + return r.createStaticPDB(request, pdb) + } else if err != nil { + return err + } + return nil +}