/* Copyright 2020 The Rook Authors. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clusterdisruption import ( "context" "fmt" "testing" "time" "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/client/clientset/versioned/scheme" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/disruption/controllerconfig" "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) const ( healthyCephStatus = `{"fsid":"877a47e0-7f6c-435e-891a-76983ab8c509","health":{"checks":{},"status":"HEALTH_OK"},"election_epoch":12,"quorum":[0,1,2],"quorum_names":["a","b","c"],"monmap":{"epoch":3,"fsid":"877a47e0-7f6c-435e-891a-76983ab8c509","modified":"2020-11-02 09:58:23.015313","created":"2020-11-02 09:57:37.719235","min_mon_release":14,"min_mon_release_name":"nautilus","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.30.74.42:3300","nonce":0},{"type":"v1","addr":"172.30.74.42:6789","nonce":0}]},"addr":"172.30.74.42:6789/0","public_addr":"172.30.74.42:6789/0"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.30.101.61:3300","nonce":0},{"type":"v1","addr":"172.30.101.61:6789","nonce":0}]},"addr":"172.30.101.61:6789/0","public_addr":"172.30.101.61:6789/0"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.30.250.55:3300","nonce":0},{"type":"v1","addr":"172.30.250.55:6789","nonce":0}]},"addr":"172.30.250.55:6789/0","public_addr":"172.30.250.55:6789/0"}]},"osdmap":{"osdmap":{"epoch":19,"num_osds":3,"num_up_osds":3,"num_in_osds":3,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":96}],"num_pgs":96,"num_pools":3,"num_objects":79,"data_bytes":81553681,"bytes_used":3255447552,"bytes_avail":1646011994112,"bytes_total":1649267441664,"read_bytes_sec":853,"write_bytes_sec":5118,"read_op_per_sec":1,"write_op_per_sec":0},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"ocs-storagecluster-cephfilesystem-b","status":"up:active","gid":14161},{"filesystem_id":1,"rank":0,"name":"ocs-storagecluster-cephfilesystem-a","status":"up:standby-replay","gid":24146}],"up:standby":0},"mgrmap":{"epoch":10,"active_gid":14122,"active_name":"a","active_addrs":{"addrvec":[{"type":"v2","addr":"10.131.0.28:6800","nonce":1},{"type":"v1","addr":"10.131.0.28:6801","nonce":1}]}}}` unHealthyCephStatus = `{"fsid":"613975f3-3025-4802-9de1-a2280b950e75","health":{"checks":{"OSD_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 osds down"}},"OSD_HOST_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 host (1 osds) down"}},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","summary":{"message":"Reduced data availability: 101 pgs stale"}},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"application not enabled on 1 pool(s)"}}},"status":"HEALTH_WARN","overall_status":"HEALTH_WARN"},"election_epoch":12,"quorum":[0,1,2],"quorum_names":["rook-ceph-mon0","rook-ceph-mon2","rook-ceph-mon1"],"monmap":{"epoch":3,"fsid":"613975f3-3025-4802-9de1-a2280b950e75","modified":"2017-08-11 20:13:02.075679","created":"2017-08-11 20:12:35.314510","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"10.3.0.45:6789/0","public_addr":"10.3.0.45:6789/0"},{"rank":1,"name":"rook-ceph-mon2","addr":"10.3.0.249:6789/0","public_addr":"10.3.0.249:6789/0"},{"rank":2,"name":"rook-ceph-mon1","addr":"10.3.0.252:6789/0","public_addr":"10.3.0.252:6789/0"}]},"osdmap":{"osdmap":{"epoch":17,"num_osds":2,"num_up_osds":1,"num_in_osds":2,"full":false,"nearfull":true,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"stale+active+clean","count":101},{"state_name":"active+clean","count":99}],"num_pgs":200,"num_pools":2,"num_objects":243,"data_bytes":976793635,"bytes_used":13611479040,"bytes_avail":19825307648,"bytes_total":33436786688},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":14111,"active_name":"rook-ceph-mgr0","active_addr":"10.2.73.6:6800/9","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","prometheus","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}` healthyCephStatusRemapped = `{"fsid":"e32d91a2-24ff-4953-bc4a-6864d31dd2a0","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1177701,"monmap":{"epoch":1,"min_mon_release_name":"quincy","num_mons":1},"osdmap":{"epoch":1800,"num_osds":5,"num_up_osds":5,"osd_up_since":1699834324,"num_in_osds":5,"osd_in_since":1699834304,"num_remapped_pgs":11},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":174},{"state_name":"active+remapped+backfilling","count":10},{"state_name":"active+clean+remapped","count":1}],"num_pgs":185,"num_pools":9,"num_objects":2383,"data_bytes":2222656224,"bytes_used":8793104384,"bytes_avail":18050441216,"bytes_total":26843545600,"misplaced_objects":139,"misplaced_total":7149,"misplaced_ratio":0.019443278780248985,"recovering_objects_per_sec":10,"recovering_bytes_per_sec":9739877,"recovering_keys_per_sec":0,"num_objects_recovered":62,"num_bytes_recovered":58471087,"num_keys_recovered":0,"write_bytes_sec":2982994,"read_op_per_sec":0,"write_op_per_sec":26},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","prometheus","restful"],"services":{"prometheus":"http://10.244.0.36:9283/"}},"servicemap":{"epoch":1,"modified":"0.000000","services":{}},"progress_events":{}}` ) var nodeName = "node01" var namespace = "rook-ceph" var cephCluster = &cephv1.CephCluster{ ObjectMeta: metav1.ObjectMeta{Name: "ceph-cluster"}, } var nodeObj = &corev1.Node{ ObjectMeta: metav1.ObjectMeta{Name: nodeName}, Spec: corev1.NodeSpec{ Unschedulable: false, }, } var unschedulableNodeObj = &corev1.Node{ ObjectMeta: metav1.ObjectMeta{Name: nodeName}, Spec: corev1.NodeSpec{ Unschedulable: true, }, } func fakeOSDDeployment(id, readyReplicas int) appsv1.Deployment { osd := appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("osd-%d", id), Namespace: namespace, Labels: map[string]string{ "app": "rook-ceph-osd", }, }, Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ "topology-location-zone": fmt.Sprintf("zone-%d", id), "ceph-osd-id": fmt.Sprintf("%d", id), }, }, }, }, Status: appsv1.DeploymentStatus{ ReadyReplicas: int32(readyReplicas), }, } return osd } func fakeOSDPod(id int, nodeName string) corev1.Pod { osdPod := corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("osd-%d", id), Namespace: namespace, Labels: map[string]string{ "app": "rook-ceph-osd", "ceph-osd-id": fmt.Sprintf("%d", id), }, }, Spec: corev1.PodSpec{ NodeName: nodeName, }, } return osdPod } func fakePDBConfigMap(drainingFailureDomain string) *corev1.ConfigMap { return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Name: pdbStateMapName, Namespace: namespace}, Data: map[string]string{drainingFailureDomainKey: drainingFailureDomain, setNoOut: ""}, } } func getFakeReconciler(t *testing.T, obj ...runtime.Object) *ReconcileClusterDisruption { scheme := scheme.Scheme err := policyv1.AddToScheme(scheme) assert.NoError(t, err) err = appsv1.AddToScheme(scheme) assert.NoError(t, err) err = corev1.AddToScheme(scheme) assert.NoError(t, err) client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(obj...).Build() return &ReconcileClusterDisruption{ client: client, scheme: scheme, clusterMap: &ClusterMap{clusterMap: map[string]*cephv1.CephCluster{namespace: cephCluster}}, } } func getFakeClusterInfo() *client.ClusterInfo { sharedClusterMap := &ClusterMap{} sharedClusterMap.UpdateClusterMap(namespace, cephCluster) return sharedClusterMap.GetClusterInfo(namespace) } func TestGetOSDFailureDomains(t *testing.T) { testcases := []struct { name string osds []appsv1.Deployment osdPods []corev1.Pod node *corev1.Node expectedAllFailureDomains []string expectedDrainingFailureDomains []string expectedOsdDownFailureDomains []string }{ { name: "case 1: all osds are running", osds: []appsv1.Deployment{fakeOSDDeployment(1, 1), fakeOSDDeployment(2, 1), fakeOSDDeployment(3, 1)}, osdPods: []corev1.Pod{fakeOSDPod(1, nodeName), fakeOSDPod(2, nodeName), fakeOSDPod(3, nodeName)}, node: nodeObj, expectedAllFailureDomains: []string{"zone-1", "zone-2", "zone-3"}, expectedOsdDownFailureDomains: []string{}, expectedDrainingFailureDomains: []string{}, }, { name: "case 2: osd in zone-1 is pending and node is unschedulable", osds: []appsv1.Deployment{fakeOSDDeployment(1, 0), fakeOSDDeployment(2, 1), fakeOSDDeployment(3, 1)}, osdPods: []corev1.Pod{fakeOSDPod(1, ""), fakeOSDPod(2, nodeName), fakeOSDPod(3, nodeName)}, node: nodeObj, expectedAllFailureDomains: []string{"zone-1", "zone-2", "zone-3"}, expectedOsdDownFailureDomains: []string{"zone-1"}, expectedDrainingFailureDomains: []string{"zone-1"}, }, { name: "case 3: osd in zone-1 and zone-2 are pending and node is unschedulable", osds: []appsv1.Deployment{fakeOSDDeployment(1, 0), fakeOSDDeployment(2, 0), fakeOSDDeployment(3, 1)}, osdPods: []corev1.Pod{fakeOSDPod(1, ""), fakeOSDPod(2, ""), fakeOSDPod(3, nodeName)}, node: nodeObj, expectedAllFailureDomains: []string{"zone-1", "zone-2", "zone-3"}, expectedOsdDownFailureDomains: []string{"zone-1", "zone-2"}, expectedDrainingFailureDomains: []string{"zone-1", "zone-2"}, }, { name: "case 4: osd in zone-1 is pending but osd node is schedulable", osds: []appsv1.Deployment{fakeOSDDeployment(1, 0), fakeOSDDeployment(2, 1), fakeOSDDeployment(3, 1)}, osdPods: []corev1.Pod{fakeOSDPod(1, nodeName), fakeOSDPod(2, nodeName), fakeOSDPod(3, nodeName)}, node: nodeObj, expectedAllFailureDomains: []string{"zone-1", "zone-2", "zone-3"}, expectedOsdDownFailureDomains: []string{"zone-1"}, expectedDrainingFailureDomains: []string{}, }, { name: "case 5: osd in zone-1 is pending but osd node is not schedulable", osds: []appsv1.Deployment{fakeOSDDeployment(1, 0), fakeOSDDeployment(2, 1), fakeOSDDeployment(3, 1)}, osdPods: []corev1.Pod{fakeOSDPod(1, nodeName), fakeOSDPod(2, nodeName), fakeOSDPod(3, nodeName)}, node: unschedulableNodeObj, expectedAllFailureDomains: []string{"zone-1", "zone-2", "zone-3"}, expectedOsdDownFailureDomains: []string{"zone-1"}, expectedDrainingFailureDomains: []string{"zone-1"}, }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { objs := []runtime.Object{ cephCluster, &corev1.ConfigMap{}, tc.node, } for _, osdDeployment := range tc.osds { objs = append(objs, osdDeployment.DeepCopy()) } for _, osdPod := range tc.osdPods { objs = append(objs, osdPod.DeepCopy()) } r := getFakeReconciler(t, objs...) clusterInfo := getFakeClusterInfo() clusterInfo.Context = context.TODO() request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}} allfailureDomains, nodeDrainFailureDomains, osdDownFailureDomains, err := r.getOSDFailureDomains(clusterInfo, request, "zone") assert.NoError(t, err) assert.Equal(t, tc.expectedAllFailureDomains, allfailureDomains) assert.Equal(t, tc.expectedDrainingFailureDomains, nodeDrainFailureDomains) assert.Equal(t, tc.expectedOsdDownFailureDomains, osdDownFailureDomains) }) } } func TestGetOSDFailureDomainsError(t *testing.T) { testcases := []struct { name string osds []appsv1.Deployment expectedAllFailureDomains []string expectedDrainingFailureDomains []string expectedOsdDownFailureDomains []string }{ { name: "case 1: one or more OSD deployment is missing crush location label", osds: []appsv1.Deployment{fakeOSDDeployment(1, 1), fakeOSDDeployment(2, 1), fakeOSDDeployment(3, 1)}, expectedAllFailureDomains: nil, expectedDrainingFailureDomains: nil, expectedOsdDownFailureDomains: nil, }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { osd := tc.osds[0].DeepCopy() osd.Spec.Template.ObjectMeta.Labels["topology-location-zone"] = "" r := getFakeReconciler(t, cephCluster, &corev1.ConfigMap{}, tc.osds[1].DeepCopy(), tc.osds[2].DeepCopy(), osd) clusterInfo := getFakeClusterInfo() clusterInfo.Context = context.TODO() request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}} allfailureDomains, nodeDrainFailureDomains, osdDownFailureDomains, err := r.getOSDFailureDomains(clusterInfo, request, "zone") assert.Error(t, err) assert.Equal(t, tc.expectedAllFailureDomains, allfailureDomains) assert.Equal(t, tc.expectedDrainingFailureDomains, nodeDrainFailureDomains) assert.Equal(t, tc.expectedOsdDownFailureDomains, osdDownFailureDomains) }) } } func TestReconcilePDBForOSD(t *testing.T) { testcases := []struct { name string fakeCephStatus string fakeOSDDump string configMap *corev1.ConfigMap allFailureDomains []string osdDownFailureDomains []string activeNodeDrains bool pgHealthyRegex string expectedSetNoOutValue string expectedOSDPDBCount int expectedMaxUnavailableCount int expectedDrainingFailureDomainName string }{ { name: "case 1: no draining failure domain and all pgs are healthy", fakeCephStatus: healthyCephStatus, fakeOSDDump: `{"OSDs": [{"OSD": 3, "Up": 3, "In": 3}]}`, allFailureDomains: []string{"zone-1", "zone-2", "zone-3"}, osdDownFailureDomains: []string{}, configMap: fakePDBConfigMap(""), activeNodeDrains: false, pgHealthyRegex: "", expectedSetNoOutValue: "", expectedOSDPDBCount: 1, expectedMaxUnavailableCount: 1, expectedDrainingFailureDomainName: "", }, { name: "case 2: zone-1 failure domain is draining and pgs are unhealthy", fakeCephStatus: unHealthyCephStatus, fakeOSDDump: `{"OSDs": [{"OSD": 3, "Up": 3, "In": 2}]}`, allFailureDomains: []string{"zone-1", "zone-2", "zone-3"}, osdDownFailureDomains: []string{"zone-1"}, configMap: fakePDBConfigMap(""), activeNodeDrains: true, pgHealthyRegex: "", expectedSetNoOutValue: "true", expectedOSDPDBCount: 2, expectedMaxUnavailableCount: 0, expectedDrainingFailureDomainName: "zone-1", }, { name: "case 3: zone-1 is back online. But pgs are still unhealthy from zone-1 drain", fakeCephStatus: unHealthyCephStatus, fakeOSDDump: `{"OSDs": [{"OSD": 3, "Up": 3, "In": 3}]}`, allFailureDomains: []string{"zone-1", "zone-2", "zone-3"}, osdDownFailureDomains: []string{}, configMap: fakePDBConfigMap("zone-1"), activeNodeDrains: true, pgHealthyRegex: "", expectedSetNoOutValue: "", expectedOSDPDBCount: 2, expectedMaxUnavailableCount: 0, expectedDrainingFailureDomainName: "zone-1", }, { name: "case 4: zone-1 is back online and pgs are also healthy", fakeCephStatus: healthyCephStatus, fakeOSDDump: `{"OSDs": [{"OSD": 3, "Up": 3, "In": 3}]}`, allFailureDomains: []string{"zone-1", "zone-2", "zone-3"}, osdDownFailureDomains: []string{}, configMap: fakePDBConfigMap("zone-1"), activeNodeDrains: true, pgHealthyRegex: "", expectedSetNoOutValue: "", expectedOSDPDBCount: 1, expectedMaxUnavailableCount: 1, expectedDrainingFailureDomainName: "", }, { name: "case 5: remapped pgs are regarded as clean if pgHealthyRegex allows it", fakeCephStatus: healthyCephStatusRemapped, fakeOSDDump: `{"OSDs": [{"OSD": 3, "Up": 3, "In": 3}]}`, allFailureDomains: []string{"zone-1", "zone-2", "zone-3"}, osdDownFailureDomains: []string{}, configMap: fakePDBConfigMap("zone-1"), activeNodeDrains: true, pgHealthyRegex: `^(active\+clean|active\+clean\+scrubbing|active\+clean\+scrubbing\+deep|active\+clean\+remapped|active\+remapped\+backfilling)$`, expectedSetNoOutValue: "", expectedOSDPDBCount: 1, expectedMaxUnavailableCount: 1, expectedDrainingFailureDomainName: "", }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { r := getFakeReconciler(t, cephCluster, tc.configMap) clusterInfo := getFakeClusterInfo() clusterInfo.Context = context.TODO() request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}} executor := &exectest.MockExecutor{} executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { logger.Infof("Command: %s %v", command, args) if args[0] == "status" { return tc.fakeCephStatus, nil } if args[0] == "osd" && args[1] == "dump" { return tc.fakeOSDDump, nil } return "", errors.Errorf("unexpected ceph command '%v'", args) } clientset := test.New(t, 3) // check for PDBV1 version test.SetFakeKubernetesVersion(clientset, "v1.21.0") r.context = &controllerconfig.Context{ClusterdContext: &clusterd.Context{Executor: executor, Clientset: clientset}} _, err := r.reconcilePDBsForOSDs(clusterInfo, request, tc.configMap, "zone", tc.allFailureDomains, tc.osdDownFailureDomains, tc.activeNodeDrains, tc.pgHealthyRegex) assert.NoError(t, err) // assert that pdb for osd are created correctly existingPDBsV1 := &policyv1.PodDisruptionBudgetList{} err = r.client.List(context.TODO(), existingPDBsV1) assert.NoError(t, err) assert.Equal(t, tc.expectedOSDPDBCount, len(existingPDBsV1.Items)) for _, pdb := range existingPDBsV1.Items { assert.Equal(t, tc.expectedMaxUnavailableCount, pdb.Spec.MaxUnavailable.IntValue()) } // assert that config map is updated with correct failure domain existingConfigMaps := &corev1.ConfigMapList{} err = r.client.List(context.TODO(), existingConfigMaps) assert.NoError(t, err) assert.Equal(t, tc.expectedDrainingFailureDomainName, existingConfigMaps.Items[0].Data[drainingFailureDomainKey]) assert.Equal(t, tc.expectedSetNoOutValue, existingConfigMaps.Items[0].Data[setNoOut]) }) } } func TestPGHealthcheckTimeout(t *testing.T) { pdbConfig := fakePDBConfigMap("") r := getFakeReconciler(t, cephCluster, pdbConfig) clusterInfo := getFakeClusterInfo() clusterInfo.Context = context.TODO() request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}} executor := &exectest.MockExecutor{} executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { logger.Infof("Command: %s %v", command, args) if args[0] == "status" { return unHealthyCephStatus, nil } if args[0] == "osd" && args[1] == "dump" { return `{"OSDs": [{"OSD": 3, "Up": 3, "In": 3}]}`, nil } return "", errors.Errorf("unexpected ceph command '%v'", args) } clientset := test.New(t, 3) r.context = &controllerconfig.Context{ClusterdContext: &clusterd.Context{Executor: executor, Clientset: clientset}} // set PG health check timeout to 10 minutes r.pgHealthCheckTimeout = time.Duration(time.Minute * 10) // reconcile OSD PDB with active drains (on zone-1) and unhealthy PGs _, err := r.reconcilePDBsForOSDs(clusterInfo, request, pdbConfig, "zone", []string{"zone-1", "zone-2"}, []string{"zone-1"}, true, "") assert.NoError(t, err) assert.Equal(t, "zone-1", pdbConfig.Data[drainingFailureDomainKey]) assert.Equal(t, "true", pdbConfig.Data[setNoOut]) // update the pgHealthCheckDuration time by -9 minutes pdbConfig.Data[pgHealthCheckDurationKey] = time.Now().Add(time.Duration(-7) * time.Minute).Format(time.RFC3339) // reconcile OSD PDB with no active drains and unhealthy PGs _, err = r.reconcilePDBsForOSDs(clusterInfo, request, pdbConfig, "zone", []string{"zone-1", "zone-2"}, []string{}, true, "") assert.NoError(t, err) // assert that pdb config map was not reset as the PG health check was not timed out assert.Equal(t, "zone-1", pdbConfig.Data[drainingFailureDomainKey]) assert.Equal(t, "true", pdbConfig.Data[setNoOut]) // update the drainingFailureDomain time by -9 minutes pdbConfig.Data[pgHealthCheckDurationKey] = time.Now().Add(time.Duration(-11) * time.Minute).Format(time.RFC3339) // reconcile OSD PDB with no active drains and unhealthy PGs _, err = r.reconcilePDBsForOSDs(clusterInfo, request, pdbConfig, "zone", []string{"zone-1", "zone-2"}, []string{}, false, "") assert.NoError(t, err) // assert that pdb config map was reset as the PG health check was timed out assert.Equal(t, "", pdbConfig.Data[drainingFailureDomainKey]) assert.Equal(t, "", pdbConfig.Data[setNoOut]) } func TestHasNodeDrained(t *testing.T) { osdPOD := fakeOSDPod(0, nodeName) ctx := context.TODO() // Not expecting node drain because OSD pod is assigned to a schedulable node r := getFakeReconciler(t, nodeObj, osdPOD.DeepCopy(), &corev1.ConfigMap{}) expected, err := hasOSDNodeDrained(ctx, r.client, namespace, "0") assert.NoError(t, err) assert.False(t, expected) // Expecting node drain because OSD pod is assigned to an unschedulable node r = getFakeReconciler(t, unschedulableNodeObj, osdPOD.DeepCopy(), &corev1.ConfigMap{}) expected, err = hasOSDNodeDrained(ctx, r.client, namespace, "0") assert.NoError(t, err) assert.True(t, expected) // Expecting node drain because OSD pod is not assigned to any node osdPodObj := osdPOD.DeepCopy() osdPodObj.Spec.NodeName = "" r = getFakeReconciler(t, nodeObj, osdPodObj, &corev1.ConfigMap{}) expected, err = hasOSDNodeDrained(ctx, r.client, namespace, "0") assert.NoError(t, err) assert.True(t, expected) } func TestGetAllowedDisruptions(t *testing.T) { r := getFakeReconciler(t) clientset := test.New(t, 3) test.SetFakeKubernetesVersion(clientset, "v1.21.0") r.context = &controllerconfig.Context{ClusterdContext: &clusterd.Context{Clientset: clientset}} // Default PDB is not available allowedDisruptions, err := r.getAllowedDisruptions(osdPDBAppName, namespace) assert.Error(t, err) assert.Equal(t, int32(-1), allowedDisruptions) // Default PDB is available pdb := &policyv1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Name: osdPDBAppName, Namespace: namespace, }, Status: policyv1.PodDisruptionBudgetStatus{ DisruptionsAllowed: int32(0), }, } err = r.client.Create(context.TODO(), pdb) assert.NoError(t, err) allowedDisruptions, err = r.getAllowedDisruptions(osdPDBAppName, namespace) assert.NoError(t, err) assert.Equal(t, int32(0), allowedDisruptions) }