504 lines
25 KiB
Go
504 lines
25 KiB
Go
/*
|
|
Copyright 2020 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package clusterdisruption
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
|
"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
|
|
"github.com/rook/rook/pkg/clusterd"
|
|
"github.com/rook/rook/pkg/daemon/ceph/client"
|
|
"github.com/rook/rook/pkg/operator/ceph/disruption/controllerconfig"
|
|
"github.com/rook/rook/pkg/operator/test"
|
|
exectest "github.com/rook/rook/pkg/util/exec/test"
|
|
"github.com/stretchr/testify/assert"
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
policyv1 "k8s.io/api/policy/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
|
)
|
|
|
|
const (
|
|
healthyCephStatus = `{"fsid":"877a47e0-7f6c-435e-891a-76983ab8c509","health":{"checks":{},"status":"HEALTH_OK"},"election_epoch":12,"quorum":[0,1,2],"quorum_names":["a","b","c"],"monmap":{"epoch":3,"fsid":"877a47e0-7f6c-435e-891a-76983ab8c509","modified":"2020-11-02 09:58:23.015313","created":"2020-11-02 09:57:37.719235","min_mon_release":14,"min_mon_release_name":"nautilus","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.30.74.42:3300","nonce":0},{"type":"v1","addr":"172.30.74.42:6789","nonce":0}]},"addr":"172.30.74.42:6789/0","public_addr":"172.30.74.42:6789/0"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.30.101.61:3300","nonce":0},{"type":"v1","addr":"172.30.101.61:6789","nonce":0}]},"addr":"172.30.101.61:6789/0","public_addr":"172.30.101.61:6789/0"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.30.250.55:3300","nonce":0},{"type":"v1","addr":"172.30.250.55:6789","nonce":0}]},"addr":"172.30.250.55:6789/0","public_addr":"172.30.250.55:6789/0"}]},"osdmap":{"osdmap":{"epoch":19,"num_osds":3,"num_up_osds":3,"num_in_osds":3,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":96}],"num_pgs":96,"num_pools":3,"num_objects":79,"data_bytes":81553681,"bytes_used":3255447552,"bytes_avail":1646011994112,"bytes_total":1649267441664,"read_bytes_sec":853,"write_bytes_sec":5118,"read_op_per_sec":1,"write_op_per_sec":0},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"ocs-storagecluster-cephfilesystem-b","status":"up:active","gid":14161},{"filesystem_id":1,"rank":0,"name":"ocs-storagecluster-cephfilesystem-a","status":"up:standby-replay","gid":24146}],"up:standby":0},"mgrmap":{"epoch":10,"active_gid":14122,"active_name":"a","active_addrs":{"addrvec":[{"type":"v2","addr":"10.131.0.28:6800","nonce":1},{"type":"v1","addr":"10.131.0.28:6801","nonce":1}]}}}`
|
|
unHealthyCephStatus = `{"fsid":"613975f3-3025-4802-9de1-a2280b950e75","health":{"checks":{"OSD_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 osds down"}},"OSD_HOST_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 host (1 osds) down"}},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","summary":{"message":"Reduced data availability: 101 pgs stale"}},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"application not enabled on 1 pool(s)"}}},"status":"HEALTH_WARN","overall_status":"HEALTH_WARN"},"election_epoch":12,"quorum":[0,1,2],"quorum_names":["rook-ceph-mon0","rook-ceph-mon2","rook-ceph-mon1"],"monmap":{"epoch":3,"fsid":"613975f3-3025-4802-9de1-a2280b950e75","modified":"2017-08-11 20:13:02.075679","created":"2017-08-11 20:12:35.314510","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"10.3.0.45:6789/0","public_addr":"10.3.0.45:6789/0"},{"rank":1,"name":"rook-ceph-mon2","addr":"10.3.0.249:6789/0","public_addr":"10.3.0.249:6789/0"},{"rank":2,"name":"rook-ceph-mon1","addr":"10.3.0.252:6789/0","public_addr":"10.3.0.252:6789/0"}]},"osdmap":{"osdmap":{"epoch":17,"num_osds":2,"num_up_osds":1,"num_in_osds":2,"full":false,"nearfull":true,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"stale+active+clean","count":101},{"state_name":"active+clean","count":99}],"num_pgs":200,"num_pools":2,"num_objects":243,"data_bytes":976793635,"bytes_used":13611479040,"bytes_avail":19825307648,"bytes_total":33436786688},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":14111,"active_name":"rook-ceph-mgr0","active_addr":"10.2.73.6:6800/9","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","prometheus","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}`
|
|
healthyCephStatusRemapped = `{"fsid":"e32d91a2-24ff-4953-bc4a-6864d31dd2a0","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1177701,"monmap":{"epoch":1,"min_mon_release_name":"quincy","num_mons":1},"osdmap":{"epoch":1800,"num_osds":5,"num_up_osds":5,"osd_up_since":1699834324,"num_in_osds":5,"osd_in_since":1699834304,"num_remapped_pgs":11},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":174},{"state_name":"active+remapped+backfilling","count":10},{"state_name":"active+clean+remapped","count":1}],"num_pgs":185,"num_pools":9,"num_objects":2383,"data_bytes":2222656224,"bytes_used":8793104384,"bytes_avail":18050441216,"bytes_total":26843545600,"misplaced_objects":139,"misplaced_total":7149,"misplaced_ratio":0.019443278780248985,"recovering_objects_per_sec":10,"recovering_bytes_per_sec":9739877,"recovering_keys_per_sec":0,"num_objects_recovered":62,"num_bytes_recovered":58471087,"num_keys_recovered":0,"write_bytes_sec":2982994,"read_op_per_sec":0,"write_op_per_sec":26},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","prometheus","restful"],"services":{"prometheus":"http://10.244.0.36:9283/"}},"servicemap":{"epoch":1,"modified":"0.000000","services":{}},"progress_events":{}}`
|
|
)
|
|
|
|
var nodeName = "node01"
|
|
var namespace = "rook-ceph"
|
|
|
|
var cephCluster = &cephv1.CephCluster{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "ceph-cluster"},
|
|
}
|
|
|
|
var nodeObj = &corev1.Node{
|
|
ObjectMeta: metav1.ObjectMeta{Name: nodeName},
|
|
Spec: corev1.NodeSpec{
|
|
Unschedulable: false,
|
|
},
|
|
}
|
|
|
|
var unschedulableNodeObj = &corev1.Node{
|
|
ObjectMeta: metav1.ObjectMeta{Name: nodeName},
|
|
Spec: corev1.NodeSpec{
|
|
Unschedulable: true,
|
|
},
|
|
}
|
|
|
|
func fakeOSDDeployment(id, readyReplicas int) appsv1.Deployment {
|
|
osd := appsv1.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: fmt.Sprintf("osd-%d", id),
|
|
Namespace: namespace,
|
|
Labels: map[string]string{
|
|
"app": "rook-ceph-osd",
|
|
},
|
|
},
|
|
Spec: appsv1.DeploymentSpec{
|
|
Template: corev1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{
|
|
"topology-location-zone": fmt.Sprintf("zone-%d", id),
|
|
"ceph-osd-id": fmt.Sprintf("%d", id),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Status: appsv1.DeploymentStatus{
|
|
ReadyReplicas: int32(readyReplicas),
|
|
},
|
|
}
|
|
return osd
|
|
}
|
|
|
|
func fakeOSDPod(id int, nodeName string) corev1.Pod {
|
|
osdPod := corev1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: fmt.Sprintf("osd-%d", id),
|
|
Namespace: namespace,
|
|
Labels: map[string]string{
|
|
"app": "rook-ceph-osd",
|
|
"ceph-osd-id": fmt.Sprintf("%d", id),
|
|
},
|
|
},
|
|
Spec: corev1.PodSpec{
|
|
NodeName: nodeName,
|
|
},
|
|
}
|
|
return osdPod
|
|
}
|
|
|
|
func fakePDBConfigMap(drainingFailureDomain string) *corev1.ConfigMap {
|
|
return &corev1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{Name: pdbStateMapName, Namespace: namespace},
|
|
Data: map[string]string{drainingFailureDomainKey: drainingFailureDomain, setNoOut: ""},
|
|
}
|
|
}
|
|
|
|
func getFakeReconciler(t *testing.T, obj ...runtime.Object) *ReconcileClusterDisruption {
|
|
scheme := scheme.Scheme
|
|
err := policyv1.AddToScheme(scheme)
|
|
assert.NoError(t, err)
|
|
|
|
err = appsv1.AddToScheme(scheme)
|
|
assert.NoError(t, err)
|
|
err = corev1.AddToScheme(scheme)
|
|
assert.NoError(t, err)
|
|
client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(obj...).Build()
|
|
|
|
return &ReconcileClusterDisruption{
|
|
client: client,
|
|
scheme: scheme,
|
|
clusterMap: &ClusterMap{clusterMap: map[string]*cephv1.CephCluster{namespace: cephCluster}},
|
|
}
|
|
}
|
|
|
|
func getFakeClusterInfo() *client.ClusterInfo {
|
|
sharedClusterMap := &ClusterMap{}
|
|
sharedClusterMap.UpdateClusterMap(namespace, cephCluster)
|
|
return sharedClusterMap.GetClusterInfo(namespace)
|
|
|
|
}
|
|
|
|
func TestGetOSDFailureDomains(t *testing.T) {
|
|
testcases := []struct {
|
|
name string
|
|
osds []appsv1.Deployment
|
|
osdPods []corev1.Pod
|
|
node *corev1.Node
|
|
expectedAllFailureDomains []string
|
|
expectedDrainingFailureDomains []string
|
|
expectedOsdDownFailureDomains []string
|
|
}{
|
|
{
|
|
name: "case 1: all osds are running",
|
|
osds: []appsv1.Deployment{fakeOSDDeployment(1, 1), fakeOSDDeployment(2, 1),
|
|
fakeOSDDeployment(3, 1)},
|
|
osdPods: []corev1.Pod{fakeOSDPod(1, nodeName), fakeOSDPod(2, nodeName),
|
|
fakeOSDPod(3, nodeName)},
|
|
node: nodeObj,
|
|
expectedAllFailureDomains: []string{"zone-1", "zone-2", "zone-3"},
|
|
expectedOsdDownFailureDomains: []string{},
|
|
expectedDrainingFailureDomains: []string{},
|
|
},
|
|
{
|
|
name: "case 2: osd in zone-1 is pending and node is unschedulable",
|
|
osds: []appsv1.Deployment{fakeOSDDeployment(1, 0), fakeOSDDeployment(2, 1),
|
|
fakeOSDDeployment(3, 1)},
|
|
osdPods: []corev1.Pod{fakeOSDPod(1, ""), fakeOSDPod(2, nodeName),
|
|
fakeOSDPod(3, nodeName)},
|
|
node: nodeObj,
|
|
expectedAllFailureDomains: []string{"zone-1", "zone-2", "zone-3"},
|
|
expectedOsdDownFailureDomains: []string{"zone-1"},
|
|
expectedDrainingFailureDomains: []string{"zone-1"},
|
|
},
|
|
{
|
|
name: "case 3: osd in zone-1 and zone-2 are pending and node is unschedulable",
|
|
osds: []appsv1.Deployment{fakeOSDDeployment(1, 0), fakeOSDDeployment(2, 0),
|
|
fakeOSDDeployment(3, 1)},
|
|
osdPods: []corev1.Pod{fakeOSDPod(1, ""), fakeOSDPod(2, ""),
|
|
fakeOSDPod(3, nodeName)},
|
|
node: nodeObj,
|
|
expectedAllFailureDomains: []string{"zone-1", "zone-2", "zone-3"},
|
|
expectedOsdDownFailureDomains: []string{"zone-1", "zone-2"},
|
|
expectedDrainingFailureDomains: []string{"zone-1", "zone-2"},
|
|
},
|
|
{
|
|
name: "case 4: osd in zone-1 is pending but osd node is schedulable",
|
|
osds: []appsv1.Deployment{fakeOSDDeployment(1, 0), fakeOSDDeployment(2, 1),
|
|
fakeOSDDeployment(3, 1)},
|
|
osdPods: []corev1.Pod{fakeOSDPod(1, nodeName), fakeOSDPod(2, nodeName),
|
|
fakeOSDPod(3, nodeName)},
|
|
node: nodeObj,
|
|
expectedAllFailureDomains: []string{"zone-1", "zone-2", "zone-3"},
|
|
expectedOsdDownFailureDomains: []string{"zone-1"},
|
|
expectedDrainingFailureDomains: []string{},
|
|
},
|
|
{
|
|
name: "case 5: osd in zone-1 is pending but osd node is not schedulable",
|
|
osds: []appsv1.Deployment{fakeOSDDeployment(1, 0), fakeOSDDeployment(2, 1),
|
|
fakeOSDDeployment(3, 1)},
|
|
osdPods: []corev1.Pod{fakeOSDPod(1, nodeName), fakeOSDPod(2, nodeName),
|
|
fakeOSDPod(3, nodeName)},
|
|
node: unschedulableNodeObj,
|
|
expectedAllFailureDomains: []string{"zone-1", "zone-2", "zone-3"},
|
|
expectedOsdDownFailureDomains: []string{"zone-1"},
|
|
expectedDrainingFailureDomains: []string{"zone-1"},
|
|
},
|
|
}
|
|
|
|
for _, tc := range testcases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
objs := []runtime.Object{
|
|
cephCluster,
|
|
&corev1.ConfigMap{},
|
|
tc.node,
|
|
}
|
|
for _, osdDeployment := range tc.osds {
|
|
objs = append(objs, osdDeployment.DeepCopy())
|
|
}
|
|
for _, osdPod := range tc.osdPods {
|
|
objs = append(objs, osdPod.DeepCopy())
|
|
}
|
|
r := getFakeReconciler(t, objs...)
|
|
clusterInfo := getFakeClusterInfo()
|
|
clusterInfo.Context = context.TODO()
|
|
request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}}
|
|
allfailureDomains, nodeDrainFailureDomains, osdDownFailureDomains, err := r.getOSDFailureDomains(clusterInfo, request, "zone")
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, tc.expectedAllFailureDomains, allfailureDomains)
|
|
assert.Equal(t, tc.expectedDrainingFailureDomains, nodeDrainFailureDomains)
|
|
assert.Equal(t, tc.expectedOsdDownFailureDomains, osdDownFailureDomains)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGetOSDFailureDomainsError(t *testing.T) {
|
|
testcases := []struct {
|
|
name string
|
|
osds []appsv1.Deployment
|
|
expectedAllFailureDomains []string
|
|
expectedDrainingFailureDomains []string
|
|
expectedOsdDownFailureDomains []string
|
|
}{
|
|
{
|
|
name: "case 1: one or more OSD deployment is missing crush location label",
|
|
osds: []appsv1.Deployment{fakeOSDDeployment(1, 1), fakeOSDDeployment(2, 1),
|
|
fakeOSDDeployment(3, 1)},
|
|
expectedAllFailureDomains: nil,
|
|
expectedDrainingFailureDomains: nil,
|
|
expectedOsdDownFailureDomains: nil,
|
|
},
|
|
}
|
|
|
|
for _, tc := range testcases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
osd := tc.osds[0].DeepCopy()
|
|
osd.Spec.Template.ObjectMeta.Labels["topology-location-zone"] = ""
|
|
r := getFakeReconciler(t, cephCluster, &corev1.ConfigMap{},
|
|
tc.osds[1].DeepCopy(), tc.osds[2].DeepCopy(), osd)
|
|
clusterInfo := getFakeClusterInfo()
|
|
clusterInfo.Context = context.TODO()
|
|
request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}}
|
|
allfailureDomains, nodeDrainFailureDomains, osdDownFailureDomains, err := r.getOSDFailureDomains(clusterInfo, request, "zone")
|
|
assert.Error(t, err)
|
|
assert.Equal(t, tc.expectedAllFailureDomains, allfailureDomains)
|
|
assert.Equal(t, tc.expectedDrainingFailureDomains, nodeDrainFailureDomains)
|
|
assert.Equal(t, tc.expectedOsdDownFailureDomains, osdDownFailureDomains)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestReconcilePDBForOSD(t *testing.T) {
|
|
testcases := []struct {
|
|
name string
|
|
fakeCephStatus string
|
|
fakeOSDDump string
|
|
configMap *corev1.ConfigMap
|
|
allFailureDomains []string
|
|
osdDownFailureDomains []string
|
|
activeNodeDrains bool
|
|
pgHealthyRegex string
|
|
expectedSetNoOutValue string
|
|
expectedOSDPDBCount int
|
|
expectedMaxUnavailableCount int
|
|
expectedDrainingFailureDomainName string
|
|
}{
|
|
{
|
|
name: "case 1: no draining failure domain and all pgs are healthy",
|
|
fakeCephStatus: healthyCephStatus,
|
|
fakeOSDDump: `{"OSDs": [{"OSD": 3, "Up": 3, "In": 3}]}`,
|
|
allFailureDomains: []string{"zone-1", "zone-2", "zone-3"},
|
|
osdDownFailureDomains: []string{},
|
|
configMap: fakePDBConfigMap(""),
|
|
activeNodeDrains: false,
|
|
pgHealthyRegex: "",
|
|
expectedSetNoOutValue: "",
|
|
expectedOSDPDBCount: 1,
|
|
expectedMaxUnavailableCount: 1,
|
|
expectedDrainingFailureDomainName: "",
|
|
},
|
|
{
|
|
name: "case 2: zone-1 failure domain is draining and pgs are unhealthy",
|
|
fakeCephStatus: unHealthyCephStatus,
|
|
fakeOSDDump: `{"OSDs": [{"OSD": 3, "Up": 3, "In": 2}]}`,
|
|
allFailureDomains: []string{"zone-1", "zone-2", "zone-3"},
|
|
osdDownFailureDomains: []string{"zone-1"},
|
|
configMap: fakePDBConfigMap(""),
|
|
activeNodeDrains: true,
|
|
pgHealthyRegex: "",
|
|
expectedSetNoOutValue: "true",
|
|
expectedOSDPDBCount: 2,
|
|
expectedMaxUnavailableCount: 0,
|
|
expectedDrainingFailureDomainName: "zone-1",
|
|
},
|
|
{
|
|
name: "case 3: zone-1 is back online. But pgs are still unhealthy from zone-1 drain",
|
|
fakeCephStatus: unHealthyCephStatus,
|
|
fakeOSDDump: `{"OSDs": [{"OSD": 3, "Up": 3, "In": 3}]}`,
|
|
allFailureDomains: []string{"zone-1", "zone-2", "zone-3"},
|
|
osdDownFailureDomains: []string{},
|
|
configMap: fakePDBConfigMap("zone-1"),
|
|
activeNodeDrains: true,
|
|
pgHealthyRegex: "",
|
|
expectedSetNoOutValue: "",
|
|
expectedOSDPDBCount: 2,
|
|
expectedMaxUnavailableCount: 0,
|
|
expectedDrainingFailureDomainName: "zone-1",
|
|
},
|
|
{
|
|
name: "case 4: zone-1 is back online and pgs are also healthy",
|
|
fakeCephStatus: healthyCephStatus,
|
|
fakeOSDDump: `{"OSDs": [{"OSD": 3, "Up": 3, "In": 3}]}`,
|
|
allFailureDomains: []string{"zone-1", "zone-2", "zone-3"},
|
|
osdDownFailureDomains: []string{},
|
|
configMap: fakePDBConfigMap("zone-1"),
|
|
activeNodeDrains: true,
|
|
pgHealthyRegex: "",
|
|
expectedSetNoOutValue: "",
|
|
expectedOSDPDBCount: 1,
|
|
expectedMaxUnavailableCount: 1,
|
|
expectedDrainingFailureDomainName: "",
|
|
},
|
|
{
|
|
name: "case 5: remapped pgs are regarded as clean if pgHealthyRegex allows it",
|
|
fakeCephStatus: healthyCephStatusRemapped,
|
|
fakeOSDDump: `{"OSDs": [{"OSD": 3, "Up": 3, "In": 3}]}`,
|
|
allFailureDomains: []string{"zone-1", "zone-2", "zone-3"},
|
|
osdDownFailureDomains: []string{},
|
|
configMap: fakePDBConfigMap("zone-1"),
|
|
activeNodeDrains: true,
|
|
pgHealthyRegex: `^(active\+clean|active\+clean\+scrubbing|active\+clean\+scrubbing\+deep|active\+clean\+remapped|active\+remapped\+backfilling)$`,
|
|
expectedSetNoOutValue: "",
|
|
expectedOSDPDBCount: 1,
|
|
expectedMaxUnavailableCount: 1,
|
|
expectedDrainingFailureDomainName: "",
|
|
},
|
|
}
|
|
|
|
for _, tc := range testcases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
r := getFakeReconciler(t, cephCluster, tc.configMap)
|
|
clusterInfo := getFakeClusterInfo()
|
|
clusterInfo.Context = context.TODO()
|
|
request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}}
|
|
executor := &exectest.MockExecutor{}
|
|
executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
|
|
logger.Infof("Command: %s %v", command, args)
|
|
if args[0] == "status" {
|
|
return tc.fakeCephStatus, nil
|
|
}
|
|
if args[0] == "osd" && args[1] == "dump" {
|
|
return tc.fakeOSDDump, nil
|
|
}
|
|
return "", errors.Errorf("unexpected ceph command '%v'", args)
|
|
}
|
|
clientset := test.New(t, 3)
|
|
|
|
// check for PDBV1 version
|
|
test.SetFakeKubernetesVersion(clientset, "v1.21.0")
|
|
r.context = &controllerconfig.Context{ClusterdContext: &clusterd.Context{Executor: executor, Clientset: clientset}}
|
|
_, err := r.reconcilePDBsForOSDs(clusterInfo, request, tc.configMap, "zone", tc.allFailureDomains, tc.osdDownFailureDomains, tc.activeNodeDrains, tc.pgHealthyRegex)
|
|
assert.NoError(t, err)
|
|
|
|
// assert that pdb for osd are created correctly
|
|
existingPDBsV1 := &policyv1.PodDisruptionBudgetList{}
|
|
err = r.client.List(context.TODO(), existingPDBsV1)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, tc.expectedOSDPDBCount, len(existingPDBsV1.Items))
|
|
for _, pdb := range existingPDBsV1.Items {
|
|
assert.Equal(t, tc.expectedMaxUnavailableCount, pdb.Spec.MaxUnavailable.IntValue())
|
|
}
|
|
// assert that config map is updated with correct failure domain
|
|
existingConfigMaps := &corev1.ConfigMapList{}
|
|
err = r.client.List(context.TODO(), existingConfigMaps)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, tc.expectedDrainingFailureDomainName, existingConfigMaps.Items[0].Data[drainingFailureDomainKey])
|
|
assert.Equal(t, tc.expectedSetNoOutValue, existingConfigMaps.Items[0].Data[setNoOut])
|
|
})
|
|
|
|
}
|
|
}
|
|
|
|
func TestPGHealthcheckTimeout(t *testing.T) {
|
|
pdbConfig := fakePDBConfigMap("")
|
|
r := getFakeReconciler(t, cephCluster, pdbConfig)
|
|
clusterInfo := getFakeClusterInfo()
|
|
clusterInfo.Context = context.TODO()
|
|
request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}}
|
|
executor := &exectest.MockExecutor{}
|
|
executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
|
|
logger.Infof("Command: %s %v", command, args)
|
|
if args[0] == "status" {
|
|
return unHealthyCephStatus, nil
|
|
}
|
|
if args[0] == "osd" && args[1] == "dump" {
|
|
return `{"OSDs": [{"OSD": 3, "Up": 3, "In": 3}]}`, nil
|
|
}
|
|
return "", errors.Errorf("unexpected ceph command '%v'", args)
|
|
}
|
|
clientset := test.New(t, 3)
|
|
r.context = &controllerconfig.Context{ClusterdContext: &clusterd.Context{Executor: executor, Clientset: clientset}}
|
|
// set PG health check timeout to 10 minutes
|
|
r.pgHealthCheckTimeout = time.Duration(time.Minute * 10)
|
|
|
|
// reconcile OSD PDB with active drains (on zone-1) and unhealthy PGs
|
|
_, err := r.reconcilePDBsForOSDs(clusterInfo, request, pdbConfig, "zone", []string{"zone-1", "zone-2"}, []string{"zone-1"}, true, "")
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "zone-1", pdbConfig.Data[drainingFailureDomainKey])
|
|
assert.Equal(t, "true", pdbConfig.Data[setNoOut])
|
|
|
|
// update the pgHealthCheckDuration time by -9 minutes
|
|
pdbConfig.Data[pgHealthCheckDurationKey] = time.Now().Add(time.Duration(-7) * time.Minute).Format(time.RFC3339)
|
|
// reconcile OSD PDB with no active drains and unhealthy PGs
|
|
_, err = r.reconcilePDBsForOSDs(clusterInfo, request, pdbConfig, "zone", []string{"zone-1", "zone-2"}, []string{}, true, "")
|
|
assert.NoError(t, err)
|
|
// assert that pdb config map was not reset as the PG health check was not timed out
|
|
assert.Equal(t, "zone-1", pdbConfig.Data[drainingFailureDomainKey])
|
|
assert.Equal(t, "true", pdbConfig.Data[setNoOut])
|
|
|
|
// update the drainingFailureDomain time by -9 minutes
|
|
pdbConfig.Data[pgHealthCheckDurationKey] = time.Now().Add(time.Duration(-11) * time.Minute).Format(time.RFC3339)
|
|
// reconcile OSD PDB with no active drains and unhealthy PGs
|
|
_, err = r.reconcilePDBsForOSDs(clusterInfo, request, pdbConfig, "zone", []string{"zone-1", "zone-2"}, []string{}, false, "")
|
|
assert.NoError(t, err)
|
|
// assert that pdb config map was reset as the PG health check was timed out
|
|
assert.Equal(t, "", pdbConfig.Data[drainingFailureDomainKey])
|
|
assert.Equal(t, "", pdbConfig.Data[setNoOut])
|
|
}
|
|
|
|
func TestHasNodeDrained(t *testing.T) {
|
|
osdPOD := fakeOSDPod(0, nodeName)
|
|
ctx := context.TODO()
|
|
// Not expecting node drain because OSD pod is assigned to a schedulable node
|
|
r := getFakeReconciler(t, nodeObj, osdPOD.DeepCopy(), &corev1.ConfigMap{})
|
|
expected, err := hasOSDNodeDrained(ctx, r.client, namespace, "0")
|
|
assert.NoError(t, err)
|
|
assert.False(t, expected)
|
|
|
|
// Expecting node drain because OSD pod is assigned to an unschedulable node
|
|
r = getFakeReconciler(t, unschedulableNodeObj, osdPOD.DeepCopy(), &corev1.ConfigMap{})
|
|
expected, err = hasOSDNodeDrained(ctx, r.client, namespace, "0")
|
|
assert.NoError(t, err)
|
|
assert.True(t, expected)
|
|
|
|
// Expecting node drain because OSD pod is not assigned to any node
|
|
osdPodObj := osdPOD.DeepCopy()
|
|
osdPodObj.Spec.NodeName = ""
|
|
r = getFakeReconciler(t, nodeObj, osdPodObj, &corev1.ConfigMap{})
|
|
expected, err = hasOSDNodeDrained(ctx, r.client, namespace, "0")
|
|
assert.NoError(t, err)
|
|
assert.True(t, expected)
|
|
}
|
|
|
|
func TestGetAllowedDisruptions(t *testing.T) {
|
|
r := getFakeReconciler(t)
|
|
clientset := test.New(t, 3)
|
|
test.SetFakeKubernetesVersion(clientset, "v1.21.0")
|
|
r.context = &controllerconfig.Context{ClusterdContext: &clusterd.Context{Clientset: clientset}}
|
|
|
|
// Default PDB is not available
|
|
allowedDisruptions, err := r.getAllowedDisruptions(osdPDBAppName, namespace)
|
|
assert.Error(t, err)
|
|
assert.Equal(t, int32(-1), allowedDisruptions)
|
|
|
|
// Default PDB is available
|
|
pdb := &policyv1.PodDisruptionBudget{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: osdPDBAppName,
|
|
Namespace: namespace,
|
|
},
|
|
Status: policyv1.PodDisruptionBudgetStatus{
|
|
DisruptionsAllowed: int32(0),
|
|
},
|
|
}
|
|
err = r.client.Create(context.TODO(), pdb)
|
|
assert.NoError(t, err)
|
|
allowedDisruptions, err = r.getAllowedDisruptions(osdPDBAppName, namespace)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, int32(0), allowedDisruptions)
|
|
}
|