forked from rook/rook
Oded Viner
f095a530c8
the .nvmeof pool was incorrectly tagged with the rbd application and initialized as an rbd pool. add .nvmeof to the built-in pool switch so it gets the nvmeof-meta application tag instead, and skip rbd pool init. Signed-off-by: Oded Viner <oviner@redhat.com>
1444 lines
48 KiB
Go
1444 lines
48 KiB
Go
/*
|
|
Copyright 2016 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package pool
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/coreos/pkg/capnslog"
|
|
"github.com/pkg/errors"
|
|
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
|
rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
|
|
"github.com/rook/rook/pkg/clusterd"
|
|
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
|
"github.com/rook/rook/pkg/operator/ceph/config"
|
|
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
|
"github.com/rook/rook/pkg/operator/k8sutil"
|
|
testop "github.com/rook/rook/pkg/operator/test"
|
|
exectest "github.com/rook/rook/pkg/util/exec/test"
|
|
"github.com/stretchr/testify/assert"
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
v1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/client-go/kubernetes/scheme"
|
|
"k8s.io/client-go/tools/events"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
|
)
|
|
|
|
func TestCreatePool(t *testing.T) {
|
|
p := &cephv1.NamedPoolSpec{}
|
|
enabledMetricsApp := false
|
|
enabledMgrApp := false
|
|
enabledNvmeofApp := false
|
|
clusterInfo := cephclient.AdminTestClusterInfo("mycluster")
|
|
executor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) {
|
|
logger.Infof("CommandTimeout: %s %v", command, args)
|
|
if command == "rbd" {
|
|
if args[0] == "pool" && args[1] == "init" {
|
|
// assert that `rbd pool init` is only run when application is set to `rbd`
|
|
assert.Equal(t, "rbd", p.Application)
|
|
assert.Equal(t, p.Name, args[2])
|
|
return "{}", nil
|
|
}
|
|
}
|
|
return "", nil
|
|
},
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
logger.Infof("Command: %s %v", command, args)
|
|
if command == "ceph" {
|
|
if args[1] == "erasure-code-profile" {
|
|
return `{"k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}`, nil
|
|
}
|
|
if args[0] == "osd" && args[1] == "pool" && args[2] == "application" {
|
|
if args[3] == "get" {
|
|
return ``, nil
|
|
}
|
|
assert.Equal(t, "enable", args[3])
|
|
if args[5] != "rbd" {
|
|
switch args[4] {
|
|
case ".mgr":
|
|
enabledMgrApp = true
|
|
assert.Equal(t, "mgr", args[5])
|
|
case ".nvmeof":
|
|
enabledNvmeofApp = true
|
|
assert.Equal(t, "nvmeof-meta", args[5])
|
|
default:
|
|
fmt.Printf("pool - %v", args)
|
|
assert.Fail(t, fmt.Sprintf("invalid pool %q", args[4]))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if command == "rbd" {
|
|
if args[0] == "mirror" && args[2] == "info" {
|
|
return "{}", nil
|
|
} else if args[0] == "mirror" && args[2] == "disable" {
|
|
return "", nil
|
|
}
|
|
}
|
|
return "", nil
|
|
},
|
|
}
|
|
|
|
context := &clusterd.Context{Executor: executor}
|
|
|
|
clusterSpec := &cephv1.ClusterSpec{Storage: cephv1.StorageScopeSpec{Config: map[string]string{cephclient.CrushRootConfigKey: "cluster-crush-root"}}}
|
|
|
|
t.Run("replicated pool", func(t *testing.T) {
|
|
p.Name = "replicapool"
|
|
p.Replicated.Size = 1
|
|
p.Replicated.RequireSafeReplicaSize = false
|
|
// reset the application name
|
|
p.Application = ""
|
|
err := createPool(context, clusterInfo, clusterSpec, p)
|
|
assert.Nil(t, err)
|
|
assert.False(t, enabledMetricsApp)
|
|
})
|
|
|
|
t.Run("built-in mgr pool", func(t *testing.T) {
|
|
p.Name = ".mgr"
|
|
// reset the application name
|
|
p.Application = ""
|
|
err := createPool(context, clusterInfo, clusterSpec, p)
|
|
assert.Nil(t, err)
|
|
assert.True(t, enabledMgrApp)
|
|
})
|
|
|
|
t.Run("built-in nvmeof pool", func(t *testing.T) {
|
|
p.Name = ".nvmeof"
|
|
p.Replicated.Size = 1
|
|
p.Replicated.RequireSafeReplicaSize = false
|
|
p.Application = ""
|
|
err := createPool(context, clusterInfo, clusterSpec, p)
|
|
assert.Nil(t, err)
|
|
assert.True(t, enabledNvmeofApp)
|
|
})
|
|
|
|
t.Run("ec pool", func(t *testing.T) {
|
|
p.Name = "ecpool"
|
|
p.Replicated.Size = 0
|
|
p.ErasureCoded.CodingChunks = 1
|
|
p.ErasureCoded.DataChunks = 2
|
|
// reset the application name
|
|
p.Application = ""
|
|
err := createPool(context, clusterInfo, clusterSpec, p)
|
|
assert.Nil(t, err)
|
|
})
|
|
}
|
|
|
|
func TestCephPoolName(t *testing.T) {
|
|
t.Run("spec not set", func(t *testing.T) {
|
|
p := cephv1.CephBlockPool{ObjectMeta: metav1.ObjectMeta{Name: "metapool"}}
|
|
name := p.ToNamedPoolSpec().Name
|
|
assert.Equal(t, "metapool", name)
|
|
})
|
|
t.Run("same name already set", func(t *testing.T) {
|
|
p := cephv1.CephBlockPool{ObjectMeta: metav1.ObjectMeta{Name: "metapool"}, Spec: cephv1.NamedBlockPoolSpec{Name: "metapool"}}
|
|
name := p.ToNamedPoolSpec().Name
|
|
assert.Equal(t, "metapool", name)
|
|
})
|
|
t.Run("override mgr", func(t *testing.T) {
|
|
p := cephv1.CephBlockPool{ObjectMeta: metav1.ObjectMeta{Name: "default-mgr"}, Spec: cephv1.NamedBlockPoolSpec{Name: ".mgr"}}
|
|
name := p.ToNamedPoolSpec().Name
|
|
assert.Equal(t, ".mgr", name)
|
|
})
|
|
t.Run("override nfs", func(t *testing.T) {
|
|
p := cephv1.CephBlockPool{ObjectMeta: metav1.ObjectMeta{Name: "default-nfs"}, Spec: cephv1.NamedBlockPoolSpec{Name: ".nfs"}}
|
|
name := p.ToNamedPoolSpec().Name
|
|
assert.Equal(t, ".nfs", name)
|
|
})
|
|
}
|
|
|
|
func TestDeletePool(t *testing.T) {
|
|
failOnDelete := false
|
|
deletedPools := map[string]bool{}
|
|
deletedCrushRules := map[string]bool{}
|
|
clusterInfo := cephclient.AdminTestClusterInfo("mycluster")
|
|
executor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
emptyPool := "{\"images\":{\"count\":0,\"provisioned_bytes\":0,\"snap_count\":0},\"trash\":{\"count\":1,\"provisioned_bytes\":2048,\"snap_count\":0}}"
|
|
p := "{\"images\":{\"count\":1,\"provisioned_bytes\":1024,\"snap_count\":0},\"trash\":{\"count\":1,\"provisioned_bytes\":2048,\"snap_count\":0}}"
|
|
logger.Infof("Command: %s %v", command, args)
|
|
if command == "ceph" && args[1] == "lspools" {
|
|
if deletedPools["mypool"] {
|
|
return `[]`, nil
|
|
}
|
|
return `[{"poolnum":1,"poolname":"mypool"}]`, nil
|
|
} else if command == "ceph" && args[1] == "pool" && args[2] == "get" {
|
|
return `{"pool": "mypool","pool_id": 1,"size":1,"crush_rule":"mypool_zone"}`, nil
|
|
} else if command == "ceph" && args[1] == "pool" && args[2] == "delete" {
|
|
deletedPools[args[3]] = true
|
|
return "", nil
|
|
} else if command == "ceph" && args[1] == "crush" && args[2] == "rule" && args[3] == "rm" {
|
|
deletedCrushRules[args[4]] = true
|
|
return "", nil
|
|
} else if args[0] == "pool" {
|
|
if args[1] == "stats" {
|
|
if !failOnDelete {
|
|
return emptyPool, nil
|
|
}
|
|
return p, nil
|
|
}
|
|
return "", errors.Errorf("rbd: error opening pool %q: (2) No such file or directory", args[3])
|
|
}
|
|
return "", errors.Errorf("unexpected rbd command %q", args)
|
|
},
|
|
}
|
|
context := &clusterd.Context{Executor: executor}
|
|
|
|
// delete a pool that exists
|
|
p := &cephv1.NamedPoolSpec{Name: "mypool"}
|
|
err := deletePool(context, clusterInfo, p)
|
|
assert.NoError(t, err)
|
|
assert.True(t, deletedPools["mypool"])
|
|
assert.True(t, deletedCrushRules["mypool_zone"])
|
|
|
|
// succeed even if the pool doesn't exist
|
|
p = &cephv1.NamedPoolSpec{Name: "otherpool"}
|
|
err = deletePool(context, clusterInfo, p)
|
|
assert.Nil(t, err)
|
|
assert.False(t, deletedPools["otherpool"])
|
|
|
|
// fail if images/snapshosts exist in the pool
|
|
failOnDelete = true
|
|
deletedPools["mypool"] = false
|
|
p = &cephv1.NamedPoolSpec{Name: "mypool"}
|
|
err = deletePool(context, clusterInfo, p)
|
|
assert.Error(t, err)
|
|
assert.False(t, deletedPools["mypool"])
|
|
|
|
// delete an erasure coded pool should also delete the EC profile
|
|
ecProfileDeleted := false
|
|
failOnDelete = false
|
|
ecDeletedPools := map[string]bool{}
|
|
ecExecutor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
emptyPool := "{\"images\":{\"count\":0,\"provisioned_bytes\":0,\"snap_count\":0},\"trash\":{\"count\":1,\"provisioned_bytes\":2048,\"snap_count\":0}}"
|
|
if command == "ceph" && args[1] == "lspools" {
|
|
if ecDeletedPools["ecpool"] {
|
|
return `[]`, nil
|
|
}
|
|
return `[{"poolnum":1,"poolname":"ecpool"}]`, nil
|
|
} else if command == "ceph" && args[1] == "pool" && args[2] == "get" {
|
|
return `{"pool": "ecpool","pool_id": 1,"size":1}`, nil
|
|
} else if command == "ceph" && args[1] == "pool" && args[2] == "delete" {
|
|
ecDeletedPools[args[3]] = true
|
|
return "", nil
|
|
} else if command == "ceph" && args[1] == "erasure-code-profile" && args[2] == "rm" {
|
|
assert.Equal(t, "ecpool_ecprofile", args[3])
|
|
ecProfileDeleted = true
|
|
return "", nil
|
|
} else if args[0] == "pool" {
|
|
if args[1] == "stats" {
|
|
return emptyPool, nil
|
|
}
|
|
return "", errors.Errorf("rbd: error opening pool %q: (2) No such file or directory", args[3])
|
|
}
|
|
return "", errors.Errorf("unexpected rbd command %q", args)
|
|
},
|
|
}
|
|
ecContext := &clusterd.Context{Executor: ecExecutor}
|
|
ecPool := &cephv1.NamedPoolSpec{
|
|
Name: "ecpool",
|
|
PoolSpec: cephv1.PoolSpec{
|
|
ErasureCoded: cephv1.ErasureCodedSpec{
|
|
DataChunks: 2,
|
|
CodingChunks: 1,
|
|
},
|
|
},
|
|
}
|
|
err = deletePool(ecContext, clusterInfo, ecPool)
|
|
assert.NoError(t, err)
|
|
assert.True(t, ecProfileDeleted)
|
|
assert.True(t, ecDeletedPools["ecpool"])
|
|
}
|
|
|
|
// TestCephBlockPoolController runs ReconcileCephBlockPool.Reconcile() against a
|
|
// fake client that tracks a CephBlockPool object.
|
|
func TestCephBlockPoolController(t *testing.T) {
|
|
ctx := context.TODO()
|
|
// Set DEBUG logging
|
|
capnslog.SetGlobalLogLevel(capnslog.DEBUG)
|
|
os.Setenv("ROOK_LOG_LEVEL", "DEBUG")
|
|
var (
|
|
name = "my-pool"
|
|
namespace = "rook-ceph"
|
|
replicas uint = 3
|
|
)
|
|
|
|
// A Pool resource with metadata and spec.
|
|
pool := &cephv1.CephBlockPool{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
Namespace: namespace,
|
|
UID: types.UID("c47cac40-9bee-4d52-823b-ccd803ba5bfe"),
|
|
Finalizers: []string{"cephblockpool.ceph.rook.io"},
|
|
},
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "CephBlockPool",
|
|
},
|
|
Spec: cephv1.NamedBlockPoolSpec{
|
|
PoolSpec: cephv1.PoolSpec{
|
|
Replicated: cephv1.ReplicatedSpec{
|
|
Size: replicas,
|
|
},
|
|
Mirroring: cephv1.MirroringSpec{
|
|
Peers: &cephv1.MirroringPeerSpec{},
|
|
},
|
|
StatusCheck: cephv1.MirrorHealthCheckSpec{
|
|
Mirror: cephv1.HealthCheckSpec{
|
|
Disabled: true,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Status: &cephv1.CephBlockPoolStatus{
|
|
Phase: "",
|
|
},
|
|
}
|
|
|
|
// Objects to track in the fake client.
|
|
object := []runtime.Object{
|
|
pool,
|
|
}
|
|
|
|
executor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
if args[0] == "status" {
|
|
return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_ERR"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil
|
|
}
|
|
|
|
return "", nil
|
|
},
|
|
}
|
|
c := &clusterd.Context{
|
|
Executor: executor,
|
|
Clientset: testop.New(t, 1),
|
|
RookClientset: rookclient.NewSimpleClientset(),
|
|
}
|
|
|
|
// Register operator types with the runtime scheme.
|
|
s := scheme.Scheme
|
|
s.AddKnownTypes(cephv1.SchemeGroupVersion, pool, &cephv1.CephClusterList{})
|
|
|
|
// Create a fake client to mock API calls.
|
|
cl := fake.NewClientBuilder().WithRuntimeObjects(object...).Build()
|
|
|
|
// Create a ReconcileCephBlockPool object with the scheme and fake client.
|
|
r := &ReconcileCephBlockPool{
|
|
client: cl,
|
|
scheme: s,
|
|
context: c,
|
|
blockPoolMirrorContexts: make(map[string]*blockPoolHealth),
|
|
opManagerContext: context.TODO(),
|
|
recorder: events.NewFakeRecorder(50),
|
|
}
|
|
|
|
// Mock request to simulate Reconcile() being called on an event for a
|
|
// watched resource .
|
|
req := reconcile.Request{
|
|
NamespacedName: types.NamespacedName{
|
|
Name: name,
|
|
Namespace: namespace,
|
|
},
|
|
}
|
|
|
|
cephCluster := &cephv1.CephCluster{}
|
|
|
|
t.Run("failure no CephCluster", func(t *testing.T) {
|
|
// Create pool for updateCephBlockPoolStatus()
|
|
_, err := c.RookClientset.CephV1().CephBlockPools(namespace).Create(ctx, pool, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
res, err := r.Reconcile(ctx, req)
|
|
assert.NoError(t, err)
|
|
assert.True(t, res.Requeue)
|
|
})
|
|
|
|
t.Run("failure CephCluster not ready", func(t *testing.T) {
|
|
cephCluster = &cephv1.CephCluster{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: namespace,
|
|
Namespace: namespace,
|
|
},
|
|
Status: cephv1.ClusterStatus{
|
|
Phase: "",
|
|
CephVersion: &cephv1.ClusterVersion{
|
|
Version: "14.2.9-0",
|
|
},
|
|
CephStatus: &cephv1.CephStatus{
|
|
Health: "",
|
|
},
|
|
Cephx: cephv1.ClusterCephxStatus{},
|
|
},
|
|
}
|
|
|
|
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{})
|
|
|
|
// Create CephCluster for updateCephBlockPoolStatus()
|
|
_, err := c.RookClientset.CephV1().CephClusters(namespace).Create(ctx, cephCluster, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
object = append(object, cephCluster)
|
|
// Create a fake client to mock API calls.
|
|
cl = fake.NewClientBuilder().WithRuntimeObjects(object...).Build()
|
|
// Create a ReconcileCephBlockPool object with the scheme and fake client.
|
|
r = &ReconcileCephBlockPool{
|
|
client: cl,
|
|
scheme: s,
|
|
context: c,
|
|
blockPoolMirrorContexts: make(map[string]*blockPoolHealth),
|
|
opManagerContext: context.TODO(),
|
|
recorder: events.NewFakeRecorder(50),
|
|
}
|
|
|
|
res, err := r.Reconcile(ctx, req)
|
|
assert.NoError(t, err)
|
|
assert.True(t, res.Requeue)
|
|
})
|
|
|
|
t.Run("failure CephCluster is ready", func(t *testing.T) {
|
|
cephCluster.Status.Phase = cephv1.ConditionReady
|
|
cephCluster.Status.CephStatus.Health = "HEALTH_OK"
|
|
objects := []runtime.Object{
|
|
pool,
|
|
cephCluster,
|
|
}
|
|
// Create a fake client to mock API calls.
|
|
cl = fake.NewClientBuilder().WithRuntimeObjects(objects...).Build()
|
|
c.Client = cl
|
|
|
|
executor = &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
if args[0] == "status" {
|
|
return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil
|
|
}
|
|
if args[0] == "config" && args[2] == "mgr" && args[3] == "mgr/prometheus/rbd_stats_pools" {
|
|
return "", nil
|
|
}
|
|
if args[0] == "mirror" && args[1] == "pool" && args[2] == "info" {
|
|
return "{}", nil
|
|
}
|
|
return "", nil
|
|
},
|
|
}
|
|
c.Executor = executor
|
|
|
|
// Mock clusterInfo
|
|
secrets := map[string][]byte{
|
|
"fsid": []byte(name),
|
|
"mon-secret": []byte("monsecret"),
|
|
"admin-secret": []byte("adminsecret"),
|
|
}
|
|
secret := &v1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "rook-ceph-mon",
|
|
Namespace: namespace,
|
|
},
|
|
Data: secrets,
|
|
Type: k8sutil.RookType,
|
|
}
|
|
_, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{})
|
|
// Create a ReconcileCephBlockPool object with the scheme and fake client.
|
|
r = &ReconcileCephBlockPool{
|
|
client: cl,
|
|
scheme: s,
|
|
context: c,
|
|
blockPoolMirrorContexts: make(map[string]*blockPoolHealth),
|
|
opManagerContext: context.TODO(),
|
|
recorder: events.NewFakeRecorder(50),
|
|
}
|
|
res, err := r.Reconcile(ctx, req)
|
|
assert.NoError(t, err)
|
|
assert.False(t, res.Requeue)
|
|
|
|
err = r.client.Get(context.TODO(), req.NamespacedName, pool)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, cephv1.ConditionReady, pool.Status.Phase)
|
|
})
|
|
|
|
t.Run("failure no mirror mode", func(t *testing.T) {
|
|
pool.Spec.Mirroring.Enabled = true
|
|
err := r.client.Update(context.TODO(), pool)
|
|
assert.NoError(t, err)
|
|
res, err := r.Reconcile(ctx, req)
|
|
assert.NoError(t, err)
|
|
assert.True(t, res.Requeue)
|
|
})
|
|
|
|
t.Setenv("POD_NAME", "test")
|
|
t.Setenv("POD_NAMESPACE", namespace)
|
|
p := &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test",
|
|
Namespace: namespace,
|
|
OwnerReferences: []metav1.OwnerReference{
|
|
{
|
|
Kind: "ReplicaSet",
|
|
Name: "testReplicaSet",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
// Create fake pod
|
|
_, err := r.context.Clientset.CoreV1().Pods(namespace).Create(context.TODO(), p, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
replicaSet := &appsv1.ReplicaSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "testReplicaSet",
|
|
Namespace: namespace,
|
|
OwnerReferences: []metav1.OwnerReference{
|
|
{
|
|
Kind: "Deployment",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
// Create fake replicaset
|
|
_, err = r.context.Clientset.AppsV1().ReplicaSets(namespace).Create(context.TODO(), replicaSet, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
t.Run("success - mirroring set", func(t *testing.T) {
|
|
executor = &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
if args[0] == "mirror" && args[1] == "pool" && args[2] == "peer" && args[3] == "bootstrap" && args[4] == "create" {
|
|
return `eyJmc2lkIjoiYzZiMDg3ZjItNzgyOS00ZGJiLWJjZmMtNTNkYzM0ZTBiMzVkIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBV1lsWmZVQ1Q2RGhBQVBtVnAwbGtubDA5YVZWS3lyRVV1NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMTExLjEwOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTA6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjEyOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTI6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjExOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTE6Njc4OV0ifQ==`, nil
|
|
}
|
|
if args[0] == "mirror" && args[1] == "pool" && args[2] == "info" {
|
|
return "{}", nil
|
|
}
|
|
return "", nil
|
|
},
|
|
}
|
|
c.Executor = executor
|
|
r = &ReconcileCephBlockPool{
|
|
client: cl,
|
|
scheme: s,
|
|
context: c,
|
|
blockPoolMirrorContexts: make(map[string]*blockPoolHealth),
|
|
opManagerContext: context.TODO(),
|
|
recorder: events.NewFakeRecorder(50),
|
|
}
|
|
|
|
pool.Spec.Mirroring.Mode = "image"
|
|
pool.Spec.Mirroring.Peers.SecretNames = []string{}
|
|
err = r.client.Update(context.TODO(), pool)
|
|
assert.NoError(t, err)
|
|
for i := 0; i < 5; i++ {
|
|
res, err := r.Reconcile(ctx, req)
|
|
assert.NoError(t, err)
|
|
assert.False(t, res.Requeue)
|
|
err = r.client.Get(context.TODO(), req.NamespacedName, pool)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, cephv1.ConditionReady, pool.Status.Phase)
|
|
if _, ok := pool.Status.Info[opcontroller.RBDMirrorBootstrapPeerSecretName]; ok {
|
|
break
|
|
}
|
|
logger.Infof("FIX: trying again to update the mirroring status")
|
|
}
|
|
assert.NotEmpty(t, pool.Status.Info[opcontroller.RBDMirrorBootstrapPeerSecretName], pool.Status.Info)
|
|
|
|
// fetch the secret
|
|
myPeerSecret, err := c.Clientset.CoreV1().Secrets(namespace).Get(ctx, pool.Status.Info[opcontroller.RBDMirrorBootstrapPeerSecretName], metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
if myPeerSecret != nil {
|
|
assert.NotEmpty(t, myPeerSecret.Data["token"], myPeerSecret.Data)
|
|
assert.NotEmpty(t, myPeerSecret.Data["pool"])
|
|
}
|
|
})
|
|
|
|
peerSecretName := "peer-secret"
|
|
t.Run("failure - import peer token but was not created", func(t *testing.T) {
|
|
// Create a fake client to mock API calls.
|
|
cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
|
|
|
|
// Create a ReconcileCephBlockPool object with the scheme and fake client.
|
|
r = &ReconcileCephBlockPool{
|
|
client: cl,
|
|
scheme: s,
|
|
context: c,
|
|
blockPoolMirrorContexts: make(map[string]*blockPoolHealth),
|
|
opManagerContext: context.TODO(),
|
|
recorder: events.NewFakeRecorder(50),
|
|
}
|
|
|
|
pool.Spec.Mirroring.Peers.SecretNames = []string{peerSecretName}
|
|
err := r.client.Update(context.TODO(), pool)
|
|
assert.NoError(t, err)
|
|
res, err := r.Reconcile(ctx, req)
|
|
// assert reconcile failure because peer token secret was not created
|
|
assert.NoError(t, err)
|
|
assert.True(t, res.Requeue)
|
|
})
|
|
|
|
t.Run("success - import peer token but was created", func(t *testing.T) {
|
|
bootstrapPeerToken := `eyJmc2lkIjoiYzZiMDg3ZjItNzgyOS00ZGJiLWJjZmMtNTNkYzM0ZTBiMzVkIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBV1lsWmZVQ1Q2RGhBQVBtVnAwbGtubDA5YVZWS3lyRVV1NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMTExLjEwOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTA6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjEyOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTI6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjExOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTE6Njc4OV0ifQ==` //nolint:gosec // This is just a var name, not a real token
|
|
peerSecret := &v1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: peerSecretName,
|
|
Namespace: namespace,
|
|
},
|
|
Data: map[string][]byte{"token": []byte(bootstrapPeerToken), "pool": []byte("goo")},
|
|
Type: k8sutil.RookType,
|
|
}
|
|
_, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, peerSecret, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
res, err := r.Reconcile(ctx, req)
|
|
assert.NoError(t, err)
|
|
assert.False(t, res.Requeue)
|
|
err = r.client.Get(context.TODO(), req.NamespacedName, pool)
|
|
assert.NoError(t, err)
|
|
})
|
|
|
|
t.Run("failure - mirroring disabled", func(t *testing.T) {
|
|
r = &ReconcileCephBlockPool{
|
|
client: cl,
|
|
scheme: s,
|
|
context: c,
|
|
blockPoolMirrorContexts: make(map[string]*blockPoolHealth),
|
|
opManagerContext: context.TODO(),
|
|
recorder: events.NewFakeRecorder(50),
|
|
}
|
|
pool.Spec.Mirroring.Enabled = false
|
|
pool.Spec.Mirroring.Mode = "image"
|
|
err := r.client.Update(context.TODO(), pool)
|
|
assert.NoError(t, err)
|
|
res, err := r.Reconcile(ctx, req)
|
|
assert.NoError(t, err)
|
|
assert.False(t, res.Requeue)
|
|
err = r.client.Get(context.TODO(), req.NamespacedName, pool)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, cephv1.ConditionReady, pool.Status.Phase)
|
|
assert.Equal(t, pool.Status.MirroringStatus, &cephv1.MirroringStatusSpec{})
|
|
})
|
|
}
|
|
|
|
func TestDeletionBlocked(t *testing.T) {
|
|
// A Pool resource with metadata and spec.
|
|
pool := &cephv1.CephBlockPool{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-pool",
|
|
Namespace: "ns",
|
|
},
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "CephBlockPool",
|
|
},
|
|
Status: &cephv1.CephBlockPoolStatus{},
|
|
}
|
|
object := []runtime.Object{pool}
|
|
// Register operator types with the runtime scheme.
|
|
s := scheme.Scheme
|
|
s.AddKnownTypes(cephv1.SchemeGroupVersion,
|
|
&cephv1.CephBlockPoolList{},
|
|
&cephv1.CephBlockPoolRadosNamespaceList{},
|
|
&cephv1.CephBlockPool{},
|
|
&cephv1.CephBlockPoolRadosNamespace{},
|
|
)
|
|
blockImageCount := 0
|
|
rnsImageCount := 0
|
|
poolPresent := false
|
|
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
|
|
executor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
logger.Infof("Command: %s %v", command, args)
|
|
if args[0] == "pool" {
|
|
if args[1] == "stats" {
|
|
response := "{\"images\":{\"count\":%d,\"provisioned_bytes\":0,\"snap_count\":0},\"trash\":{\"count\":1,\"provisioned_bytes\":2048,\"snap_count\":0}}"
|
|
if args[4] == "--namespace" {
|
|
return fmt.Sprintf(response, rnsImageCount), nil
|
|
} else {
|
|
return fmt.Sprintf(response, blockImageCount), nil
|
|
}
|
|
}
|
|
} else if command == "ceph" && args[0] == "osd" {
|
|
if args[1] == "lspools" {
|
|
if poolPresent {
|
|
return fmt.Sprintf(`[{"poolnum":1,"poolname":"%s"}]`, pool.Name), nil
|
|
} else {
|
|
return "[]", nil
|
|
}
|
|
}
|
|
}
|
|
return "not implemented", nil
|
|
},
|
|
}
|
|
c := &clusterd.Context{
|
|
Executor: executor,
|
|
Clientset: testop.New(t, 1),
|
|
RookClientset: rookclient.NewSimpleClientset(),
|
|
}
|
|
// Create a ReconcileCephBlockPool object with the scheme and fake client.
|
|
r := &ReconcileCephBlockPool{
|
|
client: cl,
|
|
scheme: s,
|
|
context: c,
|
|
blockPoolMirrorContexts: make(map[string]*blockPoolHealth),
|
|
opManagerContext: context.TODO(),
|
|
recorder: events.NewFakeRecorder(50),
|
|
clusterInfo: cephclient.AdminTestClusterInfo("mycluster"),
|
|
}
|
|
cephCluster := &cephv1.CephCluster{}
|
|
t.Run("deletion is allowed with no pool actually present", func(t *testing.T) {
|
|
err := r.handleDeletionBlocked(pool, cephCluster)
|
|
assert.NoError(t, err)
|
|
|
|
result := &cephv1.CephBlockPool{}
|
|
err = r.client.Get(context.TODO(), types.NamespacedName{Name: pool.Name, Namespace: pool.Namespace}, result)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, pool.Name, result.Name)
|
|
assert.Equal(t, pool.Namespace, result.Namespace)
|
|
assert.Equal(t, 2, len(pool.Status.Conditions))
|
|
assert.Equal(t, "PoolDeletionIsBlocked", string(pool.Status.Conditions[0].Type))
|
|
assert.Equal(t, v1.ConditionFalse, pool.Status.Conditions[0].Status)
|
|
assert.Equal(t, "PoolEmpty", string(pool.Status.Conditions[0].Reason))
|
|
assert.Equal(t, "DeletionIsBlocked", string(pool.Status.Conditions[1].Type))
|
|
assert.Equal(t, v1.ConditionFalse, pool.Status.Conditions[1].Status)
|
|
assert.Equal(t, "ObjectHasNoDependents", string(pool.Status.Conditions[1].Reason))
|
|
})
|
|
poolPresent = true
|
|
t.Run("deletion is allowed with no images or rns", func(t *testing.T) {
|
|
err := r.handleDeletionBlocked(pool, cephCluster)
|
|
assert.NoError(t, err)
|
|
|
|
result := &cephv1.CephBlockPool{}
|
|
err = r.client.Get(context.TODO(), types.NamespacedName{Name: pool.Name, Namespace: pool.Namespace}, result)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, pool.Name, result.Name)
|
|
assert.Equal(t, pool.Namespace, result.Namespace)
|
|
assert.Equal(t, 2, len(pool.Status.Conditions))
|
|
assert.Equal(t, "PoolDeletionIsBlocked", string(pool.Status.Conditions[0].Type))
|
|
assert.Equal(t, v1.ConditionFalse, pool.Status.Conditions[0].Status)
|
|
assert.Equal(t, "PoolEmpty", string(pool.Status.Conditions[0].Reason))
|
|
assert.Equal(t, "DeletionIsBlocked", string(pool.Status.Conditions[1].Type))
|
|
assert.Equal(t, v1.ConditionFalse, pool.Status.Conditions[1].Status)
|
|
assert.Equal(t, "ObjectHasNoDependents", string(pool.Status.Conditions[1].Reason))
|
|
})
|
|
t.Run("image prevents deletion", func(t *testing.T) {
|
|
blockImageCount = 1
|
|
err := r.handleDeletionBlocked(pool, cephCluster)
|
|
assert.Error(t, err)
|
|
|
|
result := &cephv1.CephBlockPool{}
|
|
err = r.client.Get(context.TODO(), types.NamespacedName{Name: pool.Name, Namespace: pool.Namespace}, result)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, pool.Name, result.Name)
|
|
assert.Equal(t, pool.Namespace, result.Namespace)
|
|
assert.Equal(t, 2, len(pool.Status.Conditions))
|
|
assert.Equal(t, "PoolDeletionIsBlocked", string(pool.Status.Conditions[0].Type))
|
|
assert.Equal(t, v1.ConditionTrue, pool.Status.Conditions[0].Status)
|
|
assert.Equal(t, "PoolNotEmpty", string(pool.Status.Conditions[0].Reason))
|
|
assert.Equal(t, "DeletionIsBlocked", string(pool.Status.Conditions[1].Type))
|
|
assert.Equal(t, v1.ConditionFalse, pool.Status.Conditions[1].Status)
|
|
assert.Equal(t, "ObjectHasNoDependents", string(pool.Status.Conditions[1].Reason))
|
|
})
|
|
t.Run("rados namespace prevents deletion", func(t *testing.T) {
|
|
radosNamespace := &cephv1.CephBlockPoolRadosNamespace{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-rns",
|
|
Namespace: pool.Namespace,
|
|
},
|
|
Spec: cephv1.CephBlockPoolRadosNamespaceSpec{
|
|
BlockPoolName: pool.Name,
|
|
},
|
|
}
|
|
_, err := c.RookClientset.CephV1().CephBlockPoolRadosNamespaces(pool.Namespace).Create(context.TODO(), radosNamespace, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
// A radosnamespaces prevents deletion of the pool
|
|
blockImageCount = 0
|
|
rnsImageCount = 1
|
|
err = r.handleDeletionBlocked(pool, cephCluster)
|
|
assert.Error(t, err)
|
|
|
|
result := &cephv1.CephBlockPool{}
|
|
err = r.client.Get(context.TODO(), types.NamespacedName{Name: pool.Name, Namespace: pool.Namespace}, result)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, pool.Name, result.Name)
|
|
assert.Equal(t, pool.Namespace, result.Namespace)
|
|
assert.Equal(t, 2, len(pool.Status.Conditions))
|
|
assert.Equal(t, "PoolDeletionIsBlocked", string(pool.Status.Conditions[0].Type))
|
|
assert.Equal(t, v1.ConditionTrue, pool.Status.Conditions[0].Status)
|
|
assert.Equal(t, "PoolNotEmpty", string(pool.Status.Conditions[0].Reason))
|
|
assert.Equal(t, "DeletionIsBlocked", string(pool.Status.Conditions[1].Type))
|
|
assert.Equal(t, v1.ConditionTrue, pool.Status.Conditions[1].Status)
|
|
assert.Equal(t, "ObjectHasDependents", string(pool.Status.Conditions[1].Reason))
|
|
})
|
|
}
|
|
|
|
func TestIsAnyRadosNamespaceMirrored(t *testing.T) {
|
|
pool := "test"
|
|
object := []runtime.Object{}
|
|
// Register operator types with the runtime scheme.
|
|
s := scheme.Scheme
|
|
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
|
|
executor := &exectest.MockExecutor{}
|
|
c := &clusterd.Context{
|
|
Executor: executor,
|
|
Clientset: testop.New(t, 1),
|
|
RookClientset: rookclient.NewSimpleClientset(),
|
|
}
|
|
// Create a ReconcileCephBlockPool object with the scheme and fake client.
|
|
r := &ReconcileCephBlockPool{
|
|
client: cl,
|
|
scheme: s,
|
|
context: c,
|
|
blockPoolMirrorContexts: make(map[string]*blockPoolHealth),
|
|
opManagerContext: context.TODO(),
|
|
recorder: events.NewFakeRecorder(50),
|
|
clusterInfo: cephclient.AdminTestClusterInfo("mycluster"),
|
|
}
|
|
|
|
t.Run("rados namespace mirroring enabled", func(t *testing.T) {
|
|
r.context.Executor = &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
if args[0] == "namespace" {
|
|
assert.Equal(t, pool, args[2])
|
|
return `[{"name":"abc"},{"name":"abc1"},{"name":"abc3"}]`, nil
|
|
}
|
|
if args[0] == "mirror" && args[1] == "pool" && args[2] == "info" {
|
|
return "{}", nil
|
|
}
|
|
return "", nil
|
|
},
|
|
}
|
|
enabled, err := r.isAnyRadosNamespaceMirrored(pool)
|
|
assert.NoError(t, err)
|
|
assert.True(t, enabled)
|
|
})
|
|
|
|
t.Run("rados namespace mirroring disabled", func(t *testing.T) {
|
|
r.context.Executor = &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
if args[0] == "namespace" {
|
|
assert.Equal(t, pool, args[2])
|
|
return `[]`, nil
|
|
}
|
|
if args[0] == "mirror" && args[1] == "pool" && args[2] == "info" {
|
|
return "{}", nil
|
|
}
|
|
return "", nil
|
|
},
|
|
}
|
|
enabled, err := r.isAnyRadosNamespaceMirrored(pool)
|
|
assert.NoError(t, err)
|
|
assert.False(t, enabled)
|
|
})
|
|
}
|
|
|
|
func TestConfigureRBDStats(t *testing.T) {
|
|
var (
|
|
s = runtime.NewScheme()
|
|
context = &clusterd.Context{}
|
|
namespace = "rook-ceph"
|
|
)
|
|
|
|
mockedPools := ""
|
|
executor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) {
|
|
logger.Infof("Command: %s %v", command, args)
|
|
if args[0] == "config" && args[2] == "mgr" && args[3] == "mgr/prometheus/rbd_stats_pools" {
|
|
if args[1] == "set" {
|
|
mockedPools = args[4]
|
|
return "", nil
|
|
}
|
|
if args[1] == "get" {
|
|
return mockedPools, nil
|
|
}
|
|
if args[1] == "rm" {
|
|
mockedPools = ""
|
|
return "", nil
|
|
}
|
|
}
|
|
return "", errors.Errorf("unexpected arguments %q", args)
|
|
},
|
|
}
|
|
|
|
context.Executor = executor
|
|
context.Client = fake.NewClientBuilder().WithScheme(s).Build()
|
|
|
|
clusterInfo := cephclient.AdminTestClusterInfo(namespace)
|
|
|
|
// Case 1: CephBlockPoolList is not registered in scheme.
|
|
// So, an error is expected as List() operation would fail.
|
|
err := configureRBDStats(context, clusterInfo, "")
|
|
assert.NotNil(t, err)
|
|
|
|
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{})
|
|
// Case 2: CephBlockPoolList is registered in schema.
|
|
// So, no error is expected.
|
|
err = configureRBDStats(context, clusterInfo, "")
|
|
assert.Nil(t, err)
|
|
|
|
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPool{})
|
|
// A Pool resource with metadata and spec.
|
|
poolWithRBDStatsDisabled := &cephv1.CephBlockPool{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "my-pool-without-rbd-stats",
|
|
Namespace: namespace,
|
|
},
|
|
Spec: cephv1.NamedBlockPoolSpec{
|
|
PoolSpec: cephv1.PoolSpec{
|
|
Replicated: cephv1.ReplicatedSpec{
|
|
Size: 3,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
// Case 3: One CephBlockPool with EnableRBDStats:false (default).
|
|
objects := []runtime.Object{
|
|
poolWithRBDStatsDisabled,
|
|
}
|
|
context.Client = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build()
|
|
err = configureRBDStats(context, clusterInfo, "")
|
|
assert.Nil(t, err)
|
|
|
|
// Case 4: Two CephBlockPools with EnableRBDStats:false & EnableRBDStats:true.
|
|
poolWithRBDStatsEnabled := poolWithRBDStatsDisabled.DeepCopy()
|
|
poolWithRBDStatsEnabled.Name = "my-pool-with-rbd-stats"
|
|
poolWithRBDStatsEnabled.Spec.EnableRBDStats = true
|
|
objects = append(objects, poolWithRBDStatsEnabled)
|
|
context.Client = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build()
|
|
err = configureRBDStats(context, clusterInfo, "")
|
|
assert.Nil(t, err)
|
|
|
|
// Case 5: Two external pools(non CephBlockPools) with rbd_stats_pools config already set
|
|
monStore := config.GetMonStore(context, clusterInfo)
|
|
e := monStore.Set("mgr", "mgr/prometheus/rbd_stats_pools", "pool1,pool2")
|
|
assert.Nil(t, e)
|
|
e = configureRBDStats(context, clusterInfo, "")
|
|
assert.Nil(t, e)
|
|
|
|
rbdStatsPools, err := monStore.Get("mgr", "mgr/prometheus/rbd_stats_pools")
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, "my-pool-with-rbd-stats,pool1,pool2", rbdStatsPools)
|
|
|
|
// Case 6: Deleted CephBlockPool should be excluded from config
|
|
err = configureRBDStats(context, clusterInfo, "my-pool-with-rbd-stats")
|
|
assert.Nil(t, err)
|
|
|
|
rbdStatsPools, err = monStore.Get("mgr", "mgr/prometheus/rbd_stats_pools")
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, "pool1,pool2", rbdStatsPools)
|
|
|
|
// Case 7: Duplicate entries should be removed from config
|
|
e = monStore.Set("mgr", "mgr/prometheus/rbd_stats_pools", "pool1,pool2,pool1")
|
|
assert.Nil(t, e)
|
|
err = configureRBDStats(context, clusterInfo, "")
|
|
assert.Nil(t, err)
|
|
|
|
rbdStatsPools, err = monStore.Get("mgr", "mgr/prometheus/rbd_stats_pools")
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, "my-pool-with-rbd-stats,pool1,pool2", rbdStatsPools)
|
|
|
|
// Case 8: Two CephBlockPools with EnableRBDStats:false & EnableRBDStats:true.
|
|
// SetConfig returns an error
|
|
context.Executor = &exectest.MockExecutor{
|
|
MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) {
|
|
logger.Infof("Command: %s %v", command, args)
|
|
return "", errors.New("mock error to simulate failure of mon store Set() function")
|
|
},
|
|
}
|
|
err = configureRBDStats(context, clusterInfo, "")
|
|
assert.NotNil(t, err)
|
|
}
|
|
|
|
func TestGenerateStatsPoolList(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
existingStatsPools []string
|
|
rookStatsPools []string
|
|
removePools []string
|
|
expectedOutput string
|
|
}{
|
|
// Basic cases
|
|
{
|
|
name: "Empty lists",
|
|
existingStatsPools: []string{},
|
|
rookStatsPools: []string{},
|
|
removePools: []string{},
|
|
expectedOutput: "",
|
|
},
|
|
{
|
|
name: "Single-item lists, no removal",
|
|
existingStatsPools: []string{"p1"},
|
|
rookStatsPools: []string{"p2"},
|
|
removePools: []string{},
|
|
expectedOutput: "p1,p2",
|
|
},
|
|
// Overlap and duplicates
|
|
{
|
|
name: "Overlapping pools, some to remove",
|
|
existingStatsPools: []string{"p1", "p2", "p3"},
|
|
rookStatsPools: []string{"p2", "p4", "p5"},
|
|
removePools: []string{"p1", "p5"},
|
|
expectedOutput: "p2,p3,p4",
|
|
},
|
|
{
|
|
name: "Non-overlapping lists",
|
|
existingStatsPools: []string{"p1", "p2"},
|
|
rookStatsPools: []string{"p3", "p4"},
|
|
removePools: []string{},
|
|
expectedOutput: "p1,p2,p3,p4",
|
|
},
|
|
// All pools removed
|
|
{
|
|
name: "All pools removed",
|
|
existingStatsPools: []string{"p1", "p2"},
|
|
rookStatsPools: []string{"p2", "p3"},
|
|
removePools: []string{"p1", "p2", "p3"},
|
|
expectedOutput: "",
|
|
},
|
|
// Mixed scenarios with edge cases
|
|
{
|
|
name: "Only removed pools",
|
|
existingStatsPools: []string{"p1", "p2"},
|
|
rookStatsPools: []string{"p2", "p3"},
|
|
removePools: []string{"p1", "p2", "p3", "p4"},
|
|
expectedOutput: "",
|
|
},
|
|
{
|
|
name: "Duplicate pools across lists",
|
|
existingStatsPools: []string{"p1", "p2", "p1"},
|
|
rookStatsPools: []string{"p2", "p3", "p3"},
|
|
removePools: []string{},
|
|
expectedOutput: "p1,p2,p3",
|
|
},
|
|
{
|
|
name: "Empty string in pools",
|
|
existingStatsPools: []string{"p1", ""},
|
|
rookStatsPools: []string{"p2", ""},
|
|
removePools: []string{""},
|
|
expectedOutput: "p1,p2",
|
|
},
|
|
{
|
|
name: "Empty string in remove pools",
|
|
existingStatsPools: []string{"p1", "p2"},
|
|
rookStatsPools: []string{"p3", "p4"},
|
|
removePools: []string{""},
|
|
expectedOutput: "p1,p2,p3,p4",
|
|
},
|
|
{
|
|
name: "All lists empty strings",
|
|
existingStatsPools: []string{""},
|
|
rookStatsPools: []string{""},
|
|
removePools: []string{""},
|
|
expectedOutput: "",
|
|
},
|
|
// Larger cases
|
|
{
|
|
name: "Large unique pool list",
|
|
existingStatsPools: []string{"p1", "p2", "p3", "p4"},
|
|
rookStatsPools: []string{"p5", "p6", "p7", "p8"},
|
|
removePools: []string{"p1", "p8"},
|
|
expectedOutput: "p2,p3,p4,p5,p6,p7",
|
|
},
|
|
{
|
|
name: "Large list with many duplicates",
|
|
existingStatsPools: []string{"p1", "p2", "p3", "p2", "p1"},
|
|
rookStatsPools: []string{"p2", "p3", "p3", "p4", "p1"},
|
|
removePools: []string{"p4"},
|
|
expectedOutput: "p1,p2,p3",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
result := generateStatsPoolList(tt.existingStatsPools, tt.rookStatsPools, tt.removePools)
|
|
assert.Equal(t, tt.expectedOutput, result)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMirrorPeerKeyRotationStatus(t *testing.T) {
|
|
ctx := context.TODO()
|
|
// Set DEBUG logging
|
|
t.Setenv("ROOK_LOG_LEVEL", "DEBUG")
|
|
var (
|
|
name = "my-pool"
|
|
namespace = "rook-ceph"
|
|
replicas uint = 3
|
|
)
|
|
|
|
cephCluster := &cephv1.CephCluster{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: namespace,
|
|
Namespace: namespace,
|
|
},
|
|
Status: cephv1.ClusterStatus{
|
|
Phase: "",
|
|
CephVersion: &cephv1.ClusterVersion{
|
|
Version: "20.2.0-0",
|
|
},
|
|
CephStatus: &cephv1.CephStatus{
|
|
Health: "HEALTH_OK",
|
|
},
|
|
Cephx: cephv1.ClusterCephxStatus{},
|
|
},
|
|
}
|
|
|
|
// A Pool resource with metadata and spec.
|
|
pool := &cephv1.CephBlockPool{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
Namespace: namespace,
|
|
UID: types.UID("c47cac40-9bee-4d52-823b-ccd803ba5bfe"),
|
|
Finalizers: []string{"cephblockpool.ceph.rook.io"},
|
|
},
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "CephBlockPool",
|
|
},
|
|
Spec: cephv1.NamedBlockPoolSpec{
|
|
PoolSpec: cephv1.PoolSpec{
|
|
Replicated: cephv1.ReplicatedSpec{
|
|
Size: replicas,
|
|
},
|
|
Mirroring: cephv1.MirroringSpec{
|
|
Enabled: true,
|
|
Mode: "image",
|
|
Peers: &cephv1.MirroringPeerSpec{},
|
|
},
|
|
StatusCheck: cephv1.MirrorHealthCheckSpec{
|
|
Mirror: cephv1.HealthCheckSpec{
|
|
Disabled: true,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Status: &cephv1.CephBlockPoolStatus{
|
|
Phase: "",
|
|
},
|
|
}
|
|
|
|
// Objects to track in the fake client.
|
|
object := []runtime.Object{
|
|
pool, cephCluster,
|
|
}
|
|
|
|
executor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
if args[0] == "status" {
|
|
return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil
|
|
}
|
|
if args[0] == "mirror" && args[1] == "pool" && args[2] == "peer" && args[3] == "bootstrap" && args[4] == "create" {
|
|
return `eyJmc2lkIjoiYzZiMDg3ZjItNzgyOS00ZGJiLWJjZmMtNTNkYzM0ZTBiMzVkIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBV1lsWmZVQ1Q2RGhBQVBtVnAwbGtubDA5YVZWS3lyRVV1NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMTExLjEwOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTA6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjEyOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTI6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjExOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTE6Njc4OV0ifQ==`, nil
|
|
}
|
|
if args[0] == "mirror" && args[1] == "pool" && args[2] == "info" {
|
|
return "{}", nil
|
|
}
|
|
return "", nil
|
|
},
|
|
}
|
|
|
|
// Register operator types with the runtime scheme.
|
|
s := scheme.Scheme
|
|
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPool{})
|
|
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{})
|
|
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{})
|
|
|
|
// Create a fake client to mock API calls.
|
|
cl := fake.NewClientBuilder().WithRuntimeObjects(object...).Build()
|
|
|
|
c := &clusterd.Context{
|
|
Client: cl,
|
|
Executor: executor,
|
|
Clientset: testop.New(t, 1),
|
|
RookClientset: rookclient.NewSimpleClientset(),
|
|
}
|
|
|
|
// Mock clusterInfo
|
|
secrets := map[string][]byte{
|
|
"fsid": []byte(name),
|
|
"mon-secret": []byte("monsecret"),
|
|
"admin-secret": []byte("adminsecret"),
|
|
}
|
|
secret := &v1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "rook-ceph-mon",
|
|
Namespace: namespace,
|
|
},
|
|
Data: secrets,
|
|
Type: k8sutil.RookType,
|
|
}
|
|
_, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
// Mock request to simulate Reconcile() being called on an event for a
|
|
// watched resource .
|
|
req := reconcile.Request{
|
|
NamespacedName: types.NamespacedName{
|
|
Name: name,
|
|
Namespace: namespace,
|
|
},
|
|
}
|
|
|
|
// Create a ReconcileCephBlockPool object with the scheme and fake client.
|
|
r := &ReconcileCephBlockPool{
|
|
client: cl,
|
|
scheme: s,
|
|
context: c,
|
|
blockPoolMirrorContexts: make(map[string]*blockPoolHealth),
|
|
opManagerContext: context.TODO(),
|
|
recorder: events.NewFakeRecorder(50),
|
|
}
|
|
|
|
t.Setenv("POD_NAME", "test")
|
|
t.Setenv("POD_NAMESPACE", namespace)
|
|
p := &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test",
|
|
Namespace: namespace,
|
|
OwnerReferences: []metav1.OwnerReference{
|
|
{
|
|
Kind: "ReplicaSet",
|
|
Name: "testReplicaSet",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
// Create fake pod
|
|
_, err = r.context.Clientset.CoreV1().Pods(namespace).Create(context.TODO(), p, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
replicaSet := &appsv1.ReplicaSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "testReplicaSet",
|
|
Namespace: namespace,
|
|
OwnerReferences: []metav1.OwnerReference{
|
|
{
|
|
Kind: "Deployment",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
// Create fake replicaset
|
|
_, err = r.context.Clientset.AppsV1().ReplicaSets(namespace).Create(context.TODO(), replicaSet, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
t.Run("first reconcile - PeerToken cephx status is set to empty", func(t *testing.T) {
|
|
_, err := r.Reconcile(ctx, req)
|
|
assert.NoError(t, err)
|
|
pool := &cephv1.CephBlockPool{}
|
|
err = r.client.Get(context.TODO(), req.NamespacedName, pool)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, cephv1.CephxStatus{}, pool.Status.Cephx.PeerToken)
|
|
})
|
|
|
|
t.Run("subsequent reconcile - retain PeerToken cephx status", func(t *testing.T) {
|
|
_, err := r.Reconcile(ctx, req)
|
|
assert.NoError(t, err)
|
|
pool := &cephv1.CephBlockPool{}
|
|
err = r.client.Get(context.TODO(), req.NamespacedName, pool)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, cephv1.CephxStatus{}, pool.Status.Cephx.PeerToken)
|
|
})
|
|
|
|
t.Run("PeerToken cephx status should be updated based on the cephCluster cephx status", func(t *testing.T) {
|
|
RDBMirrorPeerStatus := cephv1.CephxStatus{
|
|
KeyGeneration: 2,
|
|
KeyCephVersion: "20.2.0-0",
|
|
}
|
|
|
|
cephCluster.Status.Cephx.RBDMirrorPeer = RDBMirrorPeerStatus
|
|
err := r.client.Update(context.TODO(), cephCluster)
|
|
assert.NoError(t, err)
|
|
_, err = r.Reconcile(ctx, req)
|
|
assert.NoError(t, err)
|
|
pool := &cephv1.CephBlockPool{}
|
|
err = r.client.Get(context.TODO(), req.NamespacedName, pool)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, RDBMirrorPeerStatus, pool.Status.Cephx.PeerToken)
|
|
})
|
|
}
|
|
|
|
func TestCephBlockPoolControllerPoolReachesReady(t *testing.T) {
|
|
namespace := "rook-ceph"
|
|
|
|
tests := []struct {
|
|
name string
|
|
poolName string
|
|
poolSpec cephv1.PoolSpec
|
|
expectedType string
|
|
}{
|
|
{
|
|
name: "erasure coded pool",
|
|
poolName: "ec-pool",
|
|
poolSpec: cephv1.PoolSpec{
|
|
ErasureCoded: cephv1.ErasureCodedSpec{
|
|
DataChunks: 2,
|
|
CodingChunks: 1,
|
|
},
|
|
},
|
|
expectedType: "Erasure Coded",
|
|
},
|
|
{
|
|
name: "replicated pool",
|
|
poolName: "rep-pool",
|
|
poolSpec: cephv1.PoolSpec{
|
|
Replicated: cephv1.ReplicatedSpec{
|
|
Size: 3,
|
|
RequireSafeReplicaSize: true,
|
|
},
|
|
},
|
|
expectedType: "Replicated",
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
ctx := context.TODO()
|
|
capnslog.SetGlobalLogLevel(capnslog.DEBUG)
|
|
os.Setenv("ROOK_LOG_LEVEL", "DEBUG")
|
|
|
|
pool := &cephv1.CephBlockPool{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: tc.poolName,
|
|
Namespace: namespace,
|
|
UID: types.UID("c47cac40-9bee-4d52-823b-ccd803ba5bfe"),
|
|
Finalizers: []string{"cephblockpool.ceph.rook.io"},
|
|
},
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "CephBlockPool",
|
|
},
|
|
Spec: cephv1.NamedBlockPoolSpec{
|
|
PoolSpec: tc.poolSpec,
|
|
},
|
|
Status: &cephv1.CephBlockPoolStatus{
|
|
Phase: "",
|
|
},
|
|
}
|
|
|
|
cephCluster := &cephv1.CephCluster{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: namespace,
|
|
Namespace: namespace,
|
|
},
|
|
Status: cephv1.ClusterStatus{
|
|
Phase: cephv1.ConditionReady,
|
|
CephVersion: &cephv1.ClusterVersion{
|
|
Version: "14.2.9-0",
|
|
},
|
|
CephStatus: &cephv1.CephStatus{
|
|
Health: "HEALTH_OK",
|
|
},
|
|
Cephx: cephv1.ClusterCephxStatus{},
|
|
},
|
|
}
|
|
|
|
objects := []runtime.Object{pool, cephCluster}
|
|
|
|
executor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
if args[0] == "status" {
|
|
return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil
|
|
}
|
|
if args[0] == "config" && args[2] == "mgr" && args[3] == "mgr/prometheus/rbd_stats_pools" {
|
|
return "", nil
|
|
}
|
|
if command == "ceph" && args[1] == "erasure-code-profile" {
|
|
return `{"k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}`, nil
|
|
}
|
|
return "", nil
|
|
},
|
|
}
|
|
|
|
s := scheme.Scheme
|
|
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPool{}, &cephv1.CephCluster{}, &cephv1.CephClusterList{}, &cephv1.CephBlockPoolList{})
|
|
|
|
cl := fake.NewClientBuilder().WithRuntimeObjects(objects...).Build()
|
|
|
|
c := &clusterd.Context{
|
|
Executor: executor,
|
|
Client: cl,
|
|
Clientset: testop.New(t, 1),
|
|
RookClientset: rookclient.NewSimpleClientset(),
|
|
}
|
|
|
|
secrets := map[string][]byte{
|
|
"fsid": []byte(tc.poolName),
|
|
"mon-secret": []byte("monsecret"),
|
|
"admin-secret": []byte("adminsecret"),
|
|
}
|
|
secret := &v1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "rook-ceph-mon",
|
|
Namespace: namespace,
|
|
},
|
|
Data: secrets,
|
|
Type: k8sutil.RookType,
|
|
}
|
|
_, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
r := &ReconcileCephBlockPool{
|
|
client: cl,
|
|
scheme: s,
|
|
context: c,
|
|
blockPoolMirrorContexts: make(map[string]*blockPoolHealth),
|
|
opManagerContext: context.TODO(),
|
|
recorder: events.NewFakeRecorder(50),
|
|
}
|
|
|
|
req := reconcile.Request{
|
|
NamespacedName: types.NamespacedName{
|
|
Name: tc.poolName,
|
|
Namespace: namespace,
|
|
},
|
|
}
|
|
|
|
_, err = c.RookClientset.CephV1().CephBlockPools(namespace).Create(ctx, pool, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
res, err := r.Reconcile(ctx, req)
|
|
assert.NoError(t, err)
|
|
assert.Zero(t, res.RequeueAfter)
|
|
|
|
err = r.client.Get(context.TODO(), req.NamespacedName, pool)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, cephv1.ConditionReady, pool.Status.Phase, "%s pool status should reach Ready, not remain stuck in Progressing", tc.expectedType)
|
|
assert.Equal(t, tc.expectedType, pool.Status.Info["type"])
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCanConfigurePoolMirroring(t *testing.T) {
|
|
pool := &cephv1.CephBlockPool{}
|
|
assert.True(t, canConfigurePoolMirroring(pool.ToNamedPoolSpec()))
|
|
|
|
pool.Spec.Replicated.Size = 3
|
|
assert.True(t, canConfigurePoolMirroring(pool.ToNamedPoolSpec()))
|
|
|
|
ecpool := &cephv1.CephBlockPool{}
|
|
// test ec pool
|
|
ecpool.Spec.ErasureCoded.CodingChunks = 2
|
|
ecpool.Spec.ErasureCoded.DataChunks = 3
|
|
assert.False(t, canConfigurePoolMirroring(ecpool.ToNamedPoolSpec()))
|
|
}
|