tests: refactor for adding an upgrade test
Signed-off-by: travisn <tnielsen@redhat.com>
This commit is contained in:
35 files changed
+1214
-1133
No files matched your search
@@ -18,14 +18,21 @@ package clients
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
TestMountPath = "/tmp/testrook"
|
||||
)
|
||||
|
||||
//BlockOperation is wrapper for k8s rook block operations
|
||||
type BlockOperation struct {
|
||||
k8sClient *utils.K8sHelper
|
||||
manifests installer.CephManifests
|
||||
}
|
||||
|
||||
type BlockImage struct {
|
||||
@@ -41,9 +48,9 @@ var (
|
||||
readDataFromBlockPod = []string{"cat", "READ_DATA_CMD"}
|
||||
)
|
||||
|
||||
// CreateK8BlockOperation - Constructor to create BlockOperation - client to perform rook Block operations on k8s
|
||||
func CreateK8BlockOperation(k8shelp *utils.K8sHelper) *BlockOperation {
|
||||
return &BlockOperation{k8sClient: k8shelp}
|
||||
// CreateBlockOperation - Constructor to create BlockOperation - client to perform rook Block operations on k8s
|
||||
func CreateBlockOperation(k8shelp *utils.K8sHelper, manifests installer.CephManifests) *BlockOperation {
|
||||
return &BlockOperation{k8shelp, manifests}
|
||||
}
|
||||
|
||||
// BlockCreate Function to create a Block using Rook
|
||||
@@ -62,6 +69,24 @@ func (b *BlockOperation) Create(manifest string, size int) (string, error) {
|
||||
|
||||
}
|
||||
|
||||
func (b *BlockOperation) CreatePvc(claimName, storageClassName, mode string) (string, error) {
|
||||
return b.k8sClient.ResourceOperation("create", b.manifests.GetBlockPvcDef(claimName, storageClassName, mode))
|
||||
}
|
||||
|
||||
func (b *BlockOperation) CreateStorageClass(poolName, storageClassName, namespace string, varClusterName bool) (string, error) {
|
||||
return b.k8sClient.ResourceOperation("create", b.manifests.GetBlockStorageClassDef(poolName, storageClassName, namespace, varClusterName))
|
||||
}
|
||||
|
||||
func (b *BlockOperation) DeletePvc(claimName, storageClassName, mode string) error {
|
||||
_, err := b.k8sClient.ResourceOperation("delete", b.manifests.GetBlockPvcDef(claimName, storageClassName, mode))
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *BlockOperation) DeleteStorageClass(poolName, storageClassName, namespace string) error {
|
||||
_, err := b.k8sClient.ResourceOperation("delete", b.manifests.GetBlockStorageClassDef(poolName, storageClassName, namespace, false))
|
||||
return err
|
||||
}
|
||||
|
||||
// BlockDelete Function to delete a Block using Rook
|
||||
// Input parameters -
|
||||
// manifest - pod definition where pvc is described - delete is run on the the yaml definition
|
||||
@@ -117,9 +142,8 @@ func (b *BlockOperation) DeleteBlockImage(image BlockImage, namespace string) er
|
||||
// BlockMap Function to map a Block using Rook
|
||||
// Input parameters -
|
||||
// manifest - Pod definition - pod should be defined to use a pvc that was created earlier
|
||||
// mountpath - not used in this impl since mountpath is defined in the pod definition
|
||||
// Output - k8s create pod operation output and/or error
|
||||
func (b *BlockOperation) BlockMap(manifest string, mountpath string) (string, error) {
|
||||
func (b *BlockOperation) BlockMap(manifest string) (string, error) {
|
||||
args := []string{"create", "-f", "-"}
|
||||
result, err := b.k8sClient.KubectlWithStdin(manifest, args...)
|
||||
if err != nil {
|
||||
@@ -133,13 +157,26 @@ func (b *BlockOperation) BlockMap(manifest string, mountpath string) (string, er
|
||||
// Write Function to write data to block created by rook ,i.e. write data to a pod that is using a pvc
|
||||
// Input parameters -
|
||||
// manifest - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier
|
||||
// mountpath - folder on the pod were data is supposed to be written(should match the mountpath described in the pod definition)
|
||||
// data - data to be written
|
||||
// filename - file where data is written to
|
||||
// namespace - optional param - namespace of the pod
|
||||
// Output - k8s exec pod operation output and/or error
|
||||
func (b *BlockOperation) Write(name string, mountpath string, data string, filename string, namespace string) (string, error) {
|
||||
wt := "echo \"" + data + "\">" + mountpath + "/" + filename
|
||||
func (b *BlockOperation) Write(name string, data string, filename string, namespace string) (string, error) {
|
||||
return writeToPod(b.k8sClient, name, data, filename, namespace)
|
||||
}
|
||||
|
||||
// Read Function to read from block created by rook ,i.e. Read data from a pod that is using a pvc
|
||||
// Input parameters -
|
||||
// manifest - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier
|
||||
// filename - file to be read
|
||||
// namespace - optional param - namespace of the pod
|
||||
// Output - k8s exec pod operation output and/or error
|
||||
func (b *BlockOperation) Read(name string, filename string, namespace string) (string, error) {
|
||||
return readFromPod(b.k8sClient, name, filename, namespace)
|
||||
}
|
||||
|
||||
func writeToPod(k8sh *utils.K8sHelper, name string, data string, filename string, namespace string) (string, error) {
|
||||
wt := "echo \"" + data + "\">" + path.Join(TestMountPath, filename)
|
||||
args := []string{"exec", name}
|
||||
|
||||
if namespace != "" {
|
||||
@@ -147,24 +184,16 @@ func (b *BlockOperation) Write(name string, mountpath string, data string, filen
|
||||
}
|
||||
args = append(args, "--", "sh", "-c", wt)
|
||||
|
||||
result, err := b.k8sClient.Kubectl(args...)
|
||||
result, err := k8sh.Kubectl(args...)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Unable to write data to pod --: %s", err)
|
||||
|
||||
}
|
||||
return result, nil
|
||||
|
||||
}
|
||||
|
||||
// Read Function to read from block created by rook ,i.e. Read data from a pod that is using a pvc
|
||||
// Input parameters -
|
||||
// manifest - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier
|
||||
// mountpath - folder on the pod were data is supposed to be written(should match the mountpath described in the pod definition)
|
||||
// filename - file to be read
|
||||
// namespace - optional param - namespace of the pod
|
||||
// Output - k8s exec pod operation output and/or error
|
||||
func (b *BlockOperation) Read(name string, mountpath string, filename string, namespace string) (string, error) {
|
||||
rd := mountpath + "/" + filename
|
||||
func readFromPod(k8sh *utils.K8sHelper, name string, filename string, namespace string) (string, error) {
|
||||
rd := path.Join(TestMountPath, filename)
|
||||
args := []string{"exec", name}
|
||||
|
||||
if namespace != "" {
|
||||
@@ -172,27 +201,10 @@ func (b *BlockOperation) Read(name string, mountpath string, filename string, na
|
||||
}
|
||||
args = append(args, "--", "cat", rd)
|
||||
|
||||
result, err := b.k8sClient.Kubectl(args...)
|
||||
result, err := k8sh.Kubectl(args...)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Unable to read data to pod -- : %s", err)
|
||||
|
||||
}
|
||||
return result, nil
|
||||
|
||||
}
|
||||
|
||||
// BlockUnmap Function to unmap a Block using Rook
|
||||
// Input parameters -
|
||||
// manifest - pod definition - the pod described in yaml file is deleted
|
||||
// mountpath - not used in this impl since mountpath is defined in the pod definition
|
||||
// Output - k8s delete pod operation output and/or error
|
||||
func (b *BlockOperation) BlockUnmap(name string, mountpath string) (string, error) {
|
||||
args := []string{"delete", "-f", "-"}
|
||||
result, err := b.k8sClient.KubectlWithStdin(name, args...)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Unable to unmap block -- : %s", err)
|
||||
|
||||
}
|
||||
return result, nil
|
||||
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -27,12 +28,13 @@ import (
|
||||
|
||||
//FilesystemOperation is a wrapper for k8s rook file operations
|
||||
type FilesystemOperation struct {
|
||||
k8sh *utils.K8sHelper
|
||||
k8sh *utils.K8sHelper
|
||||
manifests installer.CephManifests
|
||||
}
|
||||
|
||||
// CreateK8sFilesystemOperation Constructor to create FilesystemOperation - client to perform rook file system operations on k8s
|
||||
func CreateK8sFilesystemOperation(k8sh *utils.K8sHelper) *FilesystemOperation {
|
||||
return &FilesystemOperation{k8sh: k8sh}
|
||||
// CreateFilesystemOperation Constructor to create FilesystemOperation - client to perform rook file system operations on k8s
|
||||
func CreateFilesystemOperation(k8sh *utils.K8sHelper, manifests installer.CephManifests) *FilesystemOperation {
|
||||
return &FilesystemOperation{k8sh, manifests}
|
||||
}
|
||||
|
||||
// FSCreate Function to create a filesystem in rook
|
||||
@@ -42,24 +44,7 @@ func CreateK8sFilesystemOperation(k8sh *utils.K8sHelper) *FilesystemOperation {
|
||||
func (f *FilesystemOperation) Create(name, namespace string) error {
|
||||
|
||||
logger.Infof("creating the filesystem via CRD")
|
||||
fsSpec := fmt.Sprintf(`apiVersion: ceph.rook.io/v1beta1
|
||||
kind: Filesystem
|
||||
metadata:
|
||||
name: %s
|
||||
namespace: %s
|
||||
spec:
|
||||
metadataPool:
|
||||
replicated:
|
||||
size: 1
|
||||
dataPools:
|
||||
- replicated:
|
||||
size: 1
|
||||
metadataServer:
|
||||
activeCount: 1
|
||||
activeStandby: true
|
||||
`, name, namespace)
|
||||
|
||||
if _, err := f.k8sh.ResourceOperation("create", fsSpec); err != nil {
|
||||
if _, err := f.k8sh.ResourceOperation("create", f.manifests.GetFilesystem(namespace, name)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -93,21 +78,6 @@ func (f *FilesystemOperation) List(namespace string) ([]client.CephFilesystem, e
|
||||
return filesystems, nil
|
||||
}
|
||||
|
||||
// Mount Function to Mount a file system created by rook(on a pod)
|
||||
// Input parameters -
|
||||
// name - path to the yaml definition file - definition of pod to be created that mounts existing file system
|
||||
// path - ignored in this case - mount path is defined in the path definition
|
||||
// output - output returned by k8s create pod operaton and/or error
|
||||
func (f *FilesystemOperation) Mount(name string, path string) (string, error) {
|
||||
result, err := f.k8sh.Kubectl("create", "-f", "name")
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Unable to mount Filesystem -- : %s", err)
|
||||
}
|
||||
return result, nil
|
||||
|
||||
}
|
||||
|
||||
// Write Function to write data to file system created by rook ,i.e. write data to a pod that has filesystem mounted
|
||||
// Input parameters -
|
||||
// name - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier
|
||||
@@ -116,22 +86,8 @@ func (f *FilesystemOperation) Mount(name string, path string) (string, error) {
|
||||
// filename - file where data is written to
|
||||
// namespace - optional param - namespace of the pod
|
||||
// Output - k8s exec pod operation output and/or error
|
||||
func (f *FilesystemOperation) Write(name string, mountpath string, data string, filename string, namespace string) (string, error) {
|
||||
wt := "echo \"" + data + "\">" + mountpath + "/" + filename
|
||||
args := []string{"exec", name}
|
||||
|
||||
if namespace != "" {
|
||||
args = append(args, "-n", namespace)
|
||||
}
|
||||
args = append(args, "--", "sh", "-c", wt)
|
||||
|
||||
result, err := f.k8sh.Kubectl(args...)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Unable to write data to pod -- : %s", err)
|
||||
|
||||
}
|
||||
return result, nil
|
||||
|
||||
func (f *FilesystemOperation) Write(name string, data string, filename string, namespace string) (string, error) {
|
||||
return writeToPod(f.k8sh, name, data, filename, namespace)
|
||||
}
|
||||
|
||||
// Read Function to write read from file system created by rook ,i.e. Read data from a pod that filesystem mounted
|
||||
@@ -141,35 +97,6 @@ func (f *FilesystemOperation) Write(name string, mountpath string, data string,
|
||||
// filename - file to be read
|
||||
// namespace - optional param - namespace of the pod
|
||||
// Output - k8s exec pod operation output and/or error
|
||||
func (f *FilesystemOperation) Read(name string, mountpath string, filename string, namespace string) (string, error) {
|
||||
rd := mountpath + "/" + filename
|
||||
|
||||
args := []string{"exec", name}
|
||||
|
||||
if namespace != "" {
|
||||
args = append(args, "-n", namespace)
|
||||
}
|
||||
args = append(args, "--", "cat", rd)
|
||||
|
||||
result, err := f.k8sh.Kubectl(args...)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Unable to write data to pod -- : %s", err)
|
||||
|
||||
}
|
||||
return result, nil
|
||||
|
||||
}
|
||||
|
||||
// Unmount Function to UnMount a file system created by rook(delete pod)
|
||||
// Input parameters -
|
||||
// name - path to the yaml definition file - definition of pod to be deleted that has a file system mounted
|
||||
// path - ignored in this case - mount path is defined in the path definition
|
||||
// output - output returned by k8s delete pod operaton and/or error
|
||||
func (f *FilesystemOperation) Unmount(name string) (string, error) {
|
||||
result, err := f.k8sh.Kubectl("delete", "-f", name)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Unable to unmount Filesystem -- : %s", err)
|
||||
|
||||
}
|
||||
return result, nil
|
||||
func (f *FilesystemOperation) Read(name string, filename string, namespace string) (string, error) {
|
||||
return readFromPod(f.k8sh, name, filename, namespace)
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
)
|
||||
|
||||
@@ -29,60 +30,40 @@ var logger = capnslog.NewPackageLogger("github.com/rook/rook/tests", "clients")
|
||||
|
||||
// ObjectOperation is wrapper for k8s rook object operations
|
||||
type ObjectOperation struct {
|
||||
k8sh *utils.K8sHelper
|
||||
k8sh *utils.K8sHelper
|
||||
manifests installer.CephManifests
|
||||
}
|
||||
|
||||
// CreateObjectOperation creates new rook object client
|
||||
func CreateObjectOperation(k8sh *utils.K8sHelper) *ObjectOperation {
|
||||
return &ObjectOperation{k8sh: k8sh}
|
||||
func CreateObjectOperation(k8sh *utils.K8sHelper, manifests installer.CephManifests) *ObjectOperation {
|
||||
return &ObjectOperation{k8sh, manifests}
|
||||
}
|
||||
|
||||
// ObjectCreate Function to create a object store in rook
|
||||
func (ro *ObjectOperation) Create(namespace, storeName string, replicaCount int32) error {
|
||||
func (o *ObjectOperation) Create(namespace, storeName string, replicaCount int32) error {
|
||||
|
||||
logger.Infof("creating the object store via CRD")
|
||||
storeSpec := fmt.Sprintf(`apiVersion: ceph.rook.io/v1beta1
|
||||
kind: ObjectStore
|
||||
metadata:
|
||||
name: %s
|
||||
namespace: %s
|
||||
spec:
|
||||
metadataPool:
|
||||
replicated:
|
||||
size: 1
|
||||
dataPool:
|
||||
replicated:
|
||||
size: 1
|
||||
gateway:
|
||||
type: s3
|
||||
sslCertificateRef:
|
||||
port: %d
|
||||
securePort:
|
||||
instances: %d
|
||||
allNodes: false
|
||||
`, storeName, namespace, rgwPort, replicaCount)
|
||||
|
||||
if _, err := ro.k8sh.ResourceOperation("create", storeSpec); err != nil {
|
||||
if _, err := o.k8sh.ResourceOperation("create", o.manifests.GetObjectStore(namespace, storeName, int(replicaCount), rgwPort)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err := ro.k8sh.WaitForLabeledPodToRun(fmt.Sprintf("rook_object_store=%s", storeName), namespace)
|
||||
err := o.k8sh.WaitForLabeledPodToRun(fmt.Sprintf("rook_object_store=%s", storeName), namespace)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rgw did not start via crd. %+v", err)
|
||||
}
|
||||
|
||||
// create the external service
|
||||
return ro.k8sh.CreateExternalRGWService(namespace, storeName)
|
||||
return o.k8sh.CreateExternalRGWService(namespace, storeName)
|
||||
}
|
||||
|
||||
func (ro *ObjectOperation) Delete(namespace, storeName string) error {
|
||||
func (o *ObjectOperation) Delete(namespace, storeName string) error {
|
||||
|
||||
logger.Infof("Deleting the object store via CRD")
|
||||
if _, err := ro.k8sh.DeleteResource("-n", namespace, "ObjectStore", storeName); err != nil {
|
||||
if _, err := o.k8sh.DeleteResource("-n", namespace, "ObjectStore", storeName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !ro.k8sh.WaitUntilPodWithLabelDeleted(fmt.Sprintf("rook_object_store=%s", storeName), namespace) {
|
||||
if !o.k8sh.WaitUntilPodWithLabelDeleted(fmt.Sprintf("rook_object_store=%s", storeName), namespace) {
|
||||
return fmt.Errorf("rgw did not stop via crd")
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/model"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
@@ -31,24 +30,30 @@ import (
|
||||
|
||||
//PoolOperation is a wrapper for rook pool operations
|
||||
type PoolOperation struct {
|
||||
k8sh *utils.K8sHelper
|
||||
k8sh *utils.K8sHelper
|
||||
manifests installer.CephManifests
|
||||
}
|
||||
|
||||
// CreatePoolOperation creates a new pool client
|
||||
func CreatePoolOperation(k8sh *utils.K8sHelper) *PoolOperation {
|
||||
return &PoolOperation{k8sh: k8sh}
|
||||
func CreatePoolOperation(k8sh *utils.K8sHelper, manifests installer.CephManifests) *PoolOperation {
|
||||
return &PoolOperation{k8sh, manifests}
|
||||
}
|
||||
|
||||
func (p *PoolOperation) Create(pool model.Pool, namespace string) (string, error) {
|
||||
return p.createOrUpdatePool(pool, namespace, "create")
|
||||
func (p *PoolOperation) Create(name, namespace string, replicas int) (string, error) {
|
||||
return p.createOrUpdatePool(name, namespace, "create", replicas)
|
||||
}
|
||||
|
||||
func (p *PoolOperation) Update(pool model.Pool, namespace string) (string, error) {
|
||||
return p.createOrUpdatePool(pool, namespace, "apply")
|
||||
func (p *PoolOperation) Delete(name string, namespace string) error {
|
||||
_, err := p.k8sh.ResourceOperation("delete", p.manifests.GetBlockPoolDef(name, namespace, "1"))
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PoolOperation) createOrUpdatePool(pool model.Pool, namespace, action string) (string, error) {
|
||||
return installer.BlockResourceOperation(p.k8sh, installer.GetBlockPoolDef(pool.Name, namespace, strconv.Itoa(int(pool.ReplicatedConfig.Size))), action)
|
||||
func (p *PoolOperation) Update(name, namespace string, replicas int) (string, error) {
|
||||
return p.createOrUpdatePool(name, namespace, "apply", replicas)
|
||||
}
|
||||
|
||||
func (p *PoolOperation) createOrUpdatePool(name, namespace, action string, replicas int) (string, error) {
|
||||
return p.k8sh.ResourceOperation(action, p.manifests.GetBlockPoolDef(name, namespace, strconv.Itoa(replicas)))
|
||||
}
|
||||
|
||||
func (p *PoolOperation) ListCephPools(namespace string) ([]client.CephStoragePoolSummary, error) {
|
||||
@@ -104,3 +109,21 @@ func (p *PoolOperation) CephPoolExists(namespace, name string) (bool, error) {
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (p *PoolOperation) CreateStorageClassAndPvc(namespace, poolName, storageClassName, blockName, mode string) (string, error) {
|
||||
return p.k8sh.ResourceOperation("create", p.manifests.GetBlockPoolStorageClassAndPvcDef(namespace, poolName, storageClassName, blockName, mode))
|
||||
}
|
||||
|
||||
func (p *PoolOperation) DeleteStorageClass(namespace, poolName, storageClassName string) error {
|
||||
_, err := p.k8sh.ResourceOperation("delete", p.manifests.GetBlockPoolStorageClass(namespace, poolName, storageClassName))
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PoolOperation) DeleteStorageClassAndPvc(namespace, poolName, storageClassName, blockName, mode string) error {
|
||||
_, err := p.k8sh.ResourceOperation("delete", p.manifests.GetBlockPoolStorageClassAndPvcDef(namespace, poolName, storageClassName, blockName, mode))
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *PoolOperation) CreateStorageClass(namespace, poolName, storageClassName string) (string, error) {
|
||||
return p.k8sh.ResourceOperation("create", p.manifests.GetBlockPoolStorageClass(namespace, poolName, storageClassName))
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
)
|
||||
|
||||
@@ -41,15 +42,15 @@ const (
|
||||
)
|
||||
|
||||
//CreateTestClient creates new instance of test client for a platform
|
||||
func CreateTestClient(k8sHelper *utils.K8sHelper, namespace string) (*TestClient, error) {
|
||||
func CreateTestClient(k8sHelper *utils.K8sHelper, manifests installer.CephManifests) *TestClient {
|
||||
|
||||
return &TestClient{
|
||||
CreateK8BlockOperation(k8sHelper),
|
||||
CreateK8sFilesystemOperation(k8sHelper),
|
||||
CreateObjectOperation(k8sHelper),
|
||||
CreatePoolOperation(k8sHelper),
|
||||
CreateBlockOperation(k8sHelper, manifests),
|
||||
CreateFilesystemOperation(k8sHelper, manifests),
|
||||
CreateObjectOperation(k8sHelper, manifests),
|
||||
CreatePoolOperation(k8sHelper, manifests),
|
||||
k8sHelper,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
//Status returns rook status details
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package contracts
|
||||
|
||||
type Setup interface {
|
||||
SetUp()
|
||||
TearDown()
|
||||
}
|
||||
+57
-181
@@ -22,24 +22,17 @@ import (
|
||||
"math/rand"
|
||||
"os"
|
||||
"path"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/kubernetes/pkg/kubelet/apis"
|
||||
|
||||
"flag"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
|
||||
"github.com/rook/rook/pkg/util/exec"
|
||||
"github.com/rook/rook/pkg/util/sys"
|
||||
"github.com/rook/rook/tests/framework/objects"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
@@ -49,71 +42,43 @@ import (
|
||||
|
||||
const (
|
||||
rookOperatorCreatedCrd = "clusters.ceph.rook.io"
|
||||
cockroachDBCRD = "clusters.cockroachdb.rook.io"
|
||||
nfsServerCRD = "nfsservers.nfs.rook.io"
|
||||
helmChartName = "local/rook-ceph"
|
||||
helmDeployName = "rook-ceph"
|
||||
)
|
||||
|
||||
var (
|
||||
// ** Variables that might need to be changed depending on the dev environment. The init function below will modify some of them automatically. **
|
||||
baseTestDir string
|
||||
forceUseDevices = false
|
||||
createBaseTestDir = true
|
||||
// ** end of Variables to modify
|
||||
logger = capnslog.NewPackageLogger("github.com/rook/rook", "installer")
|
||||
createArgs = []string{"create", "-f"}
|
||||
createFromStdinArgs = append(createArgs, "-")
|
||||
deleteArgs = []string{"delete", "-f"}
|
||||
deleteFromStdinArgs = append(deleteArgs, "-")
|
||||
)
|
||||
|
||||
func init() {
|
||||
// this default will only work if running kubernetes on the local machine
|
||||
baseTestDir, _ = os.Getwd()
|
||||
|
||||
// The following settings could apply to any environment when the kube context is running on the host and the tests are running inside a
|
||||
// VM such as minikube. This is a cheap test for this condition, we need to find a better way to automate these settings.
|
||||
if runtime.GOOS == "darwin" {
|
||||
createBaseTestDir = false
|
||||
baseTestDir = "/data"
|
||||
}
|
||||
}
|
||||
|
||||
//InstallHelper wraps installing and uninstalling rook on a platform
|
||||
type InstallHelper struct {
|
||||
//CephInstaller wraps installing and uninstalling rook on a platform
|
||||
type CephInstaller struct {
|
||||
Manifests CephManifests
|
||||
k8shelper *utils.K8sHelper
|
||||
installData *InstallData
|
||||
hostPathToDelete string
|
||||
helmHelper *utils.HelmHelper
|
||||
Env objects.EnvironmentManifest
|
||||
k8sVersion string
|
||||
T func() *testing.T
|
||||
}
|
||||
|
||||
func (h *InstallHelper) CreateK8sRookResources() (err error) {
|
||||
func (h *CephInstaller) CreateCephCRDs() (err error) {
|
||||
var resources string
|
||||
logger.Info("Creating Rook CRDs")
|
||||
|
||||
resources = h.installData.GetRookCRDs()
|
||||
resources = h.Manifests.GetRookCRDs()
|
||||
|
||||
_, err = h.k8shelper.KubectlWithStdin(resources, createFromStdinArgs...)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
//CreateK8sRookOperator creates rook-operator via kubectl
|
||||
func (h *InstallHelper) CreateK8sRookOperator(namespace string) (err error) {
|
||||
//CreateCephOperator creates rook-operator via kubectl
|
||||
func (h *CephInstaller) CreateCephOperator(namespace string) (err error) {
|
||||
logger.Infof("Starting Rook Operator")
|
||||
//creating clusterrolebinding for kubeadm env.
|
||||
h.k8shelper.CreateAnonSystemClusterBinding()
|
||||
|
||||
//creating rook resources
|
||||
if err = h.CreateK8sRookResources(); err != nil {
|
||||
if err = h.CreateCephCRDs(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rookOperator := h.installData.GetRookOperator(namespace)
|
||||
rookOperator := h.Manifests.GetRookOperator(namespace)
|
||||
|
||||
_, err = h.k8shelper.KubectlWithStdin(rookOperator, createFromStdinArgs...)
|
||||
if err != nil {
|
||||
@@ -130,7 +95,7 @@ func (h *InstallHelper) CreateK8sRookOperator(namespace string) (err error) {
|
||||
}
|
||||
|
||||
//CreateK8sRookOperatorViaHelm creates rook operator via Helm chart named local/rook present in local repo
|
||||
func (h *InstallHelper) CreateK8sRookOperatorViaHelm(namespace string) error {
|
||||
func (h *CephInstaller) CreateK8sRookOperatorViaHelm(namespace string) error {
|
||||
//creating clusterrolebinding for kubeadm env.
|
||||
h.k8shelper.CreateAnonSystemClusterBinding()
|
||||
|
||||
@@ -154,10 +119,10 @@ func (h *InstallHelper) CreateK8sRookOperatorViaHelm(namespace string) error {
|
||||
}
|
||||
|
||||
//CreateK8sRookToolbox creates rook-ceph-tools via kubectl
|
||||
func (h *InstallHelper) CreateK8sRookToolbox(namespace string) (err error) {
|
||||
func (h *CephInstaller) CreateK8sRookToolbox(namespace string) (err error) {
|
||||
logger.Infof("Starting Rook toolbox")
|
||||
|
||||
rookToolbox := h.installData.GetRookToolBox(namespace)
|
||||
rookToolbox := h.Manifests.GetRookToolBox(namespace)
|
||||
|
||||
_, err = h.k8shelper.KubectlWithStdin(rookToolbox, createFromStdinArgs...)
|
||||
|
||||
@@ -173,13 +138,13 @@ func (h *InstallHelper) CreateK8sRookToolbox(namespace string) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *InstallHelper) CreateK8sRookCluster(namespace, systemNamespace string, storeType string) (err error) {
|
||||
func (h *CephInstaller) CreateK8sRookCluster(namespace, systemNamespace string, storeType string) (err error) {
|
||||
return h.CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType, false,
|
||||
cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, true /* startWithAllNodes */)
|
||||
}
|
||||
|
||||
//CreateK8sRookCluster creates rook cluster via kubectl
|
||||
func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType string,
|
||||
func (h *CephInstaller) CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType string,
|
||||
useAllDevices bool, mon cephv1beta1.MonSpec, startWithAllNodes bool) error {
|
||||
|
||||
dataDirHostPath, err := h.initTestDir(namespace)
|
||||
@@ -197,7 +162,7 @@ func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, sy
|
||||
}
|
||||
|
||||
logger.Infof("Creating cluster roles")
|
||||
roles := h.installData.GetClusterRoles(namespace, systemNamespace)
|
||||
roles := h.Manifests.GetClusterRoles(namespace, systemNamespace)
|
||||
if _, err := h.k8shelper.KubectlWithStdin(roles, createFromStdinArgs...); err != nil {
|
||||
return fmt.Errorf("Failed to create cluster roles. %+v", err)
|
||||
}
|
||||
@@ -262,7 +227,7 @@ func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, sy
|
||||
}
|
||||
} else {
|
||||
logger.Infof("Starting Rook Cluster with yaml")
|
||||
rookCluster := h.installData.GetRookCluster(namespace, storeType, dataDirHostPath, useAllDevices, mon.Count)
|
||||
rookCluster := h.Manifests.GetRookCluster(&ClusterSettings{namespace, storeType, dataDirHostPath, useAllDevices, mon.Count})
|
||||
if _, err := h.k8shelper.KubectlWithStdin(rookCluster, createFromStdinArgs...); err != nil {
|
||||
return fmt.Errorf("Failed to create rook cluster : %v ", err)
|
||||
}
|
||||
@@ -281,7 +246,7 @@ func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, sy
|
||||
return err
|
||||
}
|
||||
|
||||
func (h *InstallHelper) initTestDir(namespace string) (string, error) {
|
||||
func (h *CephInstaller) initTestDir(namespace string) (string, error) {
|
||||
h.hostPathToDelete = path.Join(baseTestDir, "rook-test")
|
||||
testDir := path.Join(h.hostPathToDelete, namespace)
|
||||
|
||||
@@ -303,7 +268,7 @@ func (h *InstallHelper) initTestDir(namespace string) (string, error) {
|
||||
return testDir, nil
|
||||
}
|
||||
|
||||
func (h *InstallHelper) GetNodeHostnames() ([]string, error) {
|
||||
func (h *CephInstaller) GetNodeHostnames() ([]string, error) {
|
||||
nodes, err := h.k8shelper.Clientset.CoreV1().Nodes().List(metav1.ListOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get k8s nodes. %+v", err)
|
||||
@@ -316,17 +281,13 @@ func (h *InstallHelper) GetNodeHostnames() ([]string, error) {
|
||||
return names, nil
|
||||
}
|
||||
|
||||
func SystemNamespace(namespace string) string {
|
||||
return fmt.Sprintf("%s-system", namespace)
|
||||
}
|
||||
|
||||
//InstallRookOnK8sWithHostPathAndDevices installs rook on k8s
|
||||
func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeType string,
|
||||
func (h *CephInstaller) InstallRookOnK8sWithHostPathAndDevices(namespace, storeType string,
|
||||
helmInstalled, useDevices bool, mon cephv1beta1.MonSpec, startWithAllNodes bool) (bool, error) {
|
||||
|
||||
var err error
|
||||
//flag used for local debuggin purpose, when rook is pre-installed
|
||||
if h.Env.SkipInstallRook {
|
||||
if Env.SkipInstallRook {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -346,7 +307,7 @@ func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeT
|
||||
} else {
|
||||
onamespace = SystemNamespace(namespace)
|
||||
|
||||
err := h.CreateK8sRookOperator(onamespace)
|
||||
err := h.CreateCephOperator(onamespace)
|
||||
if err != nil {
|
||||
logger.Errorf("Rook Operator not installed ,error -> %v", err)
|
||||
return false, err
|
||||
@@ -354,7 +315,7 @@ func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeT
|
||||
}
|
||||
if !h.k8shelper.IsPodInExpectedState("rook-ceph-operator", onamespace, "Running") {
|
||||
logger.Error("rook-ceph-operator is not running")
|
||||
h.k8shelper.GetRookLogs("rook-ceph-operator", h.Env.HostType, onamespace, "test-setup")
|
||||
h.k8shelper.GetRookLogs("rook-ceph-operator", Env.HostType, onamespace, "test-setup")
|
||||
logger.Error("rook-ceph-operator is not Running, abort!")
|
||||
return false, err
|
||||
}
|
||||
@@ -387,14 +348,14 @@ func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeT
|
||||
}
|
||||
|
||||
//UninstallRookFromK8s uninstalls rook from k8s
|
||||
func (h *InstallHelper) UninstallRook(helmInstalled bool, namespace string) {
|
||||
func (h *CephInstaller) UninstallRook(helmInstalled bool, namespace string) {
|
||||
h.UninstallRookFromMultipleNS(helmInstalled, SystemNamespace(namespace), namespace)
|
||||
}
|
||||
|
||||
//UninstallRookFromK8s uninstalls rook from multiple namespaces in k8s
|
||||
func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNamespace string, namespaces ...string) {
|
||||
func (h *CephInstaller) UninstallRookFromMultipleNS(helmInstalled bool, systemNamespace string, namespaces ...string) {
|
||||
//flag used for local debugging purpose, when rook is pre-installed
|
||||
if h.Env.SkipInstallRook {
|
||||
if Env.SkipInstallRook {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -404,43 +365,43 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa
|
||||
|
||||
if !h.k8shelper.VersionAtLeast("v1.8.0") {
|
||||
_, err = h.k8shelper.DeleteResource("-n", namespace, "serviceaccount", "rook-ceph-cluster")
|
||||
h.checkError(err, "cannot remove serviceaccount rook-ceph-cluster")
|
||||
checkError(h.T(), err, "cannot remove serviceaccount rook-ceph-cluster")
|
||||
assert.NoError(h.T(), err, "%s err -> %v", namespace, err)
|
||||
|
||||
err = h.k8shelper.DeleteRoleAndBindings("rook-ceph-cluster", namespace)
|
||||
h.checkError(err, "rook-ceph-cluster cluster role and binding cannot be deleted")
|
||||
checkError(h.T(), err, "rook-ceph-cluster cluster role and binding cannot be deleted")
|
||||
assert.NoError(h.T(), err, "rook-ceph-cluster cluster role and binding cannot be deleted: %+v", err)
|
||||
|
||||
err = h.k8shelper.DeleteRoleBinding("rook-ceph-cluster-mgmt", namespace)
|
||||
h.checkError(err, "rook-ceph-cluster-mgmt binding cannot be deleted")
|
||||
checkError(h.T(), err, "rook-ceph-cluster-mgmt binding cannot be deleted")
|
||||
assert.NoError(h.T(), err, "rook-ceph-cluster-mgmt binding cannot be deleted: %+v", err)
|
||||
}
|
||||
|
||||
_, err = h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cluster.ceph.rook.io", namespace)
|
||||
h.checkError(err, fmt.Sprintf("cannot remove cluster %s", namespace))
|
||||
checkError(h.T(), err, fmt.Sprintf("cannot remove cluster %s", namespace))
|
||||
|
||||
crdCheckerFunc := func() error {
|
||||
_, err := h.k8shelper.RookClientset.RookV1alpha1().Clusters(namespace).Get(namespace, metav1.GetOptions{})
|
||||
return err
|
||||
}
|
||||
err = h.waitForCustomResourceDeletion(namespace, crdCheckerFunc)
|
||||
h.checkError(err, fmt.Sprintf("failed to wait for crd %s deletion", namespace))
|
||||
err = h.k8shelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc)
|
||||
checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace))
|
||||
|
||||
_, err = h.k8shelper.DeleteResourceAndWait(false, "namespace", namespace)
|
||||
h.checkError(err, fmt.Sprintf("cannot delete namespace %s", namespace))
|
||||
checkError(h.T(), err, fmt.Sprintf("cannot delete namespace %s", namespace))
|
||||
}
|
||||
|
||||
logger.Infof("removing the operator from namespace %s", systemNamespace)
|
||||
_, err = h.k8shelper.DeleteResource("crd", "clusters.ceph.rook.io", "pools.ceph.rook.io", "objectstores.ceph.rook.io", "filesystems.ceph.rook.io", "volumes.rook.io")
|
||||
h.checkError(err, "cannot delete CRDs")
|
||||
checkError(h.T(), err, "cannot delete CRDs")
|
||||
|
||||
if helmInstalled {
|
||||
err = h.helmHelper.DeleteLocalRookHelmChart(helmDeployName)
|
||||
} else {
|
||||
rookOperator := h.installData.GetRookOperator(systemNamespace)
|
||||
rookOperator := h.Manifests.GetRookOperator(systemNamespace)
|
||||
_, err = h.k8shelper.KubectlWithStdin(rookOperator, deleteFromStdinArgs...)
|
||||
}
|
||||
h.checkError(err, "cannot uninstall rook-operator")
|
||||
checkError(h.T(), err, "cannot uninstall rook-operator")
|
||||
|
||||
h.k8shelper.Clientset.RbacV1beta1().RoleBindings(systemNamespace).Delete("rook-ceph-system", nil)
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-ceph-global", nil)
|
||||
@@ -454,7 +415,7 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa
|
||||
// removing data dir if exists
|
||||
if h.hostPathToDelete != "" {
|
||||
nodes, err := h.GetNodeHostnames()
|
||||
h.checkError(err, "cannot get node names")
|
||||
checkError(h.T(), err, "cannot get node names")
|
||||
for _, node := range nodes {
|
||||
err = h.cleanupDir(node, h.hostPathToDelete)
|
||||
logger.Infof("removing %s from node %s. err=%v", h.hostPathToDelete, node, err)
|
||||
@@ -462,67 +423,27 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa
|
||||
}
|
||||
}
|
||||
|
||||
func (h *InstallHelper) cleanupDir(node, dir string) error {
|
||||
resources := h.installData.GetCleanupPod(node, dir)
|
||||
func (h *CephInstaller) cleanupDir(node, dir string) error {
|
||||
resources := h.Manifests.GetCleanupPod(node, dir)
|
||||
_, err := h.k8shelper.KubectlWithStdin(resources, createFromStdinArgs...)
|
||||
return err
|
||||
}
|
||||
|
||||
func (h *InstallHelper) checkError(err error, message string) {
|
||||
// During cleanup the resource might not be found because the test might have failed before the test was done and never created the resource
|
||||
if err == nil || errors.IsNotFound(err) {
|
||||
return
|
||||
}
|
||||
assert.NoError(h.T(), err, "%s. %+v", message, err)
|
||||
}
|
||||
|
||||
func (h *InstallHelper) waitForCustomResourceDeletion(namespace string, checkerFunc func() error) error {
|
||||
if !h.k8shelper.VersionAtLeast("v1.8.0") {
|
||||
// v1.7 has an intermittent issue with long delay to delete resources so we will skip waiting
|
||||
return nil
|
||||
}
|
||||
|
||||
// wait for the operator to finalize and delete the CRD
|
||||
for i := 0; i < 10; i++ {
|
||||
err := checkerFunc()
|
||||
if err == nil {
|
||||
logger.Infof("custom resource %s still exists", namespace)
|
||||
time.Sleep(2 * time.Second)
|
||||
continue
|
||||
}
|
||||
if errors.IsNotFound(err) {
|
||||
logger.Infof("custom resource %s deleted", namespace)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
logger.Errorf("gave up deleting custom resource %s", namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *InstallHelper) GatherAllRookLogs(namespace, systemNamespace string, testName string) {
|
||||
func (h *CephInstaller) GatherAllRookLogs(namespace, systemNamespace string, testName string) {
|
||||
logger.Infof("Gathering all logs from Rook Cluster %s", namespace)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-operator", h.Env.HostType, systemNamespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-agent", h.Env.HostType, systemNamespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-discover", h.Env.HostType, systemNamespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-mgr", h.Env.HostType, namespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-mon", h.Env.HostType, namespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-osd", h.Env.HostType, namespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-osd-prepare", h.Env.HostType, namespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-rgw", h.Env.HostType, namespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-mds", h.Env.HostType, namespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-operator", Env.HostType, systemNamespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-agent", Env.HostType, systemNamespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-discover", Env.HostType, systemNamespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-mgr", Env.HostType, namespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-mon", Env.HostType, namespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-osd", Env.HostType, namespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-osd-prepare", Env.HostType, namespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-rgw", Env.HostType, namespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-mds", Env.HostType, namespace, testName)
|
||||
}
|
||||
|
||||
// GatherCRDObjectDebuggingInfo gathers all the descriptions for pods, pvs and pvcs
|
||||
func (h *InstallHelper) GatherCRDObjectDebuggingInfo(namespace string) {
|
||||
h.k8shelper.PrintPodDescribeForNamespace(namespace)
|
||||
h.k8shelper.PrintPVs(true /*detailed*/)
|
||||
h.k8shelper.PrintPVCs(namespace, true /*detailed*/)
|
||||
h.k8shelper.PrintStorageClasses(true /*detailed*/)
|
||||
}
|
||||
|
||||
//NewK8sRookhelper creates new instance of InstallHelper
|
||||
func NewK8sRookhelper(clientset *kubernetes.Clientset, t func() *testing.T) *InstallHelper {
|
||||
// NewCephInstaller creates new instance of CephInstaller
|
||||
func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, rookVersion string) *CephInstaller {
|
||||
|
||||
// All e2e tests should run ceph commands in the toolbox since we are not inside a container
|
||||
client.RunAllCephCommandsInToolbox = true
|
||||
@@ -536,58 +457,13 @@ func NewK8sRookhelper(clientset *kubernetes.Clientset, t func() *testing.T) *Ins
|
||||
if err != nil {
|
||||
panic("failed to get kubectl client :" + err.Error())
|
||||
}
|
||||
ih := &InstallHelper{
|
||||
k8shelper: k8shelp,
|
||||
installData: NewK8sInstallData(),
|
||||
helmHelper: utils.NewHelmHelper(),
|
||||
Env: objects.Env,
|
||||
k8sVersion: version.String(),
|
||||
T: t,
|
||||
h := &CephInstaller{
|
||||
Manifests: NewCephManifests(rookVersion),
|
||||
k8shelper: k8shelp,
|
||||
helmHelper: utils.NewHelmHelper(),
|
||||
k8sVersion: version.String(),
|
||||
T: t,
|
||||
}
|
||||
flag.Parse()
|
||||
return ih
|
||||
}
|
||||
|
||||
func IsAdditionalDeviceAvailableOnCluster() bool {
|
||||
executor := &exec.CommandExecutor{}
|
||||
devices, err := sys.ListDevices(executor)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
disks := 0
|
||||
logger.Infof("devices : %v", devices)
|
||||
for _, device := range devices {
|
||||
if strings.Contains(device, "loop") {
|
||||
continue
|
||||
}
|
||||
props, _ := sys.GetDeviceProperties(device, executor)
|
||||
if props["TYPE"] != "disk" {
|
||||
continue
|
||||
}
|
||||
|
||||
ownPartitions, fs, err := sys.CheckIfDeviceAvailable(executor, device)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to detect device %s availability. %+v", device, err)
|
||||
continue
|
||||
}
|
||||
if !ownPartitions {
|
||||
logger.Infof("skipping device %s since don't own partitions", device)
|
||||
continue
|
||||
}
|
||||
if fs != "" {
|
||||
logger.Infof("skipping device %s since it has file system %s", device, fs)
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(device, "rbd") {
|
||||
logger.Infof("skipping unexpected rbd device: %s", device)
|
||||
continue
|
||||
}
|
||||
logger.Infof("available device: %s", device)
|
||||
disks++
|
||||
}
|
||||
if disks > 0 {
|
||||
return true
|
||||
}
|
||||
logger.Info("No additional disks found on cluster")
|
||||
return false
|
||||
return h
|
||||
}
|
||||
+141
-20
@@ -17,21 +17,51 @@ limitations under the License.
|
||||
package installer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
//InstallData wraps rook yaml definitions
|
||||
type InstallData struct {
|
||||
type CephManifests interface {
|
||||
GetRookCRDs() string
|
||||
GetRookOperator(namespace string) string
|
||||
GetClusterRoles(namespace, systemNamespace string) string
|
||||
GetRookCluster(settings *ClusterSettings) string
|
||||
GetRookToolBox(namespace string) string
|
||||
GetCleanupPod(node, removalDir string) string
|
||||
GetBlockPoolDef(poolName string, namespace string, replicaSize string) string
|
||||
GetBlockStorageClassDef(poolName string, storageClassName string, namespace string, varClusterName bool) string
|
||||
GetBlockPvcDef(claimName string, storageClassName string, accessModes string) string
|
||||
GetBlockPoolStorageClassAndPvcDef(namespace string, poolName string, storageClassName string, blockName string, accessMode string) string
|
||||
GetBlockPoolStorageClass(namespace string, poolName string, storageClassName string) string
|
||||
GetFilesystem(namepace, name string) string
|
||||
GetObjectStore(namespace, name string, replicaCount, port int) string
|
||||
}
|
||||
|
||||
//NewK8sInstallData creates new instance of InstallData struct
|
||||
func NewK8sInstallData() *InstallData {
|
||||
return &InstallData{}
|
||||
type ClusterSettings struct {
|
||||
Namespace string
|
||||
StoreType string
|
||||
DataDirHostPath string
|
||||
UseAllDevices bool
|
||||
Mons int
|
||||
}
|
||||
|
||||
func (i *InstallData) GetRookCRDs() string {
|
||||
//CephManifestsMaster wraps rook yaml definitions
|
||||
type CephManifestsMaster struct {
|
||||
imageTag string
|
||||
}
|
||||
|
||||
// NewCephManifests gets the manifest type depending on the Rook version desired
|
||||
func NewCephManifests(version string) CephManifests {
|
||||
switch version {
|
||||
case VersionMaster:
|
||||
return &CephManifestsMaster{imageTag: VersionMaster}
|
||||
}
|
||||
panic(fmt.Errorf("unrecognized ceph manifest version: %s", version))
|
||||
}
|
||||
|
||||
func (m *CephManifestsMaster) GetRookCRDs() string {
|
||||
return `apiVersion: apiextensions.k8s.io/v1beta1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
@@ -104,7 +134,7 @@ spec:
|
||||
}
|
||||
|
||||
//GetRookOperator returns rook Operator manifest
|
||||
func (i *InstallData) GetRookOperator(namespace string) string {
|
||||
func (m *CephManifestsMaster) GetRookOperator(namespace string) string {
|
||||
|
||||
return `kind: Namespace
|
||||
apiVersion: v1
|
||||
@@ -307,7 +337,7 @@ spec:
|
||||
serviceAccountName: rook-ceph-system
|
||||
containers:
|
||||
- name: rook-ceph-operator
|
||||
image: rook/ceph:master
|
||||
image: rook/ceph:` + m.imageTag + `
|
||||
args: ["ceph", "operator"]
|
||||
env:
|
||||
- name: ROOK_LOG_LEVEL
|
||||
@@ -331,7 +361,7 @@ spec:
|
||||
}
|
||||
|
||||
//GetRookCluster returns rook-cluster manifest
|
||||
func (i *InstallData) GetClusterRoles(namespace, systemNamespace string) string {
|
||||
func (m *CephManifestsMaster) GetClusterRoles(namespace, systemNamespace string) string {
|
||||
return `apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
@@ -380,36 +410,36 @@ subjects:
|
||||
}
|
||||
|
||||
//GetRookCluster returns rook-cluster manifest
|
||||
func (i *InstallData) GetRookCluster(namespace, storeType, dataDirHostPath string, useAllDevices bool, mons int) string {
|
||||
func (m *CephManifestsMaster) GetRookCluster(settings *ClusterSettings) string {
|
||||
return `apiVersion: ceph.rook.io/v1beta1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: ` + namespace + `
|
||||
namespace: ` + namespace + `
|
||||
name: ` + settings.Namespace + `
|
||||
namespace: ` + settings.Namespace + `
|
||||
spec:
|
||||
serviceAccount: rook-ceph-cluster
|
||||
dataDirHostPath: ` + dataDirHostPath + `
|
||||
dataDirHostPath: ` + settings.DataDirHostPath + `
|
||||
network:
|
||||
hostNetwork: false
|
||||
mon:
|
||||
count: ` + strconv.Itoa(mons) + `
|
||||
count: ` + strconv.Itoa(settings.Mons) + `
|
||||
allowMultiplePerNode: true
|
||||
dashboard:
|
||||
enabled: true
|
||||
metadataDevice:
|
||||
storage:
|
||||
useAllNodes: true
|
||||
useAllDevices: ` + strconv.FormatBool(useAllDevices) + `
|
||||
useAllDevices: ` + strconv.FormatBool(settings.UseAllDevices) + `
|
||||
deviceFilter:
|
||||
location:
|
||||
config:
|
||||
storeType: "` + storeType + `"
|
||||
storeType: "` + settings.StoreType + `"
|
||||
databaseSizeMB: "1024"
|
||||
journalSizeMB: "1024"`
|
||||
}
|
||||
|
||||
//GetRookToolBox returns rook-toolbox manifest
|
||||
func (i *InstallData) GetRookToolBox(namespace string) string {
|
||||
func (m *CephManifestsMaster) GetRookToolBox(namespace string) string {
|
||||
return `apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
@@ -419,7 +449,7 @@ spec:
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
containers:
|
||||
- name: rook-ceph-tools
|
||||
image: rook/ceph-toolbox:master
|
||||
image: rook/ceph-toolbox:` + m.imageTag + `
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: ROOK_ADMIN_SECRET
|
||||
@@ -458,7 +488,7 @@ spec:
|
||||
}
|
||||
|
||||
//GetCleanupPod gets a cleanup Pod manifest
|
||||
func (i *InstallData) GetCleanupPod(node, removalDir string) string {
|
||||
func (m *CephManifestsMaster) GetCleanupPod(node, removalDir string) string {
|
||||
return `apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
@@ -469,7 +499,7 @@ spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: rook-cleaner
|
||||
image: rook/rook:master
|
||||
image: rook/ceph:` + m.imageTag + `
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
@@ -486,3 +516,94 @@ spec:
|
||||
hostPath:
|
||||
path: ` + removalDir
|
||||
}
|
||||
|
||||
func (m *CephManifestsMaster) GetBlockPoolDef(poolName string, namespace string, replicaSize string) string {
|
||||
return `apiVersion: ceph.rook.io/v1beta1
|
||||
kind: Pool
|
||||
metadata:
|
||||
name: ` + poolName + `
|
||||
namespace: ` + namespace + `
|
||||
spec:
|
||||
replicated:
|
||||
size: ` + replicaSize
|
||||
}
|
||||
|
||||
func (m *CephManifestsMaster) GetBlockStorageClassDef(poolName string, storageClassName string, namespace string, varClusterName bool) string {
|
||||
namespaceParameter := "clusterNamespace"
|
||||
if varClusterName {
|
||||
namespaceParameter = "clusterName"
|
||||
}
|
||||
return `apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: ` + storageClassName + `
|
||||
provisioner: ceph.rook.io/block
|
||||
parameters:
|
||||
pool: ` + poolName + `
|
||||
` + namespaceParameter + `: ` + namespace
|
||||
}
|
||||
|
||||
func (m *CephManifestsMaster) GetBlockPvcDef(claimName string, storageClassName string, accessModes string) string {
|
||||
return `apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: ` + claimName + `
|
||||
annotations:
|
||||
volume.beta.kubernetes.io/storage-class: ` + storageClassName + `
|
||||
spec:
|
||||
accessModes:
|
||||
- ` + accessModes + `
|
||||
resources:
|
||||
requests:
|
||||
storage: 1M`
|
||||
}
|
||||
|
||||
func (m *CephManifestsMaster) GetBlockPoolStorageClassAndPvcDef(namespace string, poolName string, storageClassName string, blockName string, accessMode string) string {
|
||||
return concatYaml(m.GetBlockPoolDef(poolName, namespace, "1"),
|
||||
concatYaml(m.GetBlockStorageClassDef(poolName, storageClassName, namespace, false), m.GetBlockPvcDef(blockName, storageClassName, accessMode)))
|
||||
}
|
||||
|
||||
func (m *CephManifestsMaster) GetBlockPoolStorageClass(namespace string, poolName string, storageClassName string) string {
|
||||
return concatYaml(m.GetBlockPoolDef(poolName, namespace, "1"), m.GetBlockStorageClassDef(poolName, storageClassName, namespace, false))
|
||||
}
|
||||
|
||||
func (m *CephManifestsMaster) GetFilesystem(namespace, name string) string {
|
||||
return `apiVersion: ceph.rook.io/v1beta1
|
||||
kind: Filesystem
|
||||
metadata:
|
||||
name: ` + name + `
|
||||
namespace: ` + namespace + `
|
||||
spec:
|
||||
metadataPool:
|
||||
replicated:
|
||||
size: 1
|
||||
dataPools:
|
||||
- replicated:
|
||||
size: 1
|
||||
metadataServer:
|
||||
activeCount: 1
|
||||
activeStandby: true`
|
||||
}
|
||||
|
||||
func (m *CephManifestsMaster) GetObjectStore(namespace, name string, replicaCount, port int) string {
|
||||
return `apiVersion: ceph.rook.io/v1beta1
|
||||
kind: ObjectStore
|
||||
metadata:
|
||||
name: ` + name + `
|
||||
namespace: ` + namespace + `
|
||||
spec:
|
||||
metadataPool:
|
||||
replicated:
|
||||
size: 1
|
||||
dataPool:
|
||||
replicated:
|
||||
size: 1
|
||||
gateway:
|
||||
type: s3
|
||||
sslCertificateRef:
|
||||
port: ` + strconv.Itoa(port) + `
|
||||
securePort:
|
||||
instances: ` + strconv.Itoa(replicaCount) + `
|
||||
allNodes: false
|
||||
`
|
||||
}
|
||||
@@ -1,270 +0,0 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package installer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func (h *InstallHelper) InstallCockroachDB(systemNamespace, namespace string, count int) error {
|
||||
h.k8shelper.CreateAnonSystemClusterBinding()
|
||||
|
||||
// install hostpath provisioner if there isn't already a default storage class
|
||||
defaultExists, err := h.k8shelper.IsDefaultStorageClassPresent()
|
||||
if err != nil {
|
||||
return err
|
||||
} else if !defaultExists {
|
||||
if err := h.InstallHostPathProvisioner(); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
logger.Info("skipping install of host path provisioner because a default storage class already exists")
|
||||
}
|
||||
|
||||
// install cockroachdb operator
|
||||
if err := h.CreateCockroachDBOperator(systemNamespace); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// install cockroachdb cluster instance
|
||||
if err := h.CreateCockroachDBCluster(namespace, count); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *InstallHelper) CreateCockroachDBOperator(namespace string) error {
|
||||
logger.Infof("starting cockroachDB operator")
|
||||
|
||||
logger.Info("creating cockroachDB CRDs")
|
||||
if _, err := h.k8shelper.KubectlWithStdin(h.installData.GetCockroachDBCRDs(), createFromStdinArgs...); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cockroachDBOperator := h.installData.GetCockroachDBOperator(namespace)
|
||||
_, err := h.k8shelper.KubectlWithStdin(cockroachDBOperator, createFromStdinArgs...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create rook-cockroachdb-operator pod: %+v ", err)
|
||||
}
|
||||
|
||||
if !h.k8shelper.IsCRDPresent(cockroachDBCRD) {
|
||||
return fmt.Errorf("failed to find cockroachdb CRD %s", cockroachDBCRD)
|
||||
}
|
||||
|
||||
if !h.k8shelper.IsPodInExpectedState("rook-cockroachdb-operator", namespace, "Running") {
|
||||
return fmt.Errorf("rook-cockroachdb-operator is not running, aborting")
|
||||
}
|
||||
|
||||
logger.Infof("cockroachdb operator started")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *InstallHelper) CreateCockroachDBCluster(namespace string, count int) error {
|
||||
if err := h.k8shelper.CreateNamespace(namespace); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Infof("starting cockroachdb cluster with kubectl and yaml")
|
||||
cockroachDBCluster := h.installData.GetCockroachDBCluster(namespace, count)
|
||||
if _, err := h.k8shelper.KubectlWithStdin(cockroachDBCluster, createFromStdinArgs...); err != nil {
|
||||
return fmt.Errorf("Failed to create cockroachdb cluster: %+v ", err)
|
||||
}
|
||||
|
||||
if err := h.k8shelper.WaitForPodCount("app=rook-cockroachdb", namespace, count); err != nil {
|
||||
logger.Errorf("cockroachdb cluster pods in namespace %s not found", namespace)
|
||||
return err
|
||||
}
|
||||
|
||||
err := h.k8shelper.WaitForLabeledPodToRun("app=rook-cockroachdb", namespace)
|
||||
if err != nil {
|
||||
logger.Errorf("cockroachdb cluster pods in namespace %s are not running", namespace)
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Infof("cockroachdb cluster started")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *InstallHelper) UninstallCockroachDB(systemNamespace, namespace string) {
|
||||
logger.Infof("uninstalling cockroachdb from namespace %s", namespace)
|
||||
|
||||
_, err := h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cluster.cockroachdb.rook.io", namespace)
|
||||
h.checkError(err, fmt.Sprintf("cannot remove cluster %s", namespace))
|
||||
|
||||
crdCheckerFunc := func() error {
|
||||
_, err := h.k8shelper.RookClientset.CockroachdbV1alpha1().Clusters(namespace).Get(namespace, metav1.GetOptions{})
|
||||
return err
|
||||
}
|
||||
err = h.waitForCustomResourceDeletion(namespace, crdCheckerFunc)
|
||||
h.checkError(err, fmt.Sprintf("failed to wait for crd %s deletion", namespace))
|
||||
|
||||
_, err = h.k8shelper.DeleteResourceAndWait(false, "namespace", namespace)
|
||||
h.checkError(err, fmt.Sprintf("cannot delete namespace %s", namespace))
|
||||
|
||||
logger.Infof("removing the operator from namespace %s", systemNamespace)
|
||||
_, err = h.k8shelper.DeleteResource("crd", "clusters.cockroachdb.rook.io")
|
||||
h.checkError(err, "cannot delete CRDs")
|
||||
|
||||
cockroachDBOperator := h.installData.GetCockroachDBOperator(systemNamespace)
|
||||
_, err = h.k8shelper.KubectlWithStdin(cockroachDBOperator, deleteFromStdinArgs...)
|
||||
h.checkError(err, "cannot uninstall rook-cockroachdb-operator")
|
||||
|
||||
err = h.UninstallHostPathProvisioner()
|
||||
h.checkError(err, "cannot uninstall hostpath provisioner")
|
||||
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
|
||||
logger.Infof("done removing the operator from namespace %s", systemNamespace)
|
||||
}
|
||||
|
||||
func (h *InstallHelper) GatherAllCockroachDBLogs(systemNamespace, namespace, testName string) {
|
||||
logger.Infof("Gathering all logs from cockroachdb cluster %s", namespace)
|
||||
h.k8shelper.GetRookLogs("rook-cockroachdb-operator", h.Env.HostType, systemNamespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-cockroachdb", h.Env.HostType, namespace, testName)
|
||||
}
|
||||
|
||||
func (i *InstallData) GetCockroachDBCRDs() string {
|
||||
return `apiVersion: apiextensions.k8s.io/v1beta1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
name: clusters.cockroachdb.rook.io
|
||||
spec:
|
||||
group: cockroachdb.rook.io
|
||||
names:
|
||||
kind: Cluster
|
||||
listKind: ClusterList
|
||||
plural: clusters
|
||||
singular: cluster
|
||||
scope: Namespaced
|
||||
version: v1alpha1
|
||||
`
|
||||
}
|
||||
|
||||
func (i *InstallData) GetCockroachDBOperator(namespace string) string {
|
||||
|
||||
return `kind: Namespace
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: ` + namespace + `
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: rook-cockroachdb-operator
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- services
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- create
|
||||
- update
|
||||
- apiGroups:
|
||||
- apps
|
||||
resources:
|
||||
- statefulsets
|
||||
verbs:
|
||||
- create
|
||||
- apiGroups:
|
||||
- policy
|
||||
resources:
|
||||
- poddisruptionbudgets
|
||||
verbs:
|
||||
- create
|
||||
- apiGroups:
|
||||
- cockroachdb.rook.io
|
||||
resources:
|
||||
- "*"
|
||||
verbs:
|
||||
- "*"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-cockroachdb-operator
|
||||
namespace: ` + namespace + `
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-cockroachdb-operator
|
||||
namespace: ` + namespace + `
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: rook-cockroachdb-operator
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-cockroachdb-operator
|
||||
namespace: ` + namespace + `
|
||||
---
|
||||
apiVersion: apps/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: rook-cockroachdb-operator
|
||||
namespace: ` + namespace + `
|
||||
spec:
|
||||
replicas: 1
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: rook-cockroachdb-operator
|
||||
spec:
|
||||
serviceAccountName: rook-cockroachdb-operator
|
||||
containers:
|
||||
- name: rook-cockroachdb-operator
|
||||
image: rook/cockroachdb:master
|
||||
args: ["cockroachdb", "operator"]
|
||||
env:
|
||||
- name: POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
`
|
||||
}
|
||||
|
||||
func (i *InstallData) GetCockroachDBCluster(namespace string, count int) string {
|
||||
return `apiVersion: cockroachdb.rook.io/v1alpha1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: ` + namespace + `
|
||||
namespace: ` + namespace + `
|
||||
spec:
|
||||
scope:
|
||||
nodeCount: ` + strconv.Itoa(count) + `
|
||||
secure: false
|
||||
volumeSize: 1Gi
|
||||
cachePercent: 25
|
||||
maxSQLMemoryPercent: 25
|
||||
`
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package installer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
cockroachDBCRD = "clusters.cockroachdb.rook.io"
|
||||
)
|
||||
|
||||
type CockroachDBInstaller struct {
|
||||
k8shelper *utils.K8sHelper
|
||||
manifests *CockroachDBManifests
|
||||
T func() *testing.T
|
||||
}
|
||||
|
||||
func NewCockroachDBInstaller(k8shelper *utils.K8sHelper, t func() *testing.T) *CockroachDBInstaller {
|
||||
return &CockroachDBInstaller{k8shelper, &CockroachDBManifests{}, t}
|
||||
}
|
||||
|
||||
func (h *CockroachDBInstaller) InstallCockroachDB(systemNamespace, namespace string, count int) error {
|
||||
h.k8shelper.CreateAnonSystemClusterBinding()
|
||||
|
||||
// install hostpath provisioner if there isn't already a default storage class
|
||||
defaultExists, err := h.k8shelper.IsDefaultStorageClassPresent()
|
||||
if err != nil {
|
||||
return err
|
||||
} else if !defaultExists {
|
||||
if err := InstallHostPathProvisioner(h.k8shelper); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
logger.Info("skipping install of host path provisioner because a default storage class already exists")
|
||||
}
|
||||
|
||||
// install cockroachdb operator
|
||||
if err := h.CreateCockroachDBOperator(systemNamespace); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// install cockroachdb cluster instance
|
||||
if err := h.CreateCockroachDBCluster(namespace, count); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *CockroachDBInstaller) CreateCockroachDBOperator(namespace string) error {
|
||||
logger.Infof("starting cockroachDB operator")
|
||||
|
||||
logger.Info("creating cockroachDB CRDs")
|
||||
if _, err := h.k8shelper.KubectlWithStdin(h.manifests.GetCockroachDBCRDs(), createFromStdinArgs...); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cockroachDBOperator := h.manifests.GetCockroachDBOperator(namespace)
|
||||
_, err := h.k8shelper.KubectlWithStdin(cockroachDBOperator, createFromStdinArgs...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create rook-cockroachdb-operator pod: %+v ", err)
|
||||
}
|
||||
|
||||
if !h.k8shelper.IsCRDPresent(cockroachDBCRD) {
|
||||
return fmt.Errorf("failed to find cockroachdb CRD %s", cockroachDBCRD)
|
||||
}
|
||||
|
||||
if !h.k8shelper.IsPodInExpectedState("rook-cockroachdb-operator", namespace, "Running") {
|
||||
return fmt.Errorf("rook-cockroachdb-operator is not running, aborting")
|
||||
}
|
||||
|
||||
logger.Infof("cockroachdb operator started")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *CockroachDBInstaller) CreateCockroachDBCluster(namespace string, count int) error {
|
||||
if err := h.k8shelper.CreateNamespace(namespace); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Infof("starting cockroachdb cluster with kubectl and yaml")
|
||||
cockroachDBCluster := h.manifests.GetCockroachDBCluster(namespace, count)
|
||||
if _, err := h.k8shelper.KubectlWithStdin(cockroachDBCluster, createFromStdinArgs...); err != nil {
|
||||
return fmt.Errorf("Failed to create cockroachdb cluster: %+v ", err)
|
||||
}
|
||||
|
||||
if err := h.k8shelper.WaitForPodCount("app=rook-cockroachdb", namespace, count); err != nil {
|
||||
logger.Errorf("cockroachdb cluster pods in namespace %s not found", namespace)
|
||||
return err
|
||||
}
|
||||
|
||||
err := h.k8shelper.WaitForLabeledPodToRun("app=rook-cockroachdb", namespace)
|
||||
if err != nil {
|
||||
logger.Errorf("cockroachdb cluster pods in namespace %s are not running", namespace)
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Infof("cockroachdb cluster started")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *CockroachDBInstaller) UninstallCockroachDB(systemNamespace, namespace string) {
|
||||
logger.Infof("uninstalling cockroachdb from namespace %s", namespace)
|
||||
|
||||
_, err := h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cluster.cockroachdb.rook.io", namespace)
|
||||
checkError(h.T(), err, fmt.Sprintf("cannot remove cluster %s", namespace))
|
||||
|
||||
crdCheckerFunc := func() error {
|
||||
_, err := h.k8shelper.RookClientset.CockroachdbV1alpha1().Clusters(namespace).Get(namespace, metav1.GetOptions{})
|
||||
return err
|
||||
}
|
||||
err = h.k8shelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc)
|
||||
checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace))
|
||||
|
||||
_, err = h.k8shelper.DeleteResourceAndWait(false, "namespace", namespace)
|
||||
checkError(h.T(), err, fmt.Sprintf("cannot delete namespace %s", namespace))
|
||||
|
||||
logger.Infof("removing the operator from namespace %s", systemNamespace)
|
||||
_, err = h.k8shelper.DeleteResource("crd", "clusters.cockroachdb.rook.io")
|
||||
checkError(h.T(), err, "cannot delete CRDs")
|
||||
|
||||
cockroachDBOperator := h.manifests.GetCockroachDBOperator(systemNamespace)
|
||||
_, err = h.k8shelper.KubectlWithStdin(cockroachDBOperator, deleteFromStdinArgs...)
|
||||
checkError(h.T(), err, "cannot uninstall rook-cockroachdb-operator")
|
||||
|
||||
err = UninstallHostPathProvisioner(h.k8shelper)
|
||||
checkError(h.T(), err, "cannot uninstall hostpath provisioner")
|
||||
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
|
||||
logger.Infof("done removing the operator from namespace %s", systemNamespace)
|
||||
}
|
||||
|
||||
func (h *CockroachDBInstaller) GatherAllCockroachDBLogs(systemNamespace, namespace, testName string) {
|
||||
logger.Infof("Gathering all logs from cockroachdb cluster %s", namespace)
|
||||
h.k8shelper.GetRookLogs("rook-cockroachdb-operator", Env.HostType, systemNamespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-cockroachdb", Env.HostType, namespace, testName)
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package installer
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type CockroachDBManifests struct {
|
||||
}
|
||||
|
||||
func (i *CockroachDBManifests) GetCockroachDBCRDs() string {
|
||||
return `apiVersion: apiextensions.k8s.io/v1beta1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
name: clusters.cockroachdb.rook.io
|
||||
spec:
|
||||
group: cockroachdb.rook.io
|
||||
names:
|
||||
kind: Cluster
|
||||
listKind: ClusterList
|
||||
plural: clusters
|
||||
singular: cluster
|
||||
scope: Namespaced
|
||||
version: v1alpha1
|
||||
`
|
||||
}
|
||||
|
||||
func (i *CockroachDBManifests) GetCockroachDBOperator(namespace string) string {
|
||||
|
||||
return `kind: Namespace
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: ` + namespace + `
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: rook-cockroachdb-operator
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- services
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- create
|
||||
- update
|
||||
- apiGroups:
|
||||
- apps
|
||||
resources:
|
||||
- statefulsets
|
||||
verbs:
|
||||
- create
|
||||
- apiGroups:
|
||||
- policy
|
||||
resources:
|
||||
- poddisruptionbudgets
|
||||
verbs:
|
||||
- create
|
||||
- apiGroups:
|
||||
- cockroachdb.rook.io
|
||||
resources:
|
||||
- "*"
|
||||
verbs:
|
||||
- "*"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-cockroachdb-operator
|
||||
namespace: ` + namespace + `
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-cockroachdb-operator
|
||||
namespace: ` + namespace + `
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: rook-cockroachdb-operator
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-cockroachdb-operator
|
||||
namespace: ` + namespace + `
|
||||
---
|
||||
apiVersion: apps/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: rook-cockroachdb-operator
|
||||
namespace: ` + namespace + `
|
||||
spec:
|
||||
replicas: 1
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: rook-cockroachdb-operator
|
||||
spec:
|
||||
serviceAccountName: rook-cockroachdb-operator
|
||||
containers:
|
||||
- name: rook-cockroachdb-operator
|
||||
image: rook/cockroachdb:master
|
||||
args: ["cockroachdb", "operator"]
|
||||
env:
|
||||
- name: POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
`
|
||||
}
|
||||
|
||||
func (i *CockroachDBManifests) GetCockroachDBCluster(namespace string, count int) string {
|
||||
return `apiVersion: cockroachdb.rook.io/v1alpha1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: ` + namespace + `
|
||||
namespace: ` + namespace + `
|
||||
spec:
|
||||
scope:
|
||||
nodeCount: ` + strconv.Itoa(count) + `
|
||||
secure: false
|
||||
volumeSize: 1Gi
|
||||
cachePercent: 25
|
||||
maxSQLMemoryPercent: 25
|
||||
`
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package installer
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/rook/rook/pkg/util/exec"
|
||||
"github.com/rook/rook/pkg/util/sys"
|
||||
)
|
||||
|
||||
func IsAdditionalDeviceAvailableOnCluster() bool {
|
||||
executor := &exec.CommandExecutor{}
|
||||
devices, err := sys.ListDevices(executor)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
disks := 0
|
||||
logger.Infof("devices : %v", devices)
|
||||
for _, device := range devices {
|
||||
if strings.Contains(device, "loop") {
|
||||
continue
|
||||
}
|
||||
props, _ := sys.GetDeviceProperties(device, executor)
|
||||
if props["TYPE"] != "disk" {
|
||||
continue
|
||||
}
|
||||
|
||||
ownPartitions, fs, err := sys.CheckIfDeviceAvailable(executor, device)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to detect device %s availability. %+v", device, err)
|
||||
continue
|
||||
}
|
||||
if !ownPartitions {
|
||||
logger.Infof("skipping device %s since don't own partitions", device)
|
||||
continue
|
||||
}
|
||||
if fs != "" {
|
||||
logger.Infof("skipping device %s since it has file system %s", device, fs)
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(device, "rbd") {
|
||||
logger.Infof("skipping unexpected rbd device: %s", device)
|
||||
continue
|
||||
}
|
||||
logger.Infof("available device: %s", device)
|
||||
disks++
|
||||
}
|
||||
if disks > 0 {
|
||||
return true
|
||||
}
|
||||
logger.Info("No additional disks found on cluster")
|
||||
return false
|
||||
}
|
||||
+2
-2
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package objects
|
||||
package installer
|
||||
|
||||
import (
|
||||
"flag"
|
||||
@@ -37,7 +37,7 @@ var Env EnvironmentManifest
|
||||
|
||||
func init() {
|
||||
Env = EnvironmentManifest{}
|
||||
flag.StringVar(&Env.HostType, "host_type", "localhost", "Host were tests are run eg - localhost,GCE or AWS")
|
||||
flag.StringVar(&Env.HostType, "host_type", "localhost", "Host where tests are run eg - localhost,GCE or AWS")
|
||||
flag.StringVar(&Env.RookImageName, "rook_image", "rook/ceph", "Docker image name for the rook container to install, must be in docker hub or local environment")
|
||||
flag.StringVar(&Env.ToolboxImageName, "toolbox_image", "rook/ceph-toolbox", "Docker image name of the toolbox container to install, must be in docker hub or local environment")
|
||||
flag.BoolVar(&Env.SkipInstallRook, "skip_install_rook", false, "Indicate if Rook need to installed - false if tests are being running at Rook that is pre-installed")
|
||||
@@ -0,0 +1,90 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package installer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
// Version tag for the latest manifests
|
||||
VersionMaster = "master"
|
||||
)
|
||||
|
||||
var (
|
||||
// ** Variables that might need to be changed depending on the dev environment. The init function below will modify some of them automatically. **
|
||||
baseTestDir string
|
||||
forceUseDevices = false
|
||||
createBaseTestDir = true
|
||||
// ** end of Variables to modify
|
||||
logger = capnslog.NewPackageLogger("github.com/rook/rook", "installer")
|
||||
createArgs = []string{"create", "-f"}
|
||||
createFromStdinArgs = append(createArgs, "-")
|
||||
deleteArgs = []string{"delete", "-f"}
|
||||
deleteFromStdinArgs = append(deleteArgs, "-")
|
||||
)
|
||||
|
||||
type TestSuite interface {
|
||||
Setup()
|
||||
Teardown()
|
||||
}
|
||||
|
||||
func init() {
|
||||
// this default will only work if running kubernetes on the local machine
|
||||
baseTestDir, _ = os.Getwd()
|
||||
|
||||
// The following settings could apply to any environment when the kube context is running on the host and the tests are running inside a
|
||||
// VM such as minikube. This is a cheap test for this condition, we need to find a better way to automate these settings.
|
||||
if runtime.GOOS == "darwin" {
|
||||
createBaseTestDir = false
|
||||
baseTestDir = "/data"
|
||||
}
|
||||
}
|
||||
|
||||
func SystemNamespace(namespace string) string {
|
||||
return fmt.Sprintf("%s-system", namespace)
|
||||
}
|
||||
|
||||
func checkError(t *testing.T, err error, message string) {
|
||||
// During cleanup the resource might not be found because the test might have failed before the test was done and never created the resource
|
||||
if err == nil || errors.IsNotFound(err) {
|
||||
return
|
||||
}
|
||||
assert.NoError(t, err, "%s. %+v", message, err)
|
||||
}
|
||||
|
||||
func concatYaml(first, second string) string {
|
||||
return first + `
|
||||
---
|
||||
` + second
|
||||
}
|
||||
|
||||
// GatherCRDObjectDebuggingInfo gathers all the descriptions for pods, pvs and pvcs
|
||||
func GatherCRDObjectDebuggingInfo(k8shelper *utils.K8sHelper, namespace string) {
|
||||
k8shelper.PrintPodDescribeForNamespace(namespace)
|
||||
k8shelper.PrintPVs(true /*detailed*/)
|
||||
k8shelper.PrintPVCs(namespace, true /*detailed*/)
|
||||
k8shelper.PrintStorageClasses(true /*detailed*/)
|
||||
}
|
||||
@@ -18,12 +18,28 @@ package installer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
nfsServerCRD = "nfsservers.nfs.rook.io"
|
||||
)
|
||||
|
||||
type NFSInstaller struct {
|
||||
k8shelper *utils.K8sHelper
|
||||
manifests *NFSManifests
|
||||
T func() *testing.T
|
||||
}
|
||||
|
||||
func NewNFSInstaller(k8shelper *utils.K8sHelper, t func() *testing.T) *NFSInstaller {
|
||||
return &NFSInstaller{k8shelper, &NFSManifests{}, t}
|
||||
}
|
||||
|
||||
// InstallNFSServer installs NFS operator, NFS CRD instance and NFS volume
|
||||
func (h *InstallHelper) InstallNFSServer(systemNamespace, namespace string, count int) error {
|
||||
func (h *NFSInstaller) InstallNFSServer(systemNamespace, namespace string, count int) error {
|
||||
h.k8shelper.CreateAnonSystemClusterBinding()
|
||||
|
||||
// install hostpath provisioner if there isn't already a default storage class
|
||||
@@ -32,7 +48,7 @@ func (h *InstallHelper) InstallNFSServer(systemNamespace, namespace string, coun
|
||||
if err != nil {
|
||||
return err
|
||||
} else if !defaultExists {
|
||||
if err := h.InstallHostPathProvisioner(); err != nil {
|
||||
if err := InstallHostPathProvisioner(h.k8shelper); err != nil {
|
||||
return err
|
||||
}
|
||||
storageClassName = "hostpath"
|
||||
@@ -59,15 +75,15 @@ func (h *InstallHelper) InstallNFSServer(systemNamespace, namespace string, coun
|
||||
}
|
||||
|
||||
// CreateNFSServerOperator creates nfs server in the provided namespace
|
||||
func (h *InstallHelper) CreateNFSServerOperator(namespace string) error {
|
||||
func (h *NFSInstaller) CreateNFSServerOperator(namespace string) error {
|
||||
logger.Infof("starting nfsserver operator")
|
||||
|
||||
logger.Info("creating nfsserver CRDs")
|
||||
if _, err := h.k8shelper.KubectlWithStdin(h.installData.GetNFSServerCRDs(), createFromStdinArgs...); err != nil {
|
||||
if _, err := h.k8shelper.KubectlWithStdin(h.manifests.GetNFSServerCRDs(), createFromStdinArgs...); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
nfsOperator := h.installData.GetNFSServerOperator(namespace)
|
||||
nfsOperator := h.manifests.GetNFSServerOperator(namespace)
|
||||
_, err := h.k8shelper.KubectlWithStdin(nfsOperator, createFromStdinArgs...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create rook-nfs-operator pod: %+v ", err)
|
||||
@@ -86,13 +102,13 @@ func (h *InstallHelper) CreateNFSServerOperator(namespace string) error {
|
||||
}
|
||||
|
||||
// CreateNFSServer creates the NFS Server CRD instance
|
||||
func (h *InstallHelper) CreateNFSServer(namespace string, count int, storageClassName string) error {
|
||||
func (h *NFSInstaller) CreateNFSServer(namespace string, count int, storageClassName string) error {
|
||||
if err := h.k8shelper.CreateNamespace(namespace); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Infof("starting nfs server with kubectl and yaml")
|
||||
nfsServer := h.installData.GetNFSServer(namespace, count, storageClassName)
|
||||
nfsServer := h.manifests.GetNFSServer(namespace, count, storageClassName)
|
||||
if _, err := h.k8shelper.KubectlWithStdin(nfsServer, createFromStdinArgs...); err != nil {
|
||||
return fmt.Errorf("Failed to create nfs server: %+v ", err)
|
||||
}
|
||||
@@ -113,15 +129,15 @@ func (h *InstallHelper) CreateNFSServer(namespace string, count int, storageClas
|
||||
}
|
||||
|
||||
// CreateNFSServerVolume creates NFS export PV and PVC
|
||||
func (h *InstallHelper) CreateNFSServerVolume(namespace string) error {
|
||||
func (h *NFSInstaller) CreateNFSServerVolume(namespace string) error {
|
||||
logger.Info("creating volume from nfs server in namespace %s", namespace)
|
||||
|
||||
clusterIP, err := h.GetNFSServerClusterIP(namespace)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nfsServerPV := h.installData.GetNFSServerPV(namespace, clusterIP)
|
||||
nfsServerPVC := h.installData.GetNFSServerPVC()
|
||||
nfsServerPV := h.manifests.GetNFSServerPV(namespace, clusterIP)
|
||||
nfsServerPVC := h.manifests.GetNFSServerPVC()
|
||||
|
||||
logger.Info("creating nfs server pv")
|
||||
if _, err := h.k8shelper.KubectlWithStdin(nfsServerPV, createFromStdinArgs...); err != nil {
|
||||
@@ -137,52 +153,52 @@ func (h *InstallHelper) CreateNFSServerVolume(namespace string) error {
|
||||
}
|
||||
|
||||
// UninstallNFSServer uninstalls the NFS Server from the given namespace
|
||||
func (h *InstallHelper) UninstallNFSServer(systemNamespace, namespace string) {
|
||||
func (h *NFSInstaller) UninstallNFSServer(systemNamespace, namespace string) {
|
||||
logger.Infof("uninstalling nfsserver from namespace %s", namespace)
|
||||
|
||||
_, err := h.k8shelper.DeleteResource("pvc", "nfs-pv-claim")
|
||||
h.checkError(err, fmt.Sprintf("cannot remove nfs pvc"))
|
||||
checkError(h.T(), err, fmt.Sprintf("cannot remove nfs pvc"))
|
||||
|
||||
_, err = h.k8shelper.DeleteResource("pv", "nfs-pv")
|
||||
h.checkError(err, fmt.Sprintf("cannot remove nfs pv"))
|
||||
checkError(h.T(), err, fmt.Sprintf("cannot remove nfs pv"))
|
||||
|
||||
_, err = h.k8shelper.DeleteResource("-n", namespace, "nfsservers.nfs.rook.io", namespace)
|
||||
h.checkError(err, fmt.Sprintf("cannot remove nfsserver %s", namespace))
|
||||
checkError(h.T(), err, fmt.Sprintf("cannot remove nfsserver %s", namespace))
|
||||
|
||||
crdCheckerFunc := func() error {
|
||||
_, err := h.k8shelper.RookClientset.NfsV1alpha1().NFSServers(namespace).Get(namespace, metav1.GetOptions{})
|
||||
return err
|
||||
}
|
||||
err = h.waitForCustomResourceDeletion(namespace, crdCheckerFunc)
|
||||
h.checkError(err, fmt.Sprintf("failed to wait for crd %s deletion", namespace))
|
||||
err = h.k8shelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc)
|
||||
checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace))
|
||||
|
||||
_, err = h.k8shelper.DeleteResource("namespace", namespace)
|
||||
h.checkError(err, fmt.Sprintf("cannot delete namespace %s", namespace))
|
||||
checkError(h.T(), err, fmt.Sprintf("cannot delete namespace %s", namespace))
|
||||
|
||||
logger.Infof("removing the operator from namespace %s", systemNamespace)
|
||||
_, err = h.k8shelper.DeleteResource("crd", "nfsservers.nfs.rook.io")
|
||||
h.checkError(err, "cannot delete CRDs")
|
||||
checkError(h.T(), err, "cannot delete CRDs")
|
||||
|
||||
nfsOperator := h.installData.GetNFSServerOperator(systemNamespace)
|
||||
nfsOperator := h.manifests.GetNFSServerOperator(systemNamespace)
|
||||
_, err = h.k8shelper.KubectlWithStdin(nfsOperator, deleteFromStdinArgs...)
|
||||
h.checkError(err, "cannot uninstall rook-nfs-operator")
|
||||
checkError(h.T(), err, "cannot uninstall rook-nfs-operator")
|
||||
|
||||
err = h.UninstallHostPathProvisioner()
|
||||
h.checkError(err, "cannot uninstall hostpath provisioner")
|
||||
err = UninstallHostPathProvisioner(h.k8shelper)
|
||||
checkError(h.T(), err, "cannot uninstall hostpath provisioner")
|
||||
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
|
||||
logger.Infof("done removing the operator from namespace %s", systemNamespace)
|
||||
}
|
||||
|
||||
// GatherAllNFSServerLogs gathers all NFS Server logs
|
||||
func (h *InstallHelper) GatherAllNFSServerLogs(systemNamespace, namespace, testName string) {
|
||||
func (h *NFSInstaller) GatherAllNFSServerLogs(systemNamespace, namespace, testName string) {
|
||||
logger.Infof("Gathering all logs from NFSServer %s", namespace)
|
||||
h.k8shelper.GetRookLogs("rook-nfs-operator", h.Env.HostType, systemNamespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-nfs", h.Env.HostType, namespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-nfs-operator", Env.HostType, systemNamespace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-nfs", Env.HostType, namespace, testName)
|
||||
}
|
||||
|
||||
// GetNFSServerClusterIP gets the nfs server cluster ip on which it serves
|
||||
func (h *InstallHelper) GetNFSServerClusterIP(namespace string) (string, error) {
|
||||
func (h *NFSInstaller) GetNFSServerClusterIP(namespace string) (string, error) {
|
||||
clusterIP := ""
|
||||
service, err := h.k8shelper.GetService("rook-nfs", namespace)
|
||||
if err != nil {
|
||||
+8
-5
@@ -20,8 +20,11 @@ import (
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type NFSManifests struct {
|
||||
}
|
||||
|
||||
// GetNFSServerCRDs returns NFSServer CRD definition
|
||||
func (i *InstallData) GetNFSServerCRDs() string {
|
||||
func (i *NFSManifests) GetNFSServerCRDs() string {
|
||||
return `apiVersion: apiextensions.k8s.io/v1beta1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
@@ -39,7 +42,7 @@ spec:
|
||||
}
|
||||
|
||||
// GetNFSServerOperator returns the NFSServer operator definition
|
||||
func (i *InstallData) GetNFSServerOperator(namespace string) string {
|
||||
func (i *NFSManifests) GetNFSServerOperator(namespace string) string {
|
||||
return `kind: Namespace
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
@@ -131,7 +134,7 @@ spec:
|
||||
}
|
||||
|
||||
// GetNFSServerPV returns NFSServer PV definition
|
||||
func (i *InstallData) GetNFSServerPV(namespace string, clusterIP string) string {
|
||||
func (i *NFSManifests) GetNFSServerPV(namespace string, clusterIP string) string {
|
||||
return `apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
@@ -152,7 +155,7 @@ spec:
|
||||
}
|
||||
|
||||
// GetNFSServerPVC returns NFSServer PVC definition
|
||||
func (i *InstallData) GetNFSServerPVC() string {
|
||||
func (i *NFSManifests) GetNFSServerPVC() string {
|
||||
return `apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
@@ -168,7 +171,7 @@ spec:
|
||||
}
|
||||
|
||||
// GetNFSServer returns NFSServer CRD instance definition
|
||||
func (i *InstallData) GetNFSServer(namespace string, count int, storageClassName string) string {
|
||||
func (i *NFSManifests) GetNFSServer(namespace string, count int, storageClassName string) string {
|
||||
return `apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
@@ -34,68 +34,68 @@ const (
|
||||
// ************************************************************************************************
|
||||
// HostPath provisioner functions
|
||||
// ************************************************************************************************
|
||||
func (h *InstallHelper) InstallHostPathProvisioner() error {
|
||||
func InstallHostPathProvisioner(k8shelper *utils.K8sHelper) error {
|
||||
logger.Info("installing host path provisioner")
|
||||
|
||||
rbacResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerRBAC)
|
||||
args := append(createArgs, rbacResourceURL)
|
||||
out, err := h.k8shelper.Kubectl(args...)
|
||||
out, err := k8shelper.Kubectl(args...)
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create hostpath provisioner RBAC: %+v. %s", err, out)
|
||||
}
|
||||
|
||||
deploymentResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerDeployment)
|
||||
args = append(createArgs, deploymentResourceURL)
|
||||
out, err = h.k8shelper.Kubectl(args...)
|
||||
out, err = k8shelper.Kubectl(args...)
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create hostpath provisioner deployment: %+v. %s", err, out)
|
||||
}
|
||||
|
||||
storageClassResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerStorageClass)
|
||||
args = append(createArgs, storageClassResourceURL)
|
||||
out, err = h.k8shelper.Kubectl(args...)
|
||||
out, err = k8shelper.Kubectl(args...)
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create hostpath provisioner StorageClass: %+v. %s", err, out)
|
||||
}
|
||||
|
||||
err = h.k8shelper.WaitForLabeledPodToRun("k8s-app=hostpath-provisioner", "kube-system")
|
||||
err = k8shelper.WaitForLabeledPodToRun("k8s-app=hostpath-provisioner", "kube-system")
|
||||
if err != nil {
|
||||
logger.Errorf("hostpath provisioner pod is not running: %+v", err)
|
||||
h.k8shelper.PrintPodDescribeForNamespace("kube-system")
|
||||
h.k8shelper.PrintStorageClasses(true /*detailed*/)
|
||||
k8shelper.PrintPodDescribeForNamespace("kube-system")
|
||||
k8shelper.PrintStorageClasses(true /*detailed*/)
|
||||
return err
|
||||
}
|
||||
|
||||
err = h.k8shelper.IsStorageClassPresent(hostPathStorageClassName)
|
||||
err = k8shelper.IsStorageClassPresent(hostPathStorageClassName)
|
||||
if err != nil {
|
||||
logger.Errorf("storageClass %s not found: %+v", hostPathStorageClassName, err)
|
||||
h.k8shelper.PrintStorageClasses(true /*detailed*/)
|
||||
k8shelper.PrintStorageClasses(true /*detailed*/)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *InstallHelper) UninstallHostPathProvisioner() error {
|
||||
func UninstallHostPathProvisioner(k8shelper *utils.K8sHelper) error {
|
||||
logger.Info("uninstalling host path provisioner")
|
||||
|
||||
storageClassResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerStorageClass)
|
||||
args := append(deleteArgs, storageClassResourceURL)
|
||||
out, err := h.k8shelper.Kubectl(args...)
|
||||
out, err := k8shelper.Kubectl(args...)
|
||||
if err != nil && !utils.IsKubectlErrorNotFound(out, err) {
|
||||
return fmt.Errorf("failed to delete hostpath provisioner StorageClass: %+v. %s", err, out)
|
||||
}
|
||||
|
||||
deploymentResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerDeployment)
|
||||
args = append(deleteArgs, deploymentResourceURL)
|
||||
out, err = h.k8shelper.Kubectl(args...)
|
||||
out, err = k8shelper.Kubectl(args...)
|
||||
if err != nil && !utils.IsKubectlErrorNotFound(out, err) {
|
||||
return fmt.Errorf("failed to delete hostpath provisioner deployment: %+v. %s", err, out)
|
||||
}
|
||||
|
||||
rbacResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerRBAC)
|
||||
args = append(deleteArgs, rbacResourceURL)
|
||||
out, err = h.k8shelper.Kubectl(args...)
|
||||
out, err = k8shelper.Kubectl(args...)
|
||||
if err != nil && !utils.IsKubectlErrorNotFound(out, err) {
|
||||
return fmt.Errorf("failed to delete hostpath provisioner RBAC: %+v. %s", err, out)
|
||||
}
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
package installer
|
||||
|
||||
import (
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
)
|
||||
|
||||
func BlockResourceOperation(k8sh *utils.K8sHelper, yaml string, action string) (string, error) {
|
||||
return k8sh.ResourceOperation(action, yaml)
|
||||
}
|
||||
|
||||
func GetBlockPoolDef(poolName string, namespace string, replicaSize string) string {
|
||||
return `apiVersion: ceph.rook.io/v1beta1
|
||||
kind: Pool
|
||||
metadata:
|
||||
name: ` + poolName + `
|
||||
namespace: ` + namespace + `
|
||||
spec:
|
||||
replicated:
|
||||
size: ` + replicaSize
|
||||
}
|
||||
|
||||
func GetBlockStorageClassDef(poolName string, storageClassName string, namespace string, varClusterName bool) string {
|
||||
namespaceParameter := "clusterNamespace"
|
||||
if varClusterName {
|
||||
namespaceParameter = "clusterName"
|
||||
}
|
||||
return `apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: ` + storageClassName + `
|
||||
provisioner: ceph.rook.io/block
|
||||
parameters:
|
||||
pool: ` + poolName + `
|
||||
` + namespaceParameter + `: ` + namespace
|
||||
}
|
||||
|
||||
func GetBlockPvcDef(claimName string, storageClassName string, accessModes string) string {
|
||||
return `apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: ` + claimName + `
|
||||
annotations:
|
||||
volume.beta.kubernetes.io/storage-class: ` + storageClassName + `
|
||||
spec:
|
||||
accessModes:
|
||||
- ` + accessModes + `
|
||||
resources:
|
||||
requests:
|
||||
storage: 1M`
|
||||
}
|
||||
|
||||
func concatYaml(first, second string) string {
|
||||
return first + `
|
||||
---
|
||||
` + second
|
||||
}
|
||||
|
||||
func GetBlockPoolStorageClassAndPvcDef(namespace string, poolName string, storageClassName string, blockName string, accessMode string) string {
|
||||
return concatYaml(GetBlockPoolDef(poolName, namespace, "1"),
|
||||
concatYaml(GetBlockStorageClassDef(poolName, storageClassName, namespace, false), GetBlockPvcDef(blockName, storageClassName, accessMode)))
|
||||
}
|
||||
|
||||
func GetBlockPoolStorageClass(namespace string, poolName string, storageClassName string) string {
|
||||
return concatYaml(GetBlockPoolDef(poolName, namespace, "1"), GetBlockStorageClassDef(poolName, storageClassName, namespace, false))
|
||||
}
|
||||
@@ -290,11 +290,53 @@ func (k8sh *K8sHelper) ResourceOperation(action string, podDefinition string) (s
|
||||
return "", fmt.Errorf("Could Not create resource in args : %v -- %v", args, err)
|
||||
}
|
||||
|
||||
//DeleteResource performs a kubectl delete on give args
|
||||
//DeletePod performs a kubectl delete pod on the given pod
|
||||
func (k8sh *K8sHelper) DeletePod(namespace, name string) (string, error) {
|
||||
args := append([]string{"--grace-period=0", "pod"}, name)
|
||||
if namespace != "" {
|
||||
args = append(args, []string{"-n", namespace}...)
|
||||
}
|
||||
return k8sh.DeleteResourceAndWait(true, args...)
|
||||
}
|
||||
|
||||
//DeletePods performs a kubectl delete pod on the given pods
|
||||
func (k8sh *K8sHelper) DeletePods(pods ...string) (msg string, err error) {
|
||||
for _, pod := range pods {
|
||||
msg, err = k8sh.DeletePod("", pod)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
//DeleteResource performs a kubectl delete on the given args
|
||||
func (k8sh *K8sHelper) DeleteResource(args ...string) (string, error) {
|
||||
return k8sh.DeleteResourceAndWait(true, args...)
|
||||
}
|
||||
|
||||
//WaitForCustomResourceDeletion waits for the CRD deletion
|
||||
func (k8sh *K8sHelper) WaitForCustomResourceDeletion(namespace string, checkerFunc func() error) error {
|
||||
if !k8sh.VersionAtLeast("v1.8.0") {
|
||||
// v1.7 has an intermittent issue with long delay to delete resources so we will skip waiting
|
||||
return nil
|
||||
}
|
||||
|
||||
// wait for the operator to finalize and delete the CRD
|
||||
for i := 0; i < 10; i++ {
|
||||
err := checkerFunc()
|
||||
if err == nil {
|
||||
logger.Infof("custom resource %s still exists", namespace)
|
||||
time.Sleep(2 * time.Second)
|
||||
continue
|
||||
}
|
||||
if errors.IsNotFound(err) {
|
||||
logger.Infof("custom resource %s deleted", namespace)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
logger.Errorf("gave up deleting custom resource %s", namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteResource performs a kubectl delete on give args.
|
||||
// If wait is false, a flag will be passed to indicate the delete should return immediately
|
||||
func (k8sh *K8sHelper) DeleteResourceAndWait(wait bool, args ...string) (string, error) {
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
@@ -33,18 +34,13 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
var (
|
||||
blockMountPath = "/tmp/rook1"
|
||||
blockPodName = "block-test"
|
||||
)
|
||||
|
||||
// Smoke Test for Block Storage - Test check the following operations on Block Storage in order
|
||||
//Create,Mount,Write,Read,Unmount and Delete.
|
||||
func runBlockE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
|
||||
podName := "block-test"
|
||||
poolName := "replicapool"
|
||||
storageClassName := "rook-ceph-block"
|
||||
blockName := "block-pv-claim"
|
||||
podName := "block-test"
|
||||
|
||||
defer blockTestDataCleanUp(helper, k8sh, namespace, poolName, storageClassName, blockName, podName)
|
||||
logger.Infof("Block Storage End to End Integration Test - create, mount, write to, read from, and unmount")
|
||||
@@ -54,28 +50,22 @@ func runBlockE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.
|
||||
initBlockImages, _ := helper.BlockClient.List(namespace)
|
||||
|
||||
logger.Infof("step 1: Create block storage")
|
||||
_, cbErr := installer.BlockResourceOperation(k8sh, installer.GetBlockPoolStorageClassAndPvcDef(namespace, poolName, storageClassName, blockName, "ReadWriteOnce"), "create")
|
||||
_, cbErr := helper.PoolClient.CreateStorageClassAndPvc(namespace, poolName, storageClassName, blockName, "ReadWriteOnce")
|
||||
require.Nil(s.T(), cbErr)
|
||||
require.True(s.T(), retryBlockImageCountCheck(helper, len(initBlockImages), 1, namespace), "Make sure a new block is created")
|
||||
logger.Infof("Block Storage created successfully")
|
||||
require.True(s.T(), k8sh.WaitUntilPVCIsBound(defaultNamespace, blockName), "Make sure PVC is Bound")
|
||||
|
||||
logger.Infof("step 2: Mount block storage")
|
||||
_, mtErr := helper.BlockClient.BlockMap(getBlockPodDefintion(podName, blockName, false), blockMountPath)
|
||||
require.Nil(s.T(), mtErr)
|
||||
crdName, err := k8sh.GetVolumeResourceName(defaultNamespace, blockName)
|
||||
require.Nil(s.T(), err)
|
||||
require.True(s.T(), k8sh.IsVolumeResourcePresent(installer.SystemNamespace(namespace), crdName), fmt.Sprintf("make sure Volume %s is created", crdName))
|
||||
require.True(s.T(), k8sh.IsPodRunning(blockPodName, defaultNamespace), "make sure block-test pod is in running state")
|
||||
logger.Infof("Block Storage Mounted successfully")
|
||||
crdName := createPodWithBlock(helper, k8sh, s, namespace, blockName, podName)
|
||||
|
||||
logger.Infof("step 3: Write to block storage")
|
||||
_, wtErr := helper.BlockClient.Write(blockPodName, blockMountPath, "Smoke Test Data form Block storage", "bsFile1", "")
|
||||
_, wtErr := helper.BlockClient.Write(podName, "Smoke Test Data form Block storage", "bsFile1", "")
|
||||
require.Nil(s.T(), wtErr)
|
||||
logger.Infof("Write to Block storage successfully")
|
||||
|
||||
logger.Infof("step 4: Read from block storage")
|
||||
read, rErr := helper.BlockClient.Read(blockPodName, blockMountPath, "bsFile1", "")
|
||||
read, rErr := helper.BlockClient.Read(podName, "bsFile1", "")
|
||||
require.Nil(s.T(), rErr)
|
||||
assert.Contains(s.T(), read, "Smoke Test Data form Block storage", "make sure content of the files is unchanged")
|
||||
logger.Infof("Read from Block storage successfully")
|
||||
@@ -84,38 +74,49 @@ func runBlockE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.
|
||||
restartOSDPods(k8sh, s, namespace)
|
||||
|
||||
logger.Infof("step 6: Read from block storage again")
|
||||
read, rErr = helper.BlockClient.Read(blockPodName, blockMountPath, "bsFile1", "")
|
||||
read, rErr = helper.BlockClient.Read(podName, "bsFile1", "")
|
||||
require.Nil(s.T(), rErr)
|
||||
assert.Contains(s.T(), read, "Smoke Test Data form Block storage", "make sure content of the files is unchanged")
|
||||
logger.Infof("Read from Block storage successfully")
|
||||
|
||||
logger.Infof("step 7: Mount same block storage on a different pod. Should not be allowed")
|
||||
otherPod := "block-test2"
|
||||
_, mtErr = helper.BlockClient.BlockMap(getBlockPodDefintion(otherPod, blockName, false), blockMountPath)
|
||||
_, mtErr := helper.BlockClient.BlockMap(getBlockPodDefintion(otherPod, blockName, false))
|
||||
require.Nil(s.T(), mtErr)
|
||||
require.True(s.T(), k8sh.IsPodInError(otherPod, defaultNamespace, "FailedMount", "Volume is already attached by pod"), "make sure block-test2 pod errors out while mounting the volume")
|
||||
logger.Infof("Block Storage successfully fenced")
|
||||
|
||||
logger.Infof("step 8: Delete fenced pod")
|
||||
_, unmtErr := helper.BlockClient.BlockUnmap(getBlockPodDefintion(otherPod, blockName, false), blockMountPath)
|
||||
_, unmtErr := k8sh.DeletePod(k8sutil.DefaultNamespace, otherPod)
|
||||
require.Nil(s.T(), unmtErr)
|
||||
require.True(s.T(), k8sh.IsPodTerminated(otherPod, defaultNamespace), "make sure block-test2 pod is terminated")
|
||||
logger.Infof("Fenced pod deleted successfully")
|
||||
|
||||
logger.Infof("step 9: Unmount block storage")
|
||||
_, unmtErr = helper.BlockClient.BlockUnmap(getBlockPodDefintion(podName, blockName, false), blockMountPath)
|
||||
_, unmtErr = k8sh.DeletePod(k8sutil.DefaultNamespace, podName)
|
||||
require.Nil(s.T(), unmtErr)
|
||||
require.True(s.T(), k8sh.IsVolumeResourceAbsent(installer.SystemNamespace(namespace), crdName), fmt.Sprintf("make sure Volume %s is deleted", crdName))
|
||||
require.True(s.T(), k8sh.IsPodTerminated(blockPodName, defaultNamespace), "make sure block-test pod is terminated")
|
||||
require.True(s.T(), k8sh.IsPodTerminated(podName, defaultNamespace), "make sure block-test pod is terminated")
|
||||
logger.Infof("Block Storage unmounted successfully")
|
||||
|
||||
logger.Infof("step 10: Deleting block storage")
|
||||
_, dbErr := installer.BlockResourceOperation(k8sh, installer.GetBlockPoolStorageClassAndPvcDef(namespace, poolName, storageClassName, blockName, "ReadWriteOnce"), "delete")
|
||||
dbErr := helper.PoolClient.DeleteStorageClassAndPvc(namespace, poolName, storageClassName, blockName, "ReadWriteOnce")
|
||||
require.Nil(s.T(), dbErr)
|
||||
require.True(s.T(), retryBlockImageCountCheck(helper, len(initBlockImages), 0, namespace), "Make sure a block is deleted")
|
||||
logger.Infof("Block Storage deleted successfully")
|
||||
}
|
||||
|
||||
func createPodWithBlock(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, blockName, podName string) string {
|
||||
_, mtErr := helper.BlockClient.BlockMap(getBlockPodDefintion(podName, blockName, false))
|
||||
require.Nil(s.T(), mtErr)
|
||||
crdName, err := k8sh.GetVolumeResourceName(defaultNamespace, blockName)
|
||||
require.Nil(s.T(), err)
|
||||
require.True(s.T(), k8sh.IsVolumeResourcePresent(installer.SystemNamespace(namespace), crdName), fmt.Sprintf("make sure Volume %s is created", crdName))
|
||||
require.True(s.T(), k8sh.IsPodRunning(podName, defaultNamespace), "make sure block-test pod is in running state")
|
||||
logger.Infof("Block Storage Mounted successfully")
|
||||
return crdName
|
||||
}
|
||||
|
||||
func restartOSDPods(k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
|
||||
osdLabel := "app=rook-ceph-osd"
|
||||
|
||||
@@ -142,38 +143,51 @@ func runBlockE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s su
|
||||
logger.Infof("Block Storage End to End Integration Test - create storageclass,pool and pvc")
|
||||
logger.Infof("Running on Rook Cluster %s", clusterNamespace)
|
||||
poolName := "rookpool"
|
||||
storageClassName := "rook-ceph-block-lite"
|
||||
blockName := "test-block-claim-lite"
|
||||
podName := "test-pod-lite"
|
||||
defer blockTestDataCleanUp(helper, k8sh, clusterNamespace, poolName, storageClassName, blockName, podName)
|
||||
initBlockCount := setupBlockLite(helper, k8sh, s, clusterNamespace, poolName, storageClassName, blockName, podName)
|
||||
deleteBlockLite(helper, k8sh, s, clusterNamespace, poolName, storageClassName, blockName, podName, initBlockCount)
|
||||
}
|
||||
|
||||
func setupBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite,
|
||||
clusterNamespace, poolName, storageClassName, blockName, podName string) int {
|
||||
|
||||
//Check initial number of blocks
|
||||
defer blockTestDataCleanUp(helper, k8sh, clusterNamespace, poolName, "rook-ceph-block", "test-block", "block-test")
|
||||
bc := helper.BlockClient
|
||||
initialBlocks, err := bc.List(clusterNamespace)
|
||||
initialBlocks, err := helper.BlockClient.List(clusterNamespace)
|
||||
require.Nil(s.T(), err)
|
||||
initBlockCount := len(initialBlocks)
|
||||
|
||||
logger.Infof("step : Create Pool,StorageClass and PVC")
|
||||
|
||||
volumeDef := installer.GetBlockPoolStorageClassAndPvcDef(clusterNamespace, poolName, "rook-ceph-block", "test-block-claim", "ReadWriteOnce")
|
||||
res1, err := installer.BlockResourceOperation(k8sh, volumeDef, "create")
|
||||
checkOrderedSubstrings(s.T(), res1, poolName, "created", "rook-ceph-block", "created", "test-block-claim", "created")
|
||||
res1, err := helper.PoolClient.CreateStorageClassAndPvc(clusterNamespace, poolName, storageClassName, blockName, "ReadWriteOnce")
|
||||
checkOrderedSubstrings(s.T(), res1, poolName, "created", storageClassName, "created", blockName, "created")
|
||||
require.NoError(s.T(), err)
|
||||
|
||||
require.True(s.T(), k8sh.WaitUntilPVCIsBound(defaultNamespace, "test-block-claim"))
|
||||
require.True(s.T(), k8sh.WaitUntilPVCIsBound(defaultNamespace, blockName))
|
||||
|
||||
//Make sure new block is created
|
||||
b, _ := bc.List(clusterNamespace)
|
||||
//Make sure new block is created
|
||||
b, _ := helper.BlockClient.List(clusterNamespace)
|
||||
assert.Equal(s.T(), initBlockCount+1, len(b), "Make sure new block image is created")
|
||||
poolExists, err := helper.PoolClient.CephPoolExists(clusterNamespace, poolName)
|
||||
assert.Nil(s.T(), err)
|
||||
assert.True(s.T(), poolExists)
|
||||
return initBlockCount
|
||||
}
|
||||
|
||||
func deleteBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite,
|
||||
clusterNamespace, poolName, storageClassName, blockName, podName string, initBlockCount int) {
|
||||
|
||||
logger.Infof("deleteBlockLite: cleaning up after test")
|
||||
//Delete pvc and storageclass
|
||||
_, err = installer.BlockResourceOperation(k8sh, volumeDef, "delete")
|
||||
err := helper.PoolClient.DeleteStorageClassAndPvc(clusterNamespace, poolName, storageClassName, blockName, "ReadWriteOnce")
|
||||
assert.NoError(s.T(), err)
|
||||
|
||||
assert.True(s.T(), k8sh.WaitUntilPVCIsDeleted(defaultNamespace, "test-block-claim"))
|
||||
assert.True(s.T(), k8sh.WaitUntilPVCIsDeleted(defaultNamespace, blockName))
|
||||
require.True(s.T(), retryBlockImageCountCheck(helper, initBlockCount, 0, clusterNamespace), "Make sure a new block is deleted")
|
||||
|
||||
b, _ = bc.List(clusterNamespace)
|
||||
b, _ := helper.BlockClient.List(clusterNamespace)
|
||||
assert.Equal(s.T(), initBlockCount, len(b), "Make sure new block image is deleted")
|
||||
|
||||
checkPoolDeleted(helper, s, clusterNamespace, poolName)
|
||||
@@ -199,8 +213,8 @@ func checkPoolDeleted(helper *clients.TestClient, s suite.Suite, namespace, name
|
||||
|
||||
func blockTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, poolname, storageclassname, blockname, podname string) {
|
||||
logger.Infof("Cleaning up block storage")
|
||||
helper.BlockClient.BlockUnmap(getBlockPodDefintion(podname, blockname, false), blockMountPath)
|
||||
installer.BlockResourceOperation(k8sh, installer.GetBlockPoolStorageClassAndPvcDef(namespace, poolname, storageclassname, blockname, "ReadWriteOnce"), "delete")
|
||||
k8sh.DeletePod(k8sutil.DefaultNamespace, podname)
|
||||
helper.PoolClient.DeleteStorageClassAndPvc(namespace, poolname, storageclassname, blockname, "ReadWriteOnce")
|
||||
cleanupDynamicBlockStorage(helper, namespace)
|
||||
}
|
||||
|
||||
@@ -227,7 +241,6 @@ func cleanupDynamicBlockStorage(helper *clients.TestClient, namespace string) {
|
||||
for _, blockImage := range blockImagesList {
|
||||
helper.BlockClient.DeleteBlockImage(blockImage, namespace)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func getBlockPodDefintion(podname, blockName string, readOnly bool) string {
|
||||
@@ -245,7 +258,7 @@ spec:
|
||||
imagePullPolicy: IfNotPresent
|
||||
volumeMounts:
|
||||
- name: block-persistent-storage
|
||||
mountPath: /tmp/rook1
|
||||
mountPath: ` + clients.TestMountPath + `
|
||||
volumes:
|
||||
- name: block-persistent-storage
|
||||
persistentVolumeClaim:
|
||||
|
||||
@@ -25,7 +25,6 @@ import (
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/contracts"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -33,10 +32,12 @@ import (
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultNamespace = "default"
|
||||
)
|
||||
|
||||
var (
|
||||
logger = capnslog.NewPackageLogger("github.com/rook/rook", "integrationTest")
|
||||
|
||||
defaultNamespace = "default"
|
||||
)
|
||||
|
||||
//Test to make sure all rook components are installed and Running
|
||||
@@ -76,30 +77,18 @@ func checkIfRookClusterIsHealthy(s suite.Suite, testClient *clients.TestClient,
|
||||
require.Nil(s.T(), err)
|
||||
}
|
||||
|
||||
func HandlePanics(r interface{}, op contracts.Setup, t func() *testing.T) {
|
||||
func HandlePanics(r interface{}, op installer.TestSuite, t func() *testing.T) {
|
||||
if r != nil {
|
||||
logger.Infof("unexpected panic occurred during test %s, --> %v", t().Name(), r)
|
||||
t().Fail()
|
||||
op.TearDown()
|
||||
op.Teardown()
|
||||
t().FailNow()
|
||||
}
|
||||
}
|
||||
|
||||
//GetTestClient sets up SetTestClient for rook
|
||||
func GetTestClient(kh *utils.K8sHelper, namespace string, op contracts.Setup, t func() *testing.T) *clients.TestClient {
|
||||
helper, err := clients.CreateTestClient(kh, namespace)
|
||||
if err != nil {
|
||||
logger.Errorf("Cannot create rook test client, er -> %v", err)
|
||||
t().Fail()
|
||||
op.TearDown()
|
||||
t().FailNow()
|
||||
}
|
||||
return helper
|
||||
}
|
||||
|
||||
//BaseTestOperations struct for handling panic and test suite tear down
|
||||
type BaseTestOperations struct {
|
||||
installer *installer.InstallHelper
|
||||
//TestCluster struct for handling panic and test suite tear down
|
||||
type TestCluster struct {
|
||||
installer *installer.CephInstaller
|
||||
kh *utils.K8sHelper
|
||||
helper *clients.TestClient
|
||||
T func() *testing.T
|
||||
@@ -110,20 +99,20 @@ type BaseTestOperations struct {
|
||||
mons int
|
||||
}
|
||||
|
||||
// StartBaseTestOperations creates new instance of BaseTestOperations struct
|
||||
func StartBaseTestOperations(t func() *testing.T, namespace, storeType string, helmInstalled, useDevices bool, mons int) (BaseTestOperations, *utils.K8sHelper) {
|
||||
// StartTestCluster creates new instance of TestCluster struct
|
||||
func StartTestCluster(t func() *testing.T, namespace, storeType string, helmInstalled, useDevices bool, mons int, rookVersion string) (*TestCluster, *utils.K8sHelper) {
|
||||
kh, err := utils.CreateK8sHelper(t)
|
||||
require.NoError(t(), err)
|
||||
|
||||
i := installer.NewK8sRookhelper(kh.Clientset, t)
|
||||
i := installer.NewCephInstaller(t, kh.Clientset, rookVersion)
|
||||
|
||||
op := BaseTestOperations{i, kh, nil, t, namespace, storeType, helmInstalled, useDevices, mons}
|
||||
op.SetUp()
|
||||
op := &TestCluster{i, kh, nil, t, namespace, storeType, helmInstalled, useDevices, mons}
|
||||
op.Setup()
|
||||
return op, kh
|
||||
}
|
||||
|
||||
//SetUpRook is a wrapper for setting up rook
|
||||
func (op BaseTestOperations) SetUp() {
|
||||
func (op *TestCluster) Setup() {
|
||||
isRookInstalled, err := op.installer.InstallRookOnK8sWithHostPathAndDevices(op.namespace, op.storeType,
|
||||
op.helmInstalled, op.useDevices, cephv1beta1.MonSpec{Count: op.mons, AllowMultiplePerNode: true}, false /* startWithAllNodes */)
|
||||
|
||||
@@ -133,13 +122,18 @@ func (op BaseTestOperations) SetUp() {
|
||||
op.installer.GatherAllRookLogs(op.namespace, installer.SystemNamespace(op.namespace), op.installer.T().Name())
|
||||
}
|
||||
op.T().Fail()
|
||||
op.TearDown()
|
||||
op.Teardown()
|
||||
op.T().FailNow()
|
||||
}
|
||||
}
|
||||
|
||||
// SetInstallData updates the installer helper based on the version of Rook desired
|
||||
func (op *TestCluster) SetInstallData(version string) {
|
||||
|
||||
}
|
||||
|
||||
//TearDownRook is a wrapper for tearDown after Suite
|
||||
func (op BaseTestOperations) TearDown() {
|
||||
func (op *TestCluster) Teardown() {
|
||||
if op.installer.T().Failed() {
|
||||
op.installer.GatherAllRookLogs(op.namespace, installer.SystemNamespace(op.namespace), op.installer.T().Name())
|
||||
}
|
||||
|
||||
@@ -23,13 +23,12 @@ import (
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
"k8s.io/kubernetes/pkg/util/version"
|
||||
)
|
||||
|
||||
var (
|
||||
const (
|
||||
fileMountPath = "/tmp/rookfs"
|
||||
filePodName = "file-test"
|
||||
)
|
||||
@@ -37,21 +36,18 @@ var (
|
||||
// Smoke Test for File System Storage - Test check the following operations on Filesystem Storage in order
|
||||
//Create,Mount,Write,Read,Unmount and Delete.
|
||||
func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) {
|
||||
defer fileTestDataCleanUp(helper, k8sh, s, filePodName, namespace, filesystemName, fileMountPath)
|
||||
defer fileTestDataCleanUp(helper, k8sh, s, filePodName, namespace, filesystemName)
|
||||
logger.Infof("Running on Rook Cluster %s", namespace)
|
||||
logger.Infof("File Storage End To End Integration Test - create, mount, write to, read from, and unmount")
|
||||
|
||||
logger.Infof("Step 1: Create file System")
|
||||
fscErr := helper.FSClient.Create(filesystemName, namespace)
|
||||
require.Nil(s.T(), fscErr)
|
||||
filesystemList, _ := helper.FSClient.List(namespace)
|
||||
require.Equal(s.T(), 1, len(filesystemList), "There should one shared file system present")
|
||||
filesystemData := filesystemList[0]
|
||||
require.Equal(s.T(), filesystemName, filesystemData.Name, "make sure filesystem name matches")
|
||||
logger.Infof("File system created")
|
||||
createFilesystem(helper, k8sh, s, namespace, filesystemName)
|
||||
createFilesystemConsumerPod(helper, k8sh, s, namespace, filesystemName)
|
||||
writeAndReadToFilesystem(helper, k8sh, s, namespace, "test_file")
|
||||
cleanupFilesystemConsumer(helper, k8sh, s, namespace, filesystemName)
|
||||
}
|
||||
|
||||
logger.Infof("Step 2: Mount file System")
|
||||
mtfsErr := podWithFilesystem(k8sh, s, filePodName, namespace, filesystemName, fileMountPath, "create")
|
||||
func createFilesystemConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) {
|
||||
mtfsErr := podWithFilesystem(k8sh, s, filePodName, namespace, filesystemName, "create")
|
||||
require.Nil(s.T(), mtfsErr)
|
||||
filePodRunning := k8sh.IsPodRunning(filePodName, namespace)
|
||||
if !filePodRunning {
|
||||
@@ -61,25 +57,31 @@ func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S
|
||||
}
|
||||
require.True(s.T(), filePodRunning, "make sure file-test pod is in running state")
|
||||
logger.Infof("File system mounted successfully")
|
||||
}
|
||||
|
||||
logger.Infof("Step 3: Write to file system")
|
||||
_, wfsErr := helper.FSClient.Write(filePodName, fileMountPath, "Smoke Test Data for file system storage", "fsFile1", namespace)
|
||||
func writeAndReadToFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filename string) {
|
||||
|
||||
logger.Infof("Write to file system")
|
||||
message := "Test Data for file system storage"
|
||||
_, wfsErr := helper.FSClient.Write(filePodName, message, filename, namespace)
|
||||
require.Nil(s.T(), wfsErr)
|
||||
logger.Infof("Write to file system successful")
|
||||
|
||||
logger.Infof("Step 4: Read from file system")
|
||||
read, rdErr := helper.FSClient.Read(filePodName, fileMountPath, "fsFile1", namespace)
|
||||
logger.Infof("Read from file system")
|
||||
read, rdErr := helper.FSClient.Read(filePodName, filename, namespace)
|
||||
require.Nil(s.T(), rdErr)
|
||||
assert.Contains(s.T(), read, "Smoke Test Data for file system storage", "make sure content of the files is unchanged")
|
||||
require.Contains(s.T(), read, message)
|
||||
logger.Infof("Read from file system successful")
|
||||
}
|
||||
|
||||
logger.Infof("Step 5: UnMount file System")
|
||||
umtfsErr := podWithFilesystem(k8sh, s, filePodName, namespace, filesystemName, fileMountPath, "delete")
|
||||
require.Nil(s.T(), umtfsErr)
|
||||
func cleanupFilesystemConsumer(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) {
|
||||
logger.Infof("Unmount file System")
|
||||
_, err := k8sh.DeleteResource("pod", filePodName, "-n", namespace)
|
||||
require.Nil(s.T(), err)
|
||||
require.True(s.T(), k8sh.IsPodTerminated(filePodName, namespace), "make sure file-test pod is terminated")
|
||||
logger.Infof("File system mounted successfully")
|
||||
logger.Infof("File system unmounted successfully")
|
||||
|
||||
logger.Infof("Step 6: Deleting file storage")
|
||||
logger.Infof("Deleting file storage")
|
||||
helper.FSClient.Delete(filesystemName, namespace)
|
||||
//Delete is not deleting filesystem - known issue
|
||||
//require.Nil(suite.T(), fsd_err)
|
||||
@@ -90,22 +92,26 @@ func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S
|
||||
func runFileE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) {
|
||||
logger.Infof("File Storage End to End Integration Test - create Filesystem and make sure mds pod is running")
|
||||
logger.Infof("Running on Rook Cluster %s", namespace)
|
||||
|
||||
logger.Infof("Step 1: Create file System")
|
||||
fscErr := helper.FSClient.Create(filesystemName, namespace)
|
||||
require.Nil(s.T(), fscErr)
|
||||
filesystemList, _ := helper.FSClient.List(namespace)
|
||||
logger.Infof("Retrieved file systems: %+v", filesystemList)
|
||||
require.Equal(s.T(), 1, len(filesystemList), "There should one shared file system present")
|
||||
createFilesystem(helper, k8sh, s, namespace, filesystemName)
|
||||
}
|
||||
|
||||
func fileTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string, fileMountPath string) {
|
||||
func createFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName string) {
|
||||
logger.Infof("Create file System")
|
||||
fscErr := helper.FSClient.Create(filesystemName, namespace)
|
||||
require.Nil(s.T(), fscErr)
|
||||
logger.Infof("File system %s created", filesystemName)
|
||||
|
||||
filesystemList, _ := helper.FSClient.List(namespace)
|
||||
require.Equal(s.T(), 1, len(filesystemList), "There should be one shared file system present")
|
||||
}
|
||||
|
||||
func fileTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string) {
|
||||
logger.Infof("Cleaning up file system")
|
||||
podWithFilesystem(k8sh, s, podname, namespace, filesystemName, fileMountPath, "delete")
|
||||
podWithFilesystem(k8sh, s, podname, namespace, filesystemName, "delete")
|
||||
helper.FSClient.Delete(filesystemName, namespace)
|
||||
}
|
||||
|
||||
func podWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string, fileMountPath string, action string) error {
|
||||
func podWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string, action string) error {
|
||||
driverName := installer.SystemNamespace(namespace)
|
||||
v := version.MustParseSemantic(k8sh.GetK8sServerVersion())
|
||||
if v.LessThan(version.MustParseSemantic("1.10.0")) {
|
||||
@@ -113,7 +119,7 @@ func podWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, nam
|
||||
driverName = flexvolume.FlexDriverName
|
||||
}
|
||||
|
||||
testPod := getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath, driverName)
|
||||
testPod := getFilesystemTestPod(podname, namespace, filesystemName, driverName)
|
||||
logger.Infof("creating test pod: %s", testPod)
|
||||
_, err := k8sh.ResourceOperation(action, testPod)
|
||||
if err != nil {
|
||||
@@ -122,7 +128,7 @@ func podWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, nam
|
||||
return nil
|
||||
}
|
||||
|
||||
func getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath, driverName string) string {
|
||||
func getFilesystemTestPod(podname, namespace, filesystemName, driverName string) string {
|
||||
return `apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
@@ -138,7 +144,7 @@ spec:
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
volumeMounts:
|
||||
- mountPath: "` + fileMountPath + `"
|
||||
- mountPath: "` + clients.TestMountPath + `"
|
||||
name: ` + filesystemName + `
|
||||
volumes:
|
||||
- name: ` + filesystemName + `
|
||||
|
||||
+19
-23
@@ -20,9 +20,7 @@ import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/rook/rook/pkg/daemon/ceph/model"
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/contracts"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -48,16 +46,16 @@ type BlockCreateSuite struct {
|
||||
kh *utils.K8sHelper
|
||||
initBlockCount int
|
||||
namespace string
|
||||
installer *installer.InstallHelper
|
||||
op contracts.Setup
|
||||
installer *installer.CephInstaller
|
||||
op *TestCluster
|
||||
}
|
||||
|
||||
func (s *BlockCreateSuite) SetupSuite() {
|
||||
|
||||
var err error
|
||||
s.namespace = "block-k8s-ns"
|
||||
s.op, s.kh = StartBaseTestOperations(s.T, s.namespace, "bluestore", false, false, 1)
|
||||
s.testClient = GetTestClient(s.kh, s.namespace, s.op, s.T)
|
||||
s.op, s.kh = StartTestCluster(s.T, s.namespace, "bluestore", false, false, 1, installer.VersionMaster)
|
||||
s.testClient = clients.CreateTestClient(s.kh, s.op.installer.Manifests)
|
||||
initialBlocks, err := s.testClient.BlockClient.List(s.namespace)
|
||||
assert.Nil(s.T(), err)
|
||||
s.initBlockCount = len(initialBlocks)
|
||||
@@ -73,7 +71,7 @@ func (s *BlockCreateSuite) TestCreatePVCWhenNoStorageClassExists() {
|
||||
storageClassName := "rook-ceph-block"
|
||||
defer s.tearDownTest(claimName, poolName, storageClassName, "ReadWriteOnce")
|
||||
|
||||
result, err := installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(claimName, storageClassName, "ReadWriteOnce"), "create")
|
||||
result, err := s.testClient.BlockClient.CreatePvc(claimName, storageClassName, "ReadWriteOnce")
|
||||
checkOrderedSubstrings(s.T(), result, "persistentvolumeclaim", claimName, "created")
|
||||
require.NoError(s.T(), err)
|
||||
|
||||
@@ -105,12 +103,11 @@ func (s *BlockCreateSuite) TestCreateSamePVCTwice() {
|
||||
s.testClient.BlockClient.List(s.namespace)
|
||||
|
||||
logger.Infof("create pool and storageclass")
|
||||
pool := model.Pool{Name: poolName, ReplicatedConfig: model.ReplicatedPoolConfig{Size: 1}}
|
||||
result0, err0 := s.testClient.PoolClient.Create(pool, s.namespace)
|
||||
result0, err0 := s.testClient.PoolClient.Create(poolName, s.namespace, 1)
|
||||
checkOrderedSubstrings(s.T(), result0, poolName, "created")
|
||||
require.NoError(s.T(), err0)
|
||||
|
||||
result1, err1 := installer.BlockResourceOperation(s.kh, installer.GetBlockStorageClassDef(poolName, storageClassName, s.namespace, true), "create")
|
||||
result1, err1 := s.testClient.BlockClient.CreateStorageClass(poolName, storageClassName, s.namespace, true)
|
||||
checkOrderedSubstrings(s.T(), result1, storageClassName, "created")
|
||||
require.NoError(s.T(), err1)
|
||||
|
||||
@@ -119,7 +116,7 @@ func (s *BlockCreateSuite) TestCreateSamePVCTwice() {
|
||||
require.Nil(s.T(), err)
|
||||
|
||||
logger.Infof("create pvc")
|
||||
result2, err2 := installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(claimName, storageClassName, "ReadWriteOnce"), "create")
|
||||
result2, err2 := s.testClient.BlockClient.CreatePvc(claimName, storageClassName, "ReadWriteOnce")
|
||||
checkOrderedSubstrings(s.T(), result2, claimName, "created")
|
||||
require.NoError(s.T(), err2)
|
||||
|
||||
@@ -131,7 +128,7 @@ func (s *BlockCreateSuite) TestCreateSamePVCTwice() {
|
||||
assert.Equal(s.T(), s.initBlockCount+1, len(b1), "Make sure new block image is created")
|
||||
|
||||
logger.Infof("Create same pvc again")
|
||||
_, err3 := installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(claimName, storageClassName, "ReadWriteOnce"), "create")
|
||||
_, err3 := s.testClient.BlockClient.CreatePvc(claimName, storageClassName, "ReadWriteOnce")
|
||||
checkOrderedSubstrings(s.T(), err3.Error(), claimName, "already exists")
|
||||
|
||||
logger.Infof("check status of PVC")
|
||||
@@ -152,7 +149,8 @@ func (s *BlockCreateSuite) TestBlockStorageMountUnMountForStatefulSets() {
|
||||
defer s.statefulSetDataCleanup(defaultNamespace, poolName, storageClassName, statefulSetName, statefulPodsName)
|
||||
logger.Infof("Test case when block persistent volumes are scaled up and down along with StatefulSet")
|
||||
logger.Info("Step 1: Create pool and storageClass")
|
||||
_, cbErr := installer.BlockResourceOperation(s.kh, installer.GetBlockPoolStorageClass(s.namespace, poolName, storageClassName), "create")
|
||||
|
||||
_, cbErr := s.testClient.PoolClient.CreateStorageClass(s.namespace, poolName, storageClassName)
|
||||
assert.Nil(s.T(), cbErr)
|
||||
logger.Info("Step 2 : Deploy statefulSet with 1X replication")
|
||||
service, statefulset := getBlockStatefulSetAndServiceDefinition(defaultNamespace, statefulSetName, statefulPodsName, storageClassName)
|
||||
@@ -198,19 +196,17 @@ func (s *BlockCreateSuite) statefulSetDataCleanup(namespace, poolName, storageCl
|
||||
//Delete all PVCs
|
||||
s.kh.DeletePvcWithLabel(defaultNamespace, statefulSetName)
|
||||
//Delete storageclass and pool
|
||||
installer.BlockResourceOperation(s.kh, installer.GetBlockPoolStorageClass(s.namespace, poolName, storageClassName), "delete")
|
||||
|
||||
s.testClient.PoolClient.DeleteStorageClass(s.namespace, poolName, storageClassName)
|
||||
}
|
||||
|
||||
func (s *BlockCreateSuite) tearDownTest(claimName string, poolName string, storageClassName string, accessMode string) {
|
||||
installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(claimName, storageClassName, accessMode), "delete")
|
||||
installer.BlockResourceOperation(s.kh, installer.GetBlockPoolDef(poolName, s.namespace, "1"), "delete")
|
||||
installer.BlockResourceOperation(s.kh, installer.GetBlockStorageClassDef(poolName, storageClassName, s.namespace, true), "delete")
|
||||
|
||||
s.testClient.BlockClient.DeletePvc(claimName, storageClassName, accessMode)
|
||||
s.testClient.PoolClient.Delete(poolName, s.namespace)
|
||||
s.testClient.BlockClient.DeleteStorageClass(poolName, storageClassName, s.namespace)
|
||||
}
|
||||
|
||||
func (s *BlockCreateSuite) TearDownSuite() {
|
||||
s.op.TearDown()
|
||||
s.op.Teardown()
|
||||
}
|
||||
|
||||
func (s *BlockCreateSuite) CheckCreatingPVC(pvcName, pvcAccessMode string) {
|
||||
@@ -221,10 +217,10 @@ func (s *BlockCreateSuite) CheckCreatingPVC(pvcName, pvcAccessMode string) {
|
||||
defer s.tearDownTest(claimName, poolName, storageClassName, pvcAccessMode)
|
||||
|
||||
//create pool and storageclass
|
||||
result0, err0 := installer.BlockResourceOperation(s.kh, installer.GetBlockPoolDef(poolName, s.namespace, "1"), "create")
|
||||
result0, err0 := s.testClient.PoolClient.Create(poolName, s.namespace, 1)
|
||||
checkOrderedSubstrings(s.T(), result0, poolName, "created")
|
||||
require.NoError(s.T(), err0)
|
||||
result1, err1 := installer.BlockResourceOperation(s.kh, installer.GetBlockStorageClassDef(poolName, storageClassName, s.namespace, true), "create")
|
||||
result1, err1 := s.testClient.BlockClient.CreateStorageClass(poolName, storageClassName, s.namespace, true)
|
||||
checkOrderedSubstrings(s.T(), result1, storageClassName, "created")
|
||||
require.NoError(s.T(), err1)
|
||||
|
||||
@@ -233,7 +229,7 @@ func (s *BlockCreateSuite) CheckCreatingPVC(pvcName, pvcAccessMode string) {
|
||||
require.Nil(s.T(), err)
|
||||
|
||||
//create pvc
|
||||
result2, err2 := installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(claimName, storageClassName, pvcAccessMode), "create")
|
||||
result2, err2 := s.testClient.BlockClient.CreatePvc(claimName, storageClassName, pvcAccessMode)
|
||||
checkOrderedSubstrings(s.T(), result2, claimName, "created")
|
||||
require.NoError(s.T(), err2)
|
||||
|
||||
@@ -49,66 +49,66 @@ func TestCockroachDBSuite(t *testing.T) {
|
||||
|
||||
type CockroachDBSuite struct {
|
||||
suite.Suite
|
||||
k8sHelper *utils.K8sHelper
|
||||
installHelper *installer.InstallHelper
|
||||
k8shelper *utils.K8sHelper
|
||||
installer *installer.CockroachDBInstaller
|
||||
namespace string
|
||||
systemNamespace string
|
||||
instanceCount int
|
||||
}
|
||||
|
||||
func (suite *CockroachDBSuite) SetupSuite() {
|
||||
suite.SetUp()
|
||||
suite.Setup()
|
||||
}
|
||||
|
||||
func (suite *CockroachDBSuite) TearDownSuite() {
|
||||
suite.TearDown()
|
||||
suite.Teardown()
|
||||
}
|
||||
|
||||
func (suite *CockroachDBSuite) SetUp() {
|
||||
func (suite *CockroachDBSuite) Setup() {
|
||||
suite.namespace = "cockroachdb-ns"
|
||||
suite.systemNamespace = installer.SystemNamespace(suite.namespace)
|
||||
suite.instanceCount = 1
|
||||
|
||||
k8sHelper, err := utils.CreateK8sHelper(suite.T)
|
||||
k8shelper, err := utils.CreateK8sHelper(suite.T)
|
||||
require.NoError(suite.T(), err)
|
||||
suite.k8sHelper = k8sHelper
|
||||
suite.k8shelper = k8shelper
|
||||
|
||||
k8sversion := suite.k8sHelper.GetK8sServerVersion()
|
||||
k8sversion := suite.k8shelper.GetK8sServerVersion()
|
||||
logger.Infof("Installing cockroachdb on k8s %s", k8sversion)
|
||||
|
||||
suite.installHelper = installer.NewK8sRookhelper(suite.k8sHelper.Clientset, suite.T)
|
||||
suite.installer = installer.NewCockroachDBInstaller(suite.k8shelper, suite.T)
|
||||
|
||||
err = suite.installHelper.InstallCockroachDB(suite.systemNamespace, suite.namespace, suite.instanceCount)
|
||||
err = suite.installer.InstallCockroachDB(suite.systemNamespace, suite.namespace, suite.instanceCount)
|
||||
if err != nil {
|
||||
logger.Errorf("cockroachdb was not installed successfully: %+v", err)
|
||||
suite.T().Fail()
|
||||
suite.TearDown()
|
||||
suite.Teardown()
|
||||
suite.T().FailNow()
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *CockroachDBSuite) TearDown() {
|
||||
func (suite *CockroachDBSuite) Teardown() {
|
||||
if suite.T().Failed() {
|
||||
suite.installHelper.GatherCRDObjectDebuggingInfo(suite.systemNamespace)
|
||||
suite.installHelper.GatherCRDObjectDebuggingInfo(suite.namespace)
|
||||
suite.installHelper.GatherAllCockroachDBLogs(suite.systemNamespace, suite.namespace, suite.T().Name())
|
||||
installer.GatherCRDObjectDebuggingInfo(suite.k8shelper, suite.systemNamespace)
|
||||
installer.GatherCRDObjectDebuggingInfo(suite.k8shelper, suite.namespace)
|
||||
suite.installer.GatherAllCockroachDBLogs(suite.systemNamespace, suite.namespace, suite.T().Name())
|
||||
}
|
||||
suite.installHelper.UninstallCockroachDB(suite.systemNamespace, suite.namespace)
|
||||
suite.installer.UninstallCockroachDB(suite.systemNamespace, suite.namespace)
|
||||
}
|
||||
|
||||
func (suite *CockroachDBSuite) TestCockroachDBClusterInstallation() {
|
||||
logger.Infof("Verifying that all expected pods in cockroachdb cluster %s are running", suite.namespace)
|
||||
|
||||
// verify cockroachdb operator is running OK
|
||||
assert.True(suite.T(), suite.k8sHelper.CheckPodCountAndState("rook-cockroachdb-operator", suite.systemNamespace, 1, "Running"),
|
||||
assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-cockroachdb-operator", suite.systemNamespace, 1, "Running"),
|
||||
"1 rook-cockroachdb-operator must be in Running state")
|
||||
|
||||
// verify cockroachdb cluster instances are running OK
|
||||
assert.True(suite.T(), suite.k8sHelper.CheckPodCountAndState("rook-cockroachdb", suite.namespace, suite.instanceCount, "Running"),
|
||||
assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-cockroachdb", suite.namespace, suite.instanceCount, "Running"),
|
||||
fmt.Sprintf("%d rook-cockroachdb pods must be in Running state", suite.instanceCount))
|
||||
|
||||
// determine the cockroachdb operator pod name
|
||||
podNames, err := suite.k8sHelper.GetPodNamesForApp("rook-cockroachdb-operator", suite.systemNamespace)
|
||||
podNames, err := suite.k8shelper.GetPodNamesForApp("rook-cockroachdb-operator", suite.systemNamespace)
|
||||
require.NoError(suite.T(), err)
|
||||
require.Equal(suite.T(), 1, len(podNames))
|
||||
operatorPodName := podNames[0]
|
||||
@@ -123,7 +123,7 @@ func (suite *CockroachDBSuite) TestCockroachDBClusterInstallation() {
|
||||
inc := 0
|
||||
var result string
|
||||
for inc < utils.RetryLoop {
|
||||
result, err = suite.k8sHelper.Exec(suite.systemNamespace, operatorPodName, command, commandArgs)
|
||||
result, err = suite.k8shelper.Exec(suite.systemNamespace, operatorPodName, command, commandArgs)
|
||||
logger.Infof("cockroachdb sql command exited, err: %+v. result: %s", err, result)
|
||||
if err == nil {
|
||||
break
|
||||
|
||||
@@ -20,7 +20,7 @@ import (
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/contracts"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
@@ -57,18 +57,18 @@ type HelmSuite struct {
|
||||
suite.Suite
|
||||
helper *clients.TestClient
|
||||
kh *utils.K8sHelper
|
||||
op contracts.Setup
|
||||
op *TestCluster
|
||||
namespace string
|
||||
}
|
||||
|
||||
func (hs *HelmSuite) SetupSuite() {
|
||||
hs.namespace = "helm-ns"
|
||||
hs.op, hs.kh = StartBaseTestOperations(hs.T, hs.namespace, "bluestore", true, false, 1)
|
||||
hs.helper = GetTestClient(hs.kh, hs.namespace, hs.op, hs.T)
|
||||
hs.op, hs.kh = StartTestCluster(hs.T, hs.namespace, "bluestore", true, false, 1, installer.VersionMaster)
|
||||
hs.helper = clients.CreateTestClient(hs.kh, hs.op.installer.Manifests)
|
||||
}
|
||||
|
||||
func (hs *HelmSuite) TearDownSuite() {
|
||||
hs.op.TearDown()
|
||||
hs.op.Teardown()
|
||||
}
|
||||
|
||||
//Test to make sure all rook components are installed and Running
|
||||
|
||||
+24
-31
@@ -22,7 +22,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/contracts"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -55,12 +54,11 @@ func TestMultiClusterDeploySuite(t *testing.T) {
|
||||
|
||||
type MultiClusterDeploySuite struct {
|
||||
suite.Suite
|
||||
helper1 *clients.TestClient
|
||||
helper2 *clients.TestClient
|
||||
testClient *clients.TestClient
|
||||
k8sh *utils.K8sHelper
|
||||
namespace1 string
|
||||
namespace2 string
|
||||
op contracts.Setup
|
||||
op *MCTestOperations
|
||||
}
|
||||
|
||||
//Deploy Multiple Rook clusters
|
||||
@@ -70,89 +68,84 @@ func (mrc *MultiClusterDeploySuite) SetupSuite() {
|
||||
mrc.namespace2 = "mrc-n2"
|
||||
|
||||
mrc.op, mrc.k8sh = NewMCTestOperations(mrc.T, mrc.namespace1, mrc.namespace2)
|
||||
mrc.helper1 = GetTestClient(mrc.k8sh, mrc.namespace1, mrc.op, mrc.T)
|
||||
mrc.helper2 = GetTestClient(mrc.k8sh, mrc.namespace2, mrc.op, mrc.T)
|
||||
mrc.testClient = clients.CreateTestClient(mrc.k8sh, mrc.op.installer.Manifests)
|
||||
mrc.createPools()
|
||||
|
||||
}
|
||||
|
||||
func (mrc *MultiClusterDeploySuite) createPools() {
|
||||
// create a test pool in each cluster so that we get some PGs
|
||||
poolName := "multi-cluster-pool1"
|
||||
logger.Infof("Creating pool %s", poolName)
|
||||
result, err := installer.BlockResourceOperation(mrc.k8sh, installer.GetBlockPoolDef(poolName, mrc.namespace1, "1"), "create")
|
||||
result, err := mrc.testClient.PoolClient.Create(poolName, mrc.namespace1, 1)
|
||||
checkOrderedSubstrings(mrc.T(), result, poolName, "created")
|
||||
require.Nil(mrc.T(), err)
|
||||
|
||||
poolName = "multi-cluster-pool2"
|
||||
logger.Infof("Creating pool %s", poolName)
|
||||
result, err = installer.BlockResourceOperation(mrc.k8sh, installer.GetBlockPoolDef(poolName, mrc.namespace2, "1"), "create")
|
||||
result, err = mrc.testClient.PoolClient.Create(poolName, mrc.namespace2, 1)
|
||||
checkOrderedSubstrings(mrc.T(), result, poolName, "created")
|
||||
require.Nil(mrc.T(), err)
|
||||
}
|
||||
|
||||
func (mrc *MultiClusterDeploySuite) TearDownSuite() {
|
||||
mrc.op.TearDown()
|
||||
mrc.op.Teardown()
|
||||
}
|
||||
|
||||
//Test to make sure all rook components are installed and Running
|
||||
func (mrc *MultiClusterDeploySuite) TestInstallingMultipleRookClusters() {
|
||||
//Check if Rook cluster 1 is deployed successfully
|
||||
checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace1, 1)
|
||||
checkIfRookClusterIsHealthy(mrc.Suite, mrc.helper1, mrc.namespace1)
|
||||
checkIfRookClusterIsHealthy(mrc.Suite, mrc.testClient, mrc.namespace1)
|
||||
|
||||
//Check if Rook cluster 2 is deployed successfully
|
||||
checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace2, 1)
|
||||
checkIfRookClusterIsHealthy(mrc.Suite, mrc.helper2, mrc.namespace2)
|
||||
checkIfRookClusterIsHealthy(mrc.Suite, mrc.testClient, mrc.namespace2)
|
||||
}
|
||||
|
||||
//Test Block Store Creation on multiple rook clusters
|
||||
func (mrc *MultiClusterDeploySuite) TestBlockStoreOnMultipleRookCluster() {
|
||||
runBlockE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, mrc.namespace1)
|
||||
runBlockE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, mrc.namespace2)
|
||||
runBlockE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace1)
|
||||
runBlockE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace2)
|
||||
}
|
||||
|
||||
//Test Filesystem Creation on multiple rook clusters
|
||||
func (mrc *MultiClusterDeploySuite) TestFileStoreOnMultiRookCluster() {
|
||||
runFileE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, mrc.namespace1, "test-fs-1")
|
||||
runFileE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, mrc.namespace2, "test-fs-2")
|
||||
runFileE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace1, "test-fs-1")
|
||||
runFileE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace2, "test-fs-2")
|
||||
}
|
||||
|
||||
//Test Object Store Creation on multiple rook clusters
|
||||
func (mrc *MultiClusterDeploySuite) TestObjectStoreOnMultiRookCluster() {
|
||||
runObjectE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, mrc.namespace1, "default-c1", 2)
|
||||
runObjectE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, mrc.namespace2, "default-c2", 1)
|
||||
runObjectE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace1, "default-c1", 2)
|
||||
runObjectE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace2, "default-c2", 1)
|
||||
}
|
||||
|
||||
//MCTestOperations struct for handling panic and test suite tear down
|
||||
type MCTestOperations struct {
|
||||
installer *installer.InstallHelper
|
||||
installData *installer.InstallData
|
||||
installer *installer.CephInstaller
|
||||
kh *utils.K8sHelper
|
||||
T func() *testing.T
|
||||
namespace1 string
|
||||
namespace2 string
|
||||
systemNamespace string
|
||||
helper1 *clients.TestClient
|
||||
helper2 *clients.TestClient
|
||||
}
|
||||
|
||||
//NewMCTestOperations creates new instance of BaseTestOperations struct
|
||||
func NewMCTestOperations(t func() *testing.T, namespace1 string, namespace2 string) (MCTestOperations, *utils.K8sHelper) {
|
||||
//NewMCTestOperations creates new instance of TestCluster struct
|
||||
func NewMCTestOperations(t func() *testing.T, namespace1 string, namespace2 string) (*MCTestOperations, *utils.K8sHelper) {
|
||||
|
||||
kh, err := utils.CreateK8sHelper(t)
|
||||
require.NoError(t(), err)
|
||||
i := installer.NewK8sRookhelper(kh.Clientset, t)
|
||||
d := installer.NewK8sInstallData()
|
||||
i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster)
|
||||
|
||||
op := MCTestOperations{i, d, kh, t, namespace1, namespace2, installer.SystemNamespace(namespace1), nil, nil}
|
||||
op.SetUp()
|
||||
op := &MCTestOperations{i, kh, t, namespace1, namespace2, installer.SystemNamespace(namespace1)}
|
||||
op.Setup()
|
||||
return op, kh
|
||||
}
|
||||
|
||||
//SetUpRook is wrapper for setting up multiple rook clusters.
|
||||
func (o MCTestOperations) SetUp() {
|
||||
func (o MCTestOperations) Setup() {
|
||||
var err error
|
||||
err = o.installer.CreateK8sRookOperator(installer.SystemNamespace(o.namespace1))
|
||||
err = o.installer.CreateCephOperator(installer.SystemNamespace(o.namespace1))
|
||||
require.NoError(o.T(), err)
|
||||
|
||||
require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-operator", o.systemNamespace, "Running"),
|
||||
@@ -179,7 +172,7 @@ func (o MCTestOperations) SetUp() {
|
||||
}
|
||||
|
||||
//TearDownRook is a wrapper for tearDown after suite
|
||||
func (o MCTestOperations) TearDown() {
|
||||
func (o MCTestOperations) Teardown() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
logger.Infof("Unexpected Errors while cleaning up MultiCluster test --> %v", r)
|
||||
@@ -47,8 +47,8 @@ func TestNfsSuite(t *testing.T) {
|
||||
|
||||
type NfsSuite struct {
|
||||
suite.Suite
|
||||
k8sHelper *utils.K8sHelper
|
||||
installHelper *installer.InstallHelper
|
||||
k8shelper *utils.K8sHelper
|
||||
installer *installer.NFSInstaller
|
||||
rwClient *clients.ReadWriteOperation
|
||||
namespace string
|
||||
systemNamespace string
|
||||
@@ -56,60 +56,60 @@ type NfsSuite struct {
|
||||
}
|
||||
|
||||
func (suite *NfsSuite) SetupSuite() {
|
||||
suite.SetUp()
|
||||
suite.Setup()
|
||||
}
|
||||
|
||||
func (suite *NfsSuite) TearDownSuite() {
|
||||
suite.TearDown()
|
||||
suite.Teardown()
|
||||
}
|
||||
|
||||
func (suite *NfsSuite) SetUp() {
|
||||
func (suite *NfsSuite) Setup() {
|
||||
suite.namespace = "nfs-ns"
|
||||
suite.systemNamespace = installer.SystemNamespace(suite.namespace)
|
||||
suite.instanceCount = 1
|
||||
|
||||
k8sHelper, err := utils.CreateK8sHelper(suite.T)
|
||||
k8shelper, err := utils.CreateK8sHelper(suite.T)
|
||||
require.NoError(suite.T(), err)
|
||||
suite.k8sHelper = k8sHelper
|
||||
suite.k8shelper = k8shelper
|
||||
|
||||
k8sversion := suite.k8sHelper.GetK8sServerVersion()
|
||||
k8sversion := suite.k8shelper.GetK8sServerVersion()
|
||||
logger.Infof("Installing nfs server on k8s %s", k8sversion)
|
||||
|
||||
suite.installHelper = installer.NewK8sRookhelper(suite.k8sHelper.Clientset, suite.T)
|
||||
suite.installer = installer.NewNFSInstaller(suite.k8shelper, suite.T)
|
||||
|
||||
suite.rwClient = clients.CreateReadWriteOperation(suite.k8sHelper)
|
||||
suite.rwClient = clients.CreateReadWriteOperation(suite.k8shelper)
|
||||
|
||||
err = suite.installHelper.InstallNFSServer(suite.systemNamespace, suite.namespace, suite.instanceCount)
|
||||
err = suite.installer.InstallNFSServer(suite.systemNamespace, suite.namespace, suite.instanceCount)
|
||||
if err != nil {
|
||||
logger.Errorf("nfs server installation failed: %+v", err)
|
||||
suite.T().Fail()
|
||||
suite.TearDown()
|
||||
suite.Teardown()
|
||||
suite.T().FailNow()
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *NfsSuite) TearDown() {
|
||||
func (suite *NfsSuite) Teardown() {
|
||||
if suite.T().Failed() {
|
||||
suite.installHelper.GatherCRDObjectDebuggingInfo(suite.systemNamespace)
|
||||
suite.installHelper.GatherCRDObjectDebuggingInfo(suite.namespace)
|
||||
suite.installHelper.GatherAllNFSServerLogs(suite.systemNamespace, suite.namespace, suite.T().Name())
|
||||
installer.GatherCRDObjectDebuggingInfo(suite.k8shelper, suite.systemNamespace)
|
||||
installer.GatherCRDObjectDebuggingInfo(suite.k8shelper, suite.namespace)
|
||||
suite.installer.GatherAllNFSServerLogs(suite.systemNamespace, suite.namespace, suite.T().Name())
|
||||
}
|
||||
suite.installHelper.UninstallNFSServer(suite.systemNamespace, suite.namespace)
|
||||
suite.installer.UninstallNFSServer(suite.systemNamespace, suite.namespace)
|
||||
}
|
||||
|
||||
func (suite *NfsSuite) TestNfsServerInstallation() {
|
||||
logger.Infof("Verifying that nfs server pod %s is running", suite.namespace)
|
||||
|
||||
// verify nfs server operator is running OK
|
||||
assert.True(suite.T(), suite.k8sHelper.CheckPodCountAndState("rook-nfs-operator", suite.systemNamespace, 1, "Running"),
|
||||
assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-nfs-operator", suite.systemNamespace, 1, "Running"),
|
||||
"1 rook-nfs-operator must be in Running state")
|
||||
|
||||
// verify nfs server instances are running OK
|
||||
assert.True(suite.T(), suite.k8sHelper.CheckPodCountAndState("rook-nfs", suite.namespace, suite.instanceCount, "Running"),
|
||||
assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-nfs", suite.namespace, suite.instanceCount, "Running"),
|
||||
fmt.Sprintf("%d rook-nfs pods must be in Running state", suite.instanceCount))
|
||||
|
||||
// verify nfs server storage
|
||||
assert.True(suite.T(), true, suite.k8sHelper.WaitUntilPVCIsBound("default", "nfs-pv-claim"))
|
||||
assert.True(suite.T(), true, suite.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim"))
|
||||
|
||||
defer suite.rwClient.Delete()
|
||||
podList, err := suite.rwClient.CreateWriteClient("nfs-pv-claim")
|
||||
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/rook/rook/pkg/daemon/ceph/model"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -29,8 +28,7 @@ func (suite *SmokeSuite) TestPoolResize() {
|
||||
logger.Infof("Pool Resize Smoke Test")
|
||||
|
||||
poolName := "testpool"
|
||||
pool := model.Pool{Name: "testpool", ReplicatedConfig: model.ReplicatedPoolConfig{Size: 1}}
|
||||
out, err := suite.helper.PoolClient.Create(pool, suite.namespace)
|
||||
out, err := suite.helper.PoolClient.Create(poolName, suite.namespace, 1)
|
||||
logger.Infof("poolCreate: %+v", out)
|
||||
require.Nil(suite.T(), err)
|
||||
|
||||
@@ -56,8 +54,7 @@ func (suite *SmokeSuite) TestPoolResize() {
|
||||
|
||||
require.Equal(suite.T(), true, poolFound, "pool not found")
|
||||
|
||||
pool.ReplicatedConfig.Size = 3
|
||||
_, err = suite.helper.PoolClient.Update(pool, suite.namespace)
|
||||
_, err = suite.helper.PoolClient.Update(poolName, suite.namespace, 3)
|
||||
logger.Infof("poolCreate (modify): %+v", out)
|
||||
require.Nil(suite.T(), err)
|
||||
|
||||
|
||||
@@ -23,7 +23,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/contracts"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -69,7 +68,7 @@ func TestSmokeSuite(t *testing.T) {
|
||||
type SmokeSuite struct {
|
||||
suite.Suite
|
||||
helper *clients.TestClient
|
||||
op contracts.Setup
|
||||
op *TestCluster
|
||||
k8sh *utils.K8sHelper
|
||||
namespace string
|
||||
}
|
||||
@@ -77,12 +76,12 @@ type SmokeSuite struct {
|
||||
func (suite *SmokeSuite) SetupSuite() {
|
||||
suite.namespace = "smoke-ns"
|
||||
useDevices := true
|
||||
suite.op, suite.k8sh = StartBaseTestOperations(suite.T, suite.namespace, "bluestore", false, useDevices, 3)
|
||||
suite.helper = GetTestClient(suite.k8sh, suite.namespace, suite.op, suite.T)
|
||||
suite.op, suite.k8sh = StartTestCluster(suite.T, suite.namespace, "bluestore", false, useDevices, 3, installer.VersionMaster)
|
||||
suite.helper = clients.CreateTestClient(suite.k8sh, suite.op.installer.Manifests)
|
||||
}
|
||||
|
||||
func (suite *SmokeSuite) TearDownSuite() {
|
||||
suite.op.TearDown()
|
||||
suite.op.Teardown()
|
||||
}
|
||||
|
||||
func (suite *SmokeSuite) TestBlockStorage_SmokeTest() {
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/contracts"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -76,8 +75,8 @@ type BlockMountUnMountSuite struct {
|
||||
namespace string
|
||||
pvcNameRWO string
|
||||
pvcNameRWX string
|
||||
installer *installer.InstallHelper
|
||||
op contracts.Setup
|
||||
installer *installer.CephInstaller
|
||||
op *TestCluster
|
||||
}
|
||||
|
||||
func (s *BlockMountUnMountSuite) SetupSuite() {
|
||||
@@ -87,8 +86,8 @@ func (s *BlockMountUnMountSuite) SetupSuite() {
|
||||
s.pvcNameRWX = "block-persistent-rwx"
|
||||
useHelm := false
|
||||
useDevices := true
|
||||
s.op, s.kh = StartBaseTestOperations(s.T, s.namespace, "filestore", useHelm, useDevices, 1)
|
||||
s.testClient = GetTestClient(s.kh, s.namespace, s.op, s.T)
|
||||
s.op, s.kh = StartTestCluster(s.T, s.namespace, "filestore", useHelm, useDevices, 1, installer.VersionMaster)
|
||||
s.testClient = clients.CreateTestClient(s.kh, s.op.installer.Manifests)
|
||||
s.bc = s.testClient.BlockClient
|
||||
}
|
||||
|
||||
@@ -98,16 +97,16 @@ func (s *BlockMountUnMountSuite) setupPVCs() {
|
||||
storageClassNameRWO := "rook-ceph-block-rwo"
|
||||
|
||||
// Create PVCs
|
||||
_, cbErr := installer.BlockResourceOperation(s.kh, installer.GetBlockPoolStorageClassAndPvcDef(s.namespace, poolNameRWO, storageClassNameRWO, s.pvcNameRWO, "ReadWriteOnce"), "create")
|
||||
_, cbErr := s.testClient.PoolClient.CreateStorageClassAndPvc(s.namespace, poolNameRWO, storageClassNameRWO, s.pvcNameRWO, "ReadWriteOnce")
|
||||
require.Nil(s.T(), cbErr)
|
||||
require.True(s.T(), s.kh.WaitUntilPVCIsBound(defaultNamespace, s.pvcNameRWO), "Make sure PVC is Bound")
|
||||
|
||||
_, cbErr2 := installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(s.pvcNameRWX, storageClassNameRWO, "ReadWriteMany"), "create")
|
||||
_, cbErr2 := s.testClient.BlockClient.CreatePvc(s.pvcNameRWX, storageClassNameRWO, "ReadWriteMany")
|
||||
require.Nil(s.T(), cbErr2)
|
||||
require.True(s.T(), s.kh.WaitUntilPVCIsBound(defaultNamespace, s.pvcNameRWX), "Make sure PVC is Bound")
|
||||
|
||||
// Mount PVC on a pod and write some data.
|
||||
_, mtErr := s.bc.BlockMap(getBlockPodDefintion("setup-block-rwo", s.pvcNameRWO, false), blockMountPath)
|
||||
_, mtErr := s.bc.BlockMap(getBlockPodDefintion("setup-block-rwo", s.pvcNameRWO, false))
|
||||
require.Nil(s.T(), mtErr)
|
||||
crdName, err := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWO)
|
||||
require.Nil(s.T(), err)
|
||||
@@ -119,7 +118,7 @@ func (s *BlockMountUnMountSuite) setupPVCs() {
|
||||
}
|
||||
require.True(s.T(), rwoVolumePresent, fmt.Sprintf("make sure rwo Volume %s is created", crdName))
|
||||
|
||||
_, mtErr1 := s.bc.BlockMap(getBlockPodDefintion("setup-block-rwx", s.pvcNameRWX, false), blockMountPath)
|
||||
_, mtErr1 := s.bc.BlockMap(getBlockPodDefintion("setup-block-rwx", s.pvcNameRWX, false))
|
||||
require.Nil(s.T(), mtErr1)
|
||||
crdName1, err1 := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWX)
|
||||
require.Nil(s.T(), err1)
|
||||
@@ -128,37 +127,29 @@ func (s *BlockMountUnMountSuite) setupPVCs() {
|
||||
require.True(s.T(), s.kh.IsPodRunning("setup-block-rwx", defaultNamespace), "make sure setup-block-rwx pod is in running state")
|
||||
|
||||
// Write Data to Pod
|
||||
_, wtErr1 := s.bc.Write("setup-block-rwo", blockMountPath, "Persisted message one", "bsFile1", "")
|
||||
_, wtErr1 := s.bc.Write("setup-block-rwo", "Persisted message one", "bsFile1", "")
|
||||
require.Nil(s.T(), wtErr1)
|
||||
_, wtErr2 := s.bc.Write("setup-block-rwx", blockMountPath, "Persisted message one", "bsFile1", "")
|
||||
_, wtErr2 := s.bc.Write("setup-block-rwx", "Persisted message one", "bsFile1", "")
|
||||
require.Nil(s.T(), wtErr2)
|
||||
|
||||
// Unmount pod
|
||||
_, unmtErr1 := s.bc.BlockUnmap(getBlockPodDefintion("setup-block-rwo", s.pvcNameRWO, false), blockMountPath)
|
||||
_, unmtErr2 := s.bc.BlockUnmap(getBlockPodDefintion("setup-block-rwx", s.pvcNameRWO, false), blockMountPath)
|
||||
_, unmtErr1 := s.kh.DeletePods("setup-block-rwo", "setup-block-rwx")
|
||||
require.Nil(s.T(), unmtErr1)
|
||||
require.Nil(s.T(), unmtErr2)
|
||||
require.True(s.T(), s.kh.IsPodTerminated("setup-block-rwo", defaultNamespace), "make sure setup-block-rwo pod is terminated")
|
||||
require.True(s.T(), s.kh.IsPodTerminated("setup-block-rwx", defaultNamespace), "make sure setup-block-rwx pod is terminated")
|
||||
}
|
||||
|
||||
func (s *BlockMountUnMountSuite) TearDownSuite() {
|
||||
logger.Infof("Cleaning up block storage")
|
||||
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("setup-block-rwo", s.pvcNameRWO, false), blockMountPath)
|
||||
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("setup-block-rwx", s.pvcNameRWX, false), blockMountPath)
|
||||
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwo-block-rw-one", s.pvcNameRWO, false), blockMountPath)
|
||||
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwo-block-rw-two", s.pvcNameRWO, false), blockMountPath)
|
||||
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true), blockMountPath)
|
||||
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwo-block-ro-two", s.pvcNameRWO, true), blockMountPath)
|
||||
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwx-block-rw-one", s.pvcNameRWX, false), blockMountPath)
|
||||
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwx-block-rw-two", s.pvcNameRWX, false), blockMountPath)
|
||||
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true), blockMountPath)
|
||||
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwx-block-ro-two", s.pvcNameRWX, true), blockMountPath)
|
||||
installer.BlockResourceOperation(s.kh, installer.GetBlockPoolStorageClassAndPvcDef(s.namespace, "block-pool-rwo", "rook-ceph-block-rwo", s.pvcNameRWO, "ReadWriteOnce"), "delete")
|
||||
installer.BlockResourceOperation(s.kh, installer.GetBlockPoolStorageClassAndPvcDef(s.namespace, "block-pool-rwx", "rook-ceph-block-rwx", s.pvcNameRWX, "ReadWriteMany"), "delete")
|
||||
|
||||
s.kh.DeletePods(
|
||||
"setup-block-rwo", "setup-block-rwx", "rwo-block-rw-one", "rwo-block-rw-two", "rwo-block-ro-one",
|
||||
"rwo-block-ro-two", "rwx-block-rw-one", "rwx-block-rw-two", "rwx-block-ro-one", "rwx-block-ro-two")
|
||||
s.testClient.PoolClient.DeleteStorageClassAndPvc(s.namespace, "block-pool-rwo", "rook-ceph-block-rwo", s.pvcNameRWO, "ReadWriteOnce")
|
||||
s.testClient.PoolClient.DeleteStorageClassAndPvc(s.namespace, "block-pool-rwx", "rook-ceph-block-rwx", s.pvcNameRWX, "ReadWriteMany")
|
||||
|
||||
cleanupDynamicBlockStorage(s.testClient, s.namespace)
|
||||
s.op.TearDown()
|
||||
s.op.Teardown()
|
||||
}
|
||||
|
||||
func (s *BlockMountUnMountSuite) TestBlockStorageMountUnMountForDifferentAccessModes() {
|
||||
@@ -167,11 +158,11 @@ func (s *BlockMountUnMountSuite) TestBlockStorageMountUnMountForDifferentAccessM
|
||||
logger.Infof("Test case when existing RWO PVC is mounted and unmounted on pods with various accessModes")
|
||||
logger.Infof("Step 1.1: Mount existing ReadWriteOnce and ReadWriteMany PVC on a Pod with RW access")
|
||||
//mount PVC with RWO access on a pod with readonly set to false
|
||||
_, mtErr1 := s.bc.BlockMap(getBlockPodDefintion("rwo-block-rw-one", s.pvcNameRWO, false), blockMountPath)
|
||||
require.Nil(s.T(), mtErr1)
|
||||
_, err := s.bc.BlockMap(getBlockPodDefintion("rwo-block-rw-one", s.pvcNameRWO, false))
|
||||
require.Nil(s.T(), err)
|
||||
//mount PVC with RWX access on a pod with readonly set to false
|
||||
_, mtErr2 := s.bc.BlockMap(getBlockPodDefintion("rwx-block-rw-one", s.pvcNameRWX, false), blockMountPath)
|
||||
require.Nil(s.T(), mtErr2)
|
||||
_, err = s.bc.BlockMap(getBlockPodDefintion("rwx-block-rw-one", s.pvcNameRWX, false))
|
||||
require.Nil(s.T(), err)
|
||||
crdName1, err1 := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWO)
|
||||
crdName2, err2 := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWX)
|
||||
assert.Nil(s.T(), err1)
|
||||
@@ -184,85 +175,76 @@ func (s *BlockMountUnMountSuite) TestBlockStorageMountUnMountForDifferentAccessM
|
||||
|
||||
logger.Infof("Step 2: Check if previously persisted data is readable from ReadWriteOnce and ReadWriteMany PVC")
|
||||
//Read data on RWO PVC Mounted on pod with RW Access
|
||||
read1, rErr1 := s.bc.Read("rwo-block-rw-one", blockMountPath, "bsFile1", "")
|
||||
assert.Nil(s.T(), rErr1)
|
||||
read1, err := s.bc.Read("rwo-block-rw-one", "bsFile1", "")
|
||||
assert.Nil(s.T(), err)
|
||||
assert.Contains(s.T(), read1, "Persisted message one", "make sure previously persisted data is readable")
|
||||
//Read data on RWX PVC Mounted on pod with RW Access
|
||||
read2, rErr2 := s.bc.Read("rwx-block-rw-one", blockMountPath, "bsFile1", "")
|
||||
assert.Nil(s.T(), rErr2)
|
||||
read2, err := s.bc.Read("rwx-block-rw-one", "bsFile1", "")
|
||||
assert.Nil(s.T(), err)
|
||||
assert.Contains(s.T(), read2, "Persisted message one", "make sure previously persisted data is readable")
|
||||
|
||||
logger.Infof("Step 3: Check if read/write works on ReadWriteOnce and ReadWriteMany PVC")
|
||||
//Write data on RWO PVC Mounted on pod with RW Access
|
||||
_, wtErr1 := s.bc.Write("rwo-block-rw-one", blockMountPath, "Persisted message two", "bsFile2", "")
|
||||
assert.Nil(s.T(), wtErr1)
|
||||
_, err = s.bc.Write("rwo-block-rw-one", "Persisted message two", "bsFile2", "")
|
||||
assert.Nil(s.T(), err)
|
||||
//Read data on RWO PVC Mounted on pod with RW Access
|
||||
read1, rErr1 = s.bc.Read("rwo-block-rw-one", blockMountPath, "bsFile2", "")
|
||||
assert.Nil(s.T(), rErr1)
|
||||
read1, err = s.bc.Read("rwo-block-rw-one", "bsFile2", "")
|
||||
assert.Nil(s.T(), err)
|
||||
assert.Contains(s.T(), read1, "Persisted message two", "make sure new persisted data is readable")
|
||||
//Write data on RWX PVC Mounted on pod with RW Access
|
||||
_, wtErr2 := s.bc.Write("rwx-block-rw-one", blockMountPath, "Persisted message two", "bsFile2", "")
|
||||
assert.Nil(s.T(), wtErr2)
|
||||
_, err = s.bc.Write("rwx-block-rw-one", "Persisted message two", "bsFile2", "")
|
||||
assert.Nil(s.T(), err)
|
||||
//Read data on RWX PVC Mounted on pod with RW Access
|
||||
read2, rErr2 = s.bc.Read("rwx-block-rw-one", blockMountPath, "bsFile2", "")
|
||||
assert.Nil(s.T(), rErr2)
|
||||
read2, err = s.bc.Read("rwx-block-rw-one", "bsFile2", "")
|
||||
assert.Nil(s.T(), err)
|
||||
assert.Contains(s.T(), read2, "Persisted message two", "make sure new persisted data is readable")
|
||||
|
||||
//Mount another Pod with RW access on same PVC
|
||||
logger.Infof("Step 4: Mount existing ReadWriteOnce and ReadWriteMany PVC on a new Pod with RW access")
|
||||
//Mount RWO PVC on a new pod with ReadOnly set to false
|
||||
_, mtErr1 = s.bc.BlockMap(getBlockPodDefintion("rwo-block-rw-two", s.pvcNameRWO, false), blockMountPath)
|
||||
assert.Nil(s.T(), mtErr1)
|
||||
_, err = s.bc.BlockMap(getBlockPodDefintion("rwo-block-rw-two", s.pvcNameRWO, false))
|
||||
assert.Nil(s.T(), err)
|
||||
//Mount RWX PVC on a new pod with ReadOnly set to false
|
||||
_, mtErr2 = s.bc.BlockMap(getBlockPodDefintion("rwx-block-rw-two", s.pvcNameRWX, false), blockMountPath)
|
||||
assert.Nil(s.T(), mtErr2)
|
||||
_, err = s.bc.BlockMap(getBlockPodDefintion("rwx-block-rw-two", s.pvcNameRWX, false))
|
||||
assert.Nil(s.T(), err)
|
||||
assert.True(s.T(), s.kh.IsPodInError("rwo-block-rw-two", defaultNamespace, "FailedMount", "Volume is already attached by pod"), "make sure rwo-block-rw-two pod errors out while mounting the volume")
|
||||
assert.True(s.T(), s.kh.IsPodInError("rwx-block-rw-two", defaultNamespace, "FailedMount", "Volume is already attached by pod"), "make sure rwx-block-rw-two pod errors out while mounting the volume")
|
||||
_, unmtErr1 := s.bc.BlockUnmap(getBlockPodDefintion("rwo-block-rw-two", s.pvcNameRWO, false), blockMountPath)
|
||||
assert.Nil(s.T(), unmtErr1)
|
||||
|
||||
_, unmtErr2 := s.bc.BlockUnmap(getBlockPodDefintion("rwx-block-rw-two", s.pvcNameRWX, false), blockMountPath)
|
||||
assert.Nil(s.T(), unmtErr2)
|
||||
_, err = s.kh.DeletePods("rwo-block-rw-two", "rwx-block-rw-two")
|
||||
assert.Nil(s.T(), err)
|
||||
assert.True(s.T(), s.kh.IsPodTerminated("rwo-block-rw-two", defaultNamespace), "make sure rwo-block-rw-two pod is terminated")
|
||||
assert.True(s.T(), s.kh.IsPodTerminated("rwx-block-rw-two", defaultNamespace), "make sure rwx-block-rw-two pod is terminated")
|
||||
|
||||
logger.Infof("Step 5: Mount existing ReadWriteOnce and ReadWriteMany PVC on a new Pod with RO access")
|
||||
//Mount RWO PVC on a new pod with ReadOnly set to true
|
||||
_, mtErr1 = s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true), blockMountPath)
|
||||
assert.Nil(s.T(), mtErr1)
|
||||
_, err = s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true))
|
||||
assert.Nil(s.T(), err)
|
||||
//Mount RWX PVC on a new pod with ReadOnly set to true
|
||||
_, mtErr2 = s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true), blockMountPath)
|
||||
assert.Nil(s.T(), mtErr2)
|
||||
_, err = s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true))
|
||||
assert.Nil(s.T(), err)
|
||||
assert.True(s.T(), s.kh.IsPodInError("rwo-block-ro-one", defaultNamespace, "FailedMount", "Volume is already attached by pod"), "make sure rwo-block-ro-one pod errors out while mounting the volume")
|
||||
assert.True(s.T(), s.kh.IsPodInError("rwx-block-ro-one", defaultNamespace, "FailedMount", "Volume is already attached by pod"), "make sure rwx-block-ro-one pod errors out while mounting the volume")
|
||||
_, unmtErr1 = s.bc.BlockUnmap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true), blockMountPath)
|
||||
assert.Nil(s.T(), unmtErr1)
|
||||
_, unmtErr2 = s.bc.BlockUnmap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true), blockMountPath)
|
||||
assert.Nil(s.T(), unmtErr2)
|
||||
_, err = s.kh.DeletePods("rwo-block-ro-one", "rwx-block-ro-one")
|
||||
assert.Nil(s.T(), err)
|
||||
assert.True(s.T(), s.kh.IsPodTerminated("rwo-block-ro-one", defaultNamespace), "make sure rwo-block-ro-one pod is terminated")
|
||||
assert.True(s.T(), s.kh.IsPodTerminated("rwx-block-ro-one", defaultNamespace), "make sure rwx-block-ro-one pod is terminated")
|
||||
|
||||
logger.Infof("Step 6: UnMount Pod with RWX and RWO access")
|
||||
//UnMount RWO PVC
|
||||
_, unmtErr1 = s.bc.BlockUnmap(getBlockPodDefintion("rwo-block-rw-one", s.pvcNameRWO, false), blockMountPath)
|
||||
assert.Nil(s.T(), unmtErr1)
|
||||
//UnMount RWX PVC
|
||||
_, unmtErr2 = s.bc.BlockUnmap(getBlockPodDefintion("rwx-block-rw-one", s.pvcNameRWX, false), blockMountPath)
|
||||
assert.Nil(s.T(), unmtErr2)
|
||||
_, err = s.kh.DeletePods("rwo-block-rw-one", "rwx-block-rw-one")
|
||||
assert.Nil(s.T(), err)
|
||||
assert.True(s.T(), s.kh.IsPodTerminated("rwo-block-rw-one", defaultNamespace), "make sure rwo-block-rw-one pod is terminated")
|
||||
assert.True(s.T(), s.kh.IsPodTerminated("rwx-block-rw-one", defaultNamespace), "make sure rwx-lock-rw-one pod is terminated")
|
||||
|
||||
logger.Infof("Step 7: Mount ReadWriteOnce and ReadWriteMany PVC on two different pods with ReadOnlyMany with Readonly Access")
|
||||
//Mount RWO PVC on 2 pods with ReadOnly set to True
|
||||
_, mtErr1_1 := s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true), blockMountPath)
|
||||
_, mtErr2_1 := s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-two", s.pvcNameRWO, true), blockMountPath)
|
||||
assert.Nil(s.T(), mtErr1_1)
|
||||
assert.Nil(s.T(), mtErr2_1)
|
||||
_, err1 = s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true))
|
||||
_, err2 = s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-two", s.pvcNameRWO, true))
|
||||
assert.Nil(s.T(), err1)
|
||||
assert.Nil(s.T(), err2)
|
||||
//Mount RWX PVC on 2 pods with ReadOnly set to True
|
||||
_, mtErr1_2 := s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true), blockMountPath)
|
||||
_, mtErr2_2 := s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-two", s.pvcNameRWX, true), blockMountPath)
|
||||
assert.Nil(s.T(), mtErr1_2)
|
||||
assert.Nil(s.T(), mtErr2_2)
|
||||
_, err1 = s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true))
|
||||
_, err2 = s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-two", s.pvcNameRWX, true))
|
||||
assert.Nil(s.T(), err1)
|
||||
assert.Nil(s.T(), err2)
|
||||
assert.True(s.T(), s.kh.IsPodRunning("rwo-block-ro-one", defaultNamespace), "make sure rwo-block-ro-one pod is in running state")
|
||||
assert.True(s.T(), s.kh.IsPodRunning("rwo-block-ro-two", defaultNamespace), "make sure rwo-block-ro-two pod is in running state")
|
||||
assert.True(s.T(), s.kh.IsPodRunning("rwx-block-ro-one", defaultNamespace), "make sure rwx-block-ro-one pod is in running state")
|
||||
@@ -270,39 +252,32 @@ func (s *BlockMountUnMountSuite) TestBlockStorageMountUnMountForDifferentAccessM
|
||||
|
||||
logger.Infof("Step 8: Read Data from both ReadyOnlyMany and ReadWriteOnce pods with ReadOnly Access")
|
||||
//Read data from RWO PVC via both ReadOnly pods
|
||||
read1_1, rErr1_1 := s.bc.Read("rwo-block-ro-one", blockMountPath, "bsFile1", "")
|
||||
read2_1, rErr2_1 := s.bc.Read("rwo-block-ro-two", blockMountPath, "bsFile1", "")
|
||||
assert.Nil(s.T(), rErr1_1)
|
||||
assert.Nil(s.T(), rErr2_1)
|
||||
read1_1, err1 := s.bc.Read("rwo-block-ro-one", "bsFile1", "")
|
||||
read2_1, err2 := s.bc.Read("rwo-block-ro-two", "bsFile1", "")
|
||||
assert.Nil(s.T(), err1)
|
||||
assert.Nil(s.T(), err2)
|
||||
assert.Contains(s.T(), read1_1, "Persisted message one", "make sure previously persisted data is readable from ReadOnlyMany access Pod")
|
||||
assert.Contains(s.T(), read2_1, "Persisted message one", "make sure previously persisted data is readable from ReadOnlyMany access Pod")
|
||||
//Read data from RWX PVC via both ReadOnly pods
|
||||
read1_2, rErr1_2 := s.bc.Read("rwx-block-ro-one", blockMountPath, "bsFile1", "")
|
||||
read2_2, rErr2_2 := s.bc.Read("rwx-block-ro-two", blockMountPath, "bsFile1", "")
|
||||
assert.Nil(s.T(), rErr1_2)
|
||||
assert.Nil(s.T(), rErr2_2)
|
||||
read1_2, err1 := s.bc.Read("rwx-block-ro-one", "bsFile1", "")
|
||||
read2_2, err2 := s.bc.Read("rwx-block-ro-two", "bsFile1", "")
|
||||
assert.Nil(s.T(), err1)
|
||||
assert.Nil(s.T(), err2)
|
||||
assert.Contains(s.T(), read1_2, "Persisted message one", "make sure previously persisted data is readable from ReadOnlyMany access Pod")
|
||||
assert.Contains(s.T(), read2_2, "Persisted message one", "make sure previously persisted data is readable from ReadOnlyMany access Pod")
|
||||
|
||||
logger.Infof("Step 9: Write Data to Pod with ReadOnlyMany and ReadWriteOnce PVC mounted with ReadOnly access")
|
||||
//Write data to RWO PVC via pod with ReadOnly Set to true
|
||||
_, wtErr1 = s.bc.Write("rwo-block-ro-one", blockMountPath, "Persisted message three", "bsFile3", "")
|
||||
assert.Contains(s.T(), wtErr1.Error(), "Unable to write data to pod")
|
||||
_, err = s.bc.Write("rwo-block-ro-one", "Persisted message three", "bsFile3", "")
|
||||
assert.Contains(s.T(), err.Error(), "Unable to write data to pod")
|
||||
//Write data to RWx PVC via pod with ReadOnly Set to true
|
||||
_, wtErr2 = s.bc.Write("rwx-block-ro-one", blockMountPath, "Persisted message three", "bsFile3", "")
|
||||
assert.Contains(s.T(), wtErr2.Error(), "Unable to write data to pod")
|
||||
_, err = s.bc.Write("rwx-block-ro-one", "Persisted message three", "bsFile3", "")
|
||||
assert.Contains(s.T(), err.Error(), "Unable to write data to pod")
|
||||
|
||||
logger.Infof("Step 10: UnMount Pod with ReadOnlyMany and ReadWriteOnce PVCs")
|
||||
//UnMount RWO PVC from both ReadOnly Pods
|
||||
_, unmtErr1_1 := s.bc.BlockUnmap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true), blockMountPath)
|
||||
_, unmtErr2_1 := s.bc.BlockUnmap(getBlockPodDefintion("rwo-block-ro-two", s.pvcNameRWO, true), blockMountPath)
|
||||
assert.Nil(s.T(), unmtErr1_1)
|
||||
assert.Nil(s.T(), unmtErr2_1)
|
||||
//UnMount RWX PVC from both ReadOnly Pods
|
||||
_, unmtErr1_2 := s.bc.BlockUnmap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true), blockMountPath)
|
||||
_, unmtErr2_2 := s.bc.BlockUnmap(getBlockPodDefintion("rwx-block-ro-two", s.pvcNameRWX, true), blockMountPath)
|
||||
assert.Nil(s.T(), unmtErr1_2)
|
||||
assert.Nil(s.T(), unmtErr2_2)
|
||||
_, err = s.kh.DeletePods("rwo-block-ro-one", "rwo-block-ro-two", "rwx-block-ro-one", "rwx-block-ro-two")
|
||||
assert.Nil(s.T(), err)
|
||||
assert.True(s.T(), s.kh.IsPodTerminated("rwo-block-ro-one", defaultNamespace), "make sure rwo-block-ro-one pod is terminated")
|
||||
assert.True(s.T(), s.kh.IsPodTerminated("rwo-block-ro-two", defaultNamespace), "make sure rwo-block-ro-two pod is terminated")
|
||||
assert.True(s.T(), s.kh.IsPodTerminated("rwx-block-ro-one", defaultNamespace), "make sure rwx-lock-ro-one pod is terminated")
|
||||
|
||||
@@ -18,13 +18,13 @@ var (
|
||||
)
|
||||
|
||||
// Create StorageClass and poll if needed
|
||||
func createStorageClassAndPool(t func() *testing.T, kh *utils.K8sHelper, namespace string, storageClassName string, poolName string) {
|
||||
func createStorageClassAndPool(t func() *testing.T, testClient *clients.TestClient, kh *utils.K8sHelper, namespace string, storageClassName string, poolName string) {
|
||||
//create storage class
|
||||
if err := kh.IsStorageClassPresent(storageClassName); err != nil {
|
||||
logger.Infof("Install pool and storage class for rook block")
|
||||
_, err := installer.BlockResourceOperation(kh, installer.GetBlockPoolDef(poolName, namespace, "3"), "create")
|
||||
_, err := testClient.PoolClient.Create(poolName, namespace, 3)
|
||||
require.NoError(t(), err)
|
||||
_, err = installer.BlockResourceOperation(kh, installer.GetBlockStorageClassDef(poolName, storageClassName, namespace, false), "create")
|
||||
_, err = testClient.BlockClient.CreateStorageClass(poolName, storageClassName, namespace, false)
|
||||
require.NoError(t(), err)
|
||||
|
||||
//make sure storageclass is created
|
||||
@@ -90,7 +90,7 @@ func mountUnmountPVCOnPod(k8sh *utils.K8sHelper, podName string, pvcName string,
|
||||
return result, err
|
||||
}
|
||||
|
||||
func performBlockOperations(installer *installer.InstallHelper, db *utils.MySQLHelper) {
|
||||
func performBlockOperations(db *utils.MySQLHelper) {
|
||||
var wg sync.WaitGroup
|
||||
for i := 1; i <= installer.Env.LoadConcurrentRuns; i++ {
|
||||
wg.Add(1)
|
||||
@@ -133,29 +133,29 @@ func dbOperation(db *utils.MySQLHelper, wg *sync.WaitGroup, runtime int, loadSiz
|
||||
|
||||
}
|
||||
|
||||
//BaseTestOperations struct for handling panic and test suite tear down
|
||||
type BaseLoadTestOperations struct {
|
||||
installer *installer.InstallHelper
|
||||
// LoadTestCluster struct for handling panic and test suite tear down
|
||||
type LoadTestCluster struct {
|
||||
installer *installer.CephInstaller
|
||||
kh *utils.K8sHelper
|
||||
helper *clients.TestClient
|
||||
T func() *testing.T
|
||||
namespace string
|
||||
}
|
||||
|
||||
// StartBaseTestOperations creates new instance of BaseTestOperations struct
|
||||
func StartBaseLoadTestOperations(t func() *testing.T, namespace string) (BaseLoadTestOperations, *utils.K8sHelper, *installer.InstallHelper) {
|
||||
// StartLoadTestCluster creates new instance of TestCluster struct
|
||||
func StartLoadTestCluster(t func() *testing.T, namespace string) (LoadTestCluster, *utils.K8sHelper, *installer.CephInstaller) {
|
||||
kh, err := utils.CreateK8sHelper(t)
|
||||
require.NoError(t(), err)
|
||||
|
||||
i := installer.NewK8sRookhelper(kh.Clientset, t)
|
||||
i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster)
|
||||
|
||||
op := BaseLoadTestOperations{i, kh, nil, t, namespace}
|
||||
op.SetUp()
|
||||
op := LoadTestCluster{i, kh, nil, t, namespace}
|
||||
op.Setup()
|
||||
return op, kh, i
|
||||
}
|
||||
|
||||
//SetUpRook is a wrapper for setting up rook
|
||||
func (o BaseLoadTestOperations) SetUp() {
|
||||
// Setup is a wrapper for setting up rook
|
||||
func (o LoadTestCluster) Setup() {
|
||||
|
||||
if !o.kh.IsRookInstalled(o.namespace) {
|
||||
isRookInstalled, err := o.installer.InstallRookOnK8sWithHostPathAndDevices(o.namespace, "bluestore",
|
||||
@@ -166,14 +166,14 @@ func (o BaseLoadTestOperations) SetUp() {
|
||||
}
|
||||
|
||||
// Enable chaos monkey if enable_chaos flag is present
|
||||
if o.installer.Env.EnableChaos {
|
||||
if installer.Env.EnableChaos {
|
||||
c := NewChaosHelper(o.namespace, o.kh)
|
||||
go c.Monkey()
|
||||
}
|
||||
}
|
||||
|
||||
//TearDownRook is a wrapper for tearDown after suite
|
||||
func (o BaseLoadTestOperations) TearDown() {
|
||||
func (o LoadTestCluster) Teardown() {
|
||||
// No Clean up for load test
|
||||
}
|
||||
|
||||
@@ -262,7 +262,7 @@ spec:
|
||||
imagePullPolicy: IfNotPresent
|
||||
volumeMounts:
|
||||
- name: block-persistent-storage
|
||||
mountPath: /tmp/rook1
|
||||
mountPath: ` + clients.TestMountPath + `
|
||||
volumes:
|
||||
- name: block-persistent-storage
|
||||
persistentVolumeClaim:
|
||||
|
||||
@@ -69,7 +69,7 @@ func isObjectStorePresent(kh *utils.K8sHelper, namespace string, storeName strin
|
||||
return false
|
||||
}
|
||||
|
||||
func performObjectStoreOperations(installer *installer.InstallHelper, s3 *utils.S3Helper, bucketName string) {
|
||||
func performObjectStoreOperations(s3 *utils.S3Helper, bucketName string) {
|
||||
var wg sync.WaitGroup
|
||||
for i := 1; i <= installer.Env.LoadConcurrentRuns; i++ {
|
||||
wg.Add(1)
|
||||
|
||||
@@ -21,9 +21,10 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
|
||||
"time"
|
||||
|
||||
"github.com/rook/rook/tests/framework/contracts"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/suite"
|
||||
@@ -44,18 +45,20 @@ func TestBlockLongHaul(t *testing.T) {
|
||||
|
||||
type BlockLongHaulSuite struct {
|
||||
suite.Suite
|
||||
kh *utils.K8sHelper
|
||||
installer *installer.InstallHelper
|
||||
namespace string
|
||||
op contracts.Setup
|
||||
kh *utils.K8sHelper
|
||||
installer *installer.CephInstaller
|
||||
namespace string
|
||||
op installer.TestSuite
|
||||
testClient *clients.TestClient
|
||||
}
|
||||
|
||||
//Test set up - does the following in order
|
||||
//create pool and storage class, create a PVC, Create a MySQL app/service that uses pvc
|
||||
func (s *BlockLongHaulSuite) SetupSuite() {
|
||||
s.namespace = "longhaul-ns"
|
||||
s.op, s.kh, s.installer = StartBaseLoadTestOperations(s.T, s.namespace)
|
||||
createStorageClassAndPool(s.T, s.kh, s.namespace, "rook-ceph-block", "rook-pool")
|
||||
s.op, s.kh, s.installer = StartLoadTestCluster(s.T, s.namespace)
|
||||
s.testClient = clients.CreateTestClient(s.kh, s.installer.Manifests)
|
||||
createStorageClassAndPool(s.T, s.testClient, s.kh, s.namespace, "rook-ceph-block", "rook-pool")
|
||||
}
|
||||
|
||||
//create a n number ofPVC, Create a MySQL app/service that uses pvc
|
||||
@@ -63,8 +66,8 @@ func (s *BlockLongHaulSuite) SetupSuite() {
|
||||
//All other PVC and mounts are created and deleted/unmounted per test
|
||||
func (s *BlockLongHaulSuite) TestBlockLonghaulRun() {
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(s.installer.Env.LoadVolumeNumber)
|
||||
for i := 1; i <= s.installer.Env.LoadVolumeNumber; i++ {
|
||||
wg.Add(installer.Env.LoadVolumeNumber)
|
||||
for i := 1; i <= installer.Env.LoadVolumeNumber; i++ {
|
||||
if i == 1 {
|
||||
go BlockVolumeOperations(s, &wg, "rook-ceph-block", "mysqlapp-persist", "mysqldb", "mysql-persist-claim", false)
|
||||
} else {
|
||||
@@ -79,7 +82,7 @@ func (s *BlockLongHaulSuite) TestBlockLonghaulRun() {
|
||||
func BlockVolumeOperations(s *BlockLongHaulSuite, wg *sync.WaitGroup, storageClassName string, appName string, appLabel string, pvcName string, cleanup bool) {
|
||||
defer wg.Done()
|
||||
db := createPVCAndMountMysqlPod(s.T, s.kh, storageClassName, appName, appLabel, pvcName)
|
||||
performBlockOperations(s.installer, db)
|
||||
performBlockOperations(db)
|
||||
if cleanup {
|
||||
mySqlPodOperation(s.kh, storageClassName, appName, appLabel, pvcName, "delete")
|
||||
s.kh.WaitUntilPodWithLabelDeleted(appLabel, "default")
|
||||
@@ -91,7 +94,7 @@ func BlockVolumeOperations(s *BlockLongHaulSuite, wg *sync.WaitGroup, storageCla
|
||||
|
||||
func (s *BlockLongHaulSuite) TearDownSuite() {
|
||||
//clean up all ephemeral block storage, index 1 is persistent block storage which is going to be used among different test runs.
|
||||
for i := 2; i <= s.installer.Env.LoadVolumeNumber; i++ {
|
||||
for i := 2; i <= installer.Env.LoadVolumeNumber; i++ {
|
||||
mySqlPodOperation(s.kh, "rook-ceph-block", "mysqlapp-ephemeral-"+strconv.Itoa(i), "mysqldbeph"+strconv.Itoa(i), "mysql-ephemeral-claim"+strconv.Itoa(i), "delete")
|
||||
}
|
||||
s.kh = nil
|
||||
|
||||
@@ -24,7 +24,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/contracts"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -46,42 +45,40 @@ func TestBlockWithFencingLongHaul(t *testing.T) {
|
||||
type BlockLongHaulSuiteWithFencing struct {
|
||||
suite.Suite
|
||||
kh *utils.K8sHelper
|
||||
installer *installer.InstallHelper
|
||||
installer *installer.CephInstaller
|
||||
testClient *clients.TestClient
|
||||
namespace string
|
||||
op contracts.Setup
|
||||
op installer.TestSuite
|
||||
}
|
||||
|
||||
//Test set up - does the following in order
|
||||
//create pool and storage class, Create a PVC and mount a pod with ReadOnly=false.
|
||||
//Write some data to the pvc and unmount the pod
|
||||
func (s *BlockLongHaulSuiteWithFencing) SetupSuite() {
|
||||
var err error
|
||||
s.namespace = "longhaul-ns"
|
||||
s.op, s.kh, s.installer = StartBaseLoadTestOperations(s.T, s.namespace)
|
||||
createStorageClassAndPool(s.T, s.kh, s.namespace, "rook-ceph-block", "rook-pool")
|
||||
s.testClient, err = clients.CreateTestClient(s.kh, s.namespace)
|
||||
require.Nil(s.T(), err)
|
||||
s.op, s.kh, s.installer = StartLoadTestCluster(s.T, s.namespace)
|
||||
s.testClient = clients.CreateTestClient(s.kh, s.installer.Manifests)
|
||||
createStorageClassAndPool(s.T, s.testClient, s.kh, s.namespace, "rook-ceph-block", "rook-pool")
|
||||
if _, err := s.kh.GetPVCStatus(defaultNamespace, "block-pv-one"); err != nil {
|
||||
logger.Infof("Creating PVC and mounting it to pod with readOnly set to false")
|
||||
installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef("block-pv-one", "rook-ceph-block", "ReadWriteOnce"), "create")
|
||||
_, err = s.testClient.BlockClient.CreatePvc("block-pv-one", "rook-ceph-block", "ReadWriteOnce")
|
||||
require.Nil(s.T(), err)
|
||||
mountUnmountPVCOnPod(s.kh, "block-rw", "block-pv-one", "false", "create")
|
||||
require.True(s.T(), s.kh.IsPodRunning("block-rw", defaultNamespace))
|
||||
|
||||
s.testClient.BlockClient.Write("block-rw", "/tmp/rook1", "this is long running test", "longhaul", defaultNamespace)
|
||||
s.testClient.BlockClient.Write("block-rw", "this is long running test", "longhaul", defaultNamespace)
|
||||
mountUnmountPVCOnPod(s.kh, "block-rw", "block-pv-one", "false", "delete")
|
||||
require.True(s.T(), s.kh.IsPodTerminated("block-rw", defaultNamespace))
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//Mount n number of pods on the same PVC created earlier with readOnly=True.
|
||||
func (s *BlockLongHaulSuiteWithFencing) TestBlockWithFencingLonghaulRun() {
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(s.installer.Env.LoadVolumeNumber)
|
||||
for i := 1; i <= s.installer.Env.LoadVolumeNumber; i++ {
|
||||
wg.Add(installer.Env.LoadVolumeNumber)
|
||||
for i := 1; i <= installer.Env.LoadVolumeNumber; i++ {
|
||||
go blockVolumeFencingOperations(s, &wg, "block-read"+strconv.Itoa(i), "block-pv-one")
|
||||
}
|
||||
wg.Wait()
|
||||
@@ -91,7 +88,7 @@ func blockVolumeFencingOperations(s *BlockLongHaulSuiteWithFencing, wg *sync.Wai
|
||||
defer wg.Done()
|
||||
mountUnmountPVCOnPod(s.kh, podName, pvcName, "true", "create")
|
||||
require.True(s.T(), s.kh.IsPodRunning(podName, defaultNamespace))
|
||||
read, rErr := s.testClient.BlockClient.Read(podName, "/tmp/rook1", "longhaul", "default")
|
||||
read, rErr := s.testClient.BlockClient.Read(podName, "longhaul", "default")
|
||||
require.Nil(s.T(), rErr)
|
||||
assert.Contains(s.T(), read, "this is long running test")
|
||||
mountUnmountPVCOnPod(s.kh, podName, pvcName, "true", "delete")
|
||||
|
||||
@@ -22,10 +22,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/contracts"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
@@ -47,25 +45,23 @@ func TestObjectLongHaul(t *testing.T) {
|
||||
type ObjectLongHaulSuite struct {
|
||||
suite.Suite
|
||||
kh *utils.K8sHelper
|
||||
installer *installer.InstallHelper
|
||||
installer *installer.CephInstaller
|
||||
tc *clients.TestClient
|
||||
namespace string
|
||||
op contracts.Setup
|
||||
op installer.TestSuite
|
||||
}
|
||||
|
||||
func (s *ObjectLongHaulSuite) SetupSuite() {
|
||||
var err error
|
||||
s.namespace = "longhaul-ns"
|
||||
s.op, s.kh, s.installer = StartBaseLoadTestOperations(s.T, s.namespace)
|
||||
s.tc, err = clients.CreateTestClient(s.kh, s.namespace)
|
||||
require.Nil(s.T(), err)
|
||||
s.op, s.kh, s.installer = StartLoadTestCluster(s.T, s.namespace)
|
||||
s.tc = clients.CreateTestClient(s.kh, s.installer.Manifests)
|
||||
}
|
||||
|
||||
func (s *ObjectLongHaulSuite) TestObjectLonghaulRun() {
|
||||
var wg sync.WaitGroup
|
||||
storeName := "longhaulstore"
|
||||
wg.Add(s.installer.Env.LoadVolumeNumber)
|
||||
for i := 1; i <= s.installer.Env.LoadVolumeNumber; i++ {
|
||||
wg.Add(installer.Env.LoadVolumeNumber)
|
||||
for i := 1; i <= installer.Env.LoadVolumeNumber; i++ {
|
||||
if i == 1 {
|
||||
go ObjectStoreOperations(s, &wg, s.namespace, storeName+strconv.Itoa(i), false)
|
||||
} else {
|
||||
@@ -90,7 +86,7 @@ func ObjectStoreOperations(s *ObjectLongHaulSuite, wg *sync.WaitGroup, namespace
|
||||
s.T().Fail()
|
||||
}
|
||||
|
||||
performObjectStoreOperations(s.installer, s3, bucketName)
|
||||
performObjectStoreOperations(s3, bucketName)
|
||||
if deleteStore {
|
||||
delOpts := metav1.DeleteOptions{}
|
||||
s.tc.ObjectClient.Delete(namespace, storeName)
|
||||
|
||||
Reference in new issue
Block a user