From b2c039fc726b4f76d6cb992e53fb6a2c91d5f401 Mon Sep 17 00:00:00 2001 From: travisn Date: Tue, 21 Aug 2018 15:24:52 -0600 Subject: [PATCH] tests: refactor for adding an upgrade test Signed-off-by: travisn --- tests/framework/clients/block.go | 86 +++--- tests/framework/clients/filesystem.go | 95 +----- tests/framework/clients/object.go | 43 +-- tests/framework/clients/pool.go | 43 ++- tests/framework/clients/test_client.go | 13 +- tests/framework/contracts/operations.go | 22 -- .../{install_helper.go => ceph_installer.go} | 238 ++++----------- .../{install_data.go => ceph_manifests.go} | 161 +++++++++-- tests/framework/installer/cockroachdb.go | 270 ------------------ .../installer/cockroachdb_installer.go | 156 ++++++++++ .../installer/cockroachdb_manifests.go | 153 ++++++++++ tests/framework/installer/device.go | 68 +++++ .../environment.go} | 4 +- tests/framework/installer/installer.go | 90 ++++++ .../installer/{nfs.go => nfs_installer.go} | 68 +++-- .../{nfs_install_data.go => nfs_manifests.go} | 13 +- tests/framework/installer/provisioners.go | 26 +- tests/framework/installer/test_data.go | 65 ----- tests/framework/utils/k8s_helper.go | 44 ++- tests/integration/base_block_test.go | 89 +++--- tests/integration/base_deploy_test.go | 50 ++-- tests/integration/base_file_test.go | 78 ++--- ...Block_k8s_test.go => block_create_test.go} | 42 ++- tests/integration/cockroachdb_test.go | 40 +-- tests/integration/helm_test.go | 10 +- ...r_deploy_test.go => multi_cluster_test.go} | 55 ++-- tests/integration/nfs_test.go | 40 +-- tests/integration/pool_test.go | 7 +- tests/integration/smoke_test.go | 9 +- .../integration/zblock_mount_unmount_test.go | 165 +++++------ tests/longhaul/base_block_test.go | 34 +-- tests/longhaul/base_object_test.go | 2 +- tests/longhaul/block_test.go | 25 +- tests/longhaul/block_with_fencing_test.go | 25 +- tests/longhaul/object_test.go | 18 +- 35 files changed, 1214 insertions(+), 1133 deletions(-) delete mode 100644 tests/framework/contracts/operations.go rename tests/framework/installer/{install_helper.go => ceph_installer.go} (64%) rename tests/framework/installer/{install_data.go => ceph_manifests.go} (64%) delete mode 100644 tests/framework/installer/cockroachdb.go create mode 100644 tests/framework/installer/cockroachdb_installer.go create mode 100644 tests/framework/installer/cockroachdb_manifests.go create mode 100644 tests/framework/installer/device.go rename tests/framework/{objects/environment_manifest.go => installer/environment.go} (97%) create mode 100644 tests/framework/installer/installer.go rename tests/framework/installer/{nfs.go => nfs_installer.go} (68%) rename tests/framework/installer/{nfs_install_data.go => nfs_manifests.go} (90%) delete mode 100644 tests/framework/installer/test_data.go rename tests/integration/{block_createBlock_k8s_test.go => block_create_test.go} (81%) rename tests/integration/{multi_cluster_deploy_test.go => multi_cluster_test.go} (76%) diff --git a/tests/framework/clients/block.go b/tests/framework/clients/block.go index 07d5f8027..722aa4e52 100644 --- a/tests/framework/clients/block.go +++ b/tests/framework/clients/block.go @@ -18,14 +18,21 @@ package clients import ( "fmt" + "path" "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" ) +const ( + TestMountPath = "/tmp/testrook" +) + //BlockOperation is wrapper for k8s rook block operations type BlockOperation struct { k8sClient *utils.K8sHelper + manifests installer.CephManifests } type BlockImage struct { @@ -41,9 +48,9 @@ var ( readDataFromBlockPod = []string{"cat", "READ_DATA_CMD"} ) -// CreateK8BlockOperation - Constructor to create BlockOperation - client to perform rook Block operations on k8s -func CreateK8BlockOperation(k8shelp *utils.K8sHelper) *BlockOperation { - return &BlockOperation{k8sClient: k8shelp} +// CreateBlockOperation - Constructor to create BlockOperation - client to perform rook Block operations on k8s +func CreateBlockOperation(k8shelp *utils.K8sHelper, manifests installer.CephManifests) *BlockOperation { + return &BlockOperation{k8shelp, manifests} } // BlockCreate Function to create a Block using Rook @@ -62,6 +69,24 @@ func (b *BlockOperation) Create(manifest string, size int) (string, error) { } +func (b *BlockOperation) CreatePvc(claimName, storageClassName, mode string) (string, error) { + return b.k8sClient.ResourceOperation("create", b.manifests.GetBlockPvcDef(claimName, storageClassName, mode)) +} + +func (b *BlockOperation) CreateStorageClass(poolName, storageClassName, namespace string, varClusterName bool) (string, error) { + return b.k8sClient.ResourceOperation("create", b.manifests.GetBlockStorageClassDef(poolName, storageClassName, namespace, varClusterName)) +} + +func (b *BlockOperation) DeletePvc(claimName, storageClassName, mode string) error { + _, err := b.k8sClient.ResourceOperation("delete", b.manifests.GetBlockPvcDef(claimName, storageClassName, mode)) + return err +} + +func (b *BlockOperation) DeleteStorageClass(poolName, storageClassName, namespace string) error { + _, err := b.k8sClient.ResourceOperation("delete", b.manifests.GetBlockStorageClassDef(poolName, storageClassName, namespace, false)) + return err +} + // BlockDelete Function to delete a Block using Rook // Input parameters - // manifest - pod definition where pvc is described - delete is run on the the yaml definition @@ -117,9 +142,8 @@ func (b *BlockOperation) DeleteBlockImage(image BlockImage, namespace string) er // BlockMap Function to map a Block using Rook // Input parameters - // manifest - Pod definition - pod should be defined to use a pvc that was created earlier -// mountpath - not used in this impl since mountpath is defined in the pod definition // Output - k8s create pod operation output and/or error -func (b *BlockOperation) BlockMap(manifest string, mountpath string) (string, error) { +func (b *BlockOperation) BlockMap(manifest string) (string, error) { args := []string{"create", "-f", "-"} result, err := b.k8sClient.KubectlWithStdin(manifest, args...) if err != nil { @@ -133,13 +157,26 @@ func (b *BlockOperation) BlockMap(manifest string, mountpath string) (string, er // Write Function to write data to block created by rook ,i.e. write data to a pod that is using a pvc // Input parameters - // manifest - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier -// mountpath - folder on the pod were data is supposed to be written(should match the mountpath described in the pod definition) // data - data to be written // filename - file where data is written to // namespace - optional param - namespace of the pod // Output - k8s exec pod operation output and/or error -func (b *BlockOperation) Write(name string, mountpath string, data string, filename string, namespace string) (string, error) { - wt := "echo \"" + data + "\">" + mountpath + "/" + filename +func (b *BlockOperation) Write(name string, data string, filename string, namespace string) (string, error) { + return writeToPod(b.k8sClient, name, data, filename, namespace) +} + +// Read Function to read from block created by rook ,i.e. Read data from a pod that is using a pvc +// Input parameters - +// manifest - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier +// filename - file to be read +// namespace - optional param - namespace of the pod +// Output - k8s exec pod operation output and/or error +func (b *BlockOperation) Read(name string, filename string, namespace string) (string, error) { + return readFromPod(b.k8sClient, name, filename, namespace) +} + +func writeToPod(k8sh *utils.K8sHelper, name string, data string, filename string, namespace string) (string, error) { + wt := "echo \"" + data + "\">" + path.Join(TestMountPath, filename) args := []string{"exec", name} if namespace != "" { @@ -147,24 +184,16 @@ func (b *BlockOperation) Write(name string, mountpath string, data string, filen } args = append(args, "--", "sh", "-c", wt) - result, err := b.k8sClient.Kubectl(args...) + result, err := k8sh.Kubectl(args...) if err != nil { return "", fmt.Errorf("Unable to write data to pod --: %s", err) } return result, nil - } -// Read Function to read from block created by rook ,i.e. Read data from a pod that is using a pvc -// Input parameters - -// manifest - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier -// mountpath - folder on the pod were data is supposed to be written(should match the mountpath described in the pod definition) -// filename - file to be read -// namespace - optional param - namespace of the pod -// Output - k8s exec pod operation output and/or error -func (b *BlockOperation) Read(name string, mountpath string, filename string, namespace string) (string, error) { - rd := mountpath + "/" + filename +func readFromPod(k8sh *utils.K8sHelper, name string, filename string, namespace string) (string, error) { + rd := path.Join(TestMountPath, filename) args := []string{"exec", name} if namespace != "" { @@ -172,27 +201,10 @@ func (b *BlockOperation) Read(name string, mountpath string, filename string, na } args = append(args, "--", "cat", rd) - result, err := b.k8sClient.Kubectl(args...) + result, err := k8sh.Kubectl(args...) if err != nil { return "", fmt.Errorf("Unable to read data to pod -- : %s", err) } return result, nil - -} - -// BlockUnmap Function to unmap a Block using Rook -// Input parameters - -// manifest - pod definition - the pod described in yaml file is deleted -// mountpath - not used in this impl since mountpath is defined in the pod definition -// Output - k8s delete pod operation output and/or error -func (b *BlockOperation) BlockUnmap(name string, mountpath string) (string, error) { - args := []string{"delete", "-f", "-"} - result, err := b.k8sClient.KubectlWithStdin(name, args...) - if err != nil { - return "", fmt.Errorf("Unable to unmap block -- : %s", err) - - } - return result, nil - } diff --git a/tests/framework/clients/filesystem.go b/tests/framework/clients/filesystem.go index a05540f86..7e4e87b2d 100644 --- a/tests/framework/clients/filesystem.go +++ b/tests/framework/clients/filesystem.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -27,12 +28,13 @@ import ( //FilesystemOperation is a wrapper for k8s rook file operations type FilesystemOperation struct { - k8sh *utils.K8sHelper + k8sh *utils.K8sHelper + manifests installer.CephManifests } -// CreateK8sFilesystemOperation Constructor to create FilesystemOperation - client to perform rook file system operations on k8s -func CreateK8sFilesystemOperation(k8sh *utils.K8sHelper) *FilesystemOperation { - return &FilesystemOperation{k8sh: k8sh} +// CreateFilesystemOperation Constructor to create FilesystemOperation - client to perform rook file system operations on k8s +func CreateFilesystemOperation(k8sh *utils.K8sHelper, manifests installer.CephManifests) *FilesystemOperation { + return &FilesystemOperation{k8sh, manifests} } // FSCreate Function to create a filesystem in rook @@ -42,24 +44,7 @@ func CreateK8sFilesystemOperation(k8sh *utils.K8sHelper) *FilesystemOperation { func (f *FilesystemOperation) Create(name, namespace string) error { logger.Infof("creating the filesystem via CRD") - fsSpec := fmt.Sprintf(`apiVersion: ceph.rook.io/v1beta1 -kind: Filesystem -metadata: - name: %s - namespace: %s -spec: - metadataPool: - replicated: - size: 1 - dataPools: - - replicated: - size: 1 - metadataServer: - activeCount: 1 - activeStandby: true -`, name, namespace) - - if _, err := f.k8sh.ResourceOperation("create", fsSpec); err != nil { + if _, err := f.k8sh.ResourceOperation("create", f.manifests.GetFilesystem(namespace, name)); err != nil { return err } @@ -93,21 +78,6 @@ func (f *FilesystemOperation) List(namespace string) ([]client.CephFilesystem, e return filesystems, nil } -// Mount Function to Mount a file system created by rook(on a pod) -// Input parameters - -// name - path to the yaml definition file - definition of pod to be created that mounts existing file system -// path - ignored in this case - mount path is defined in the path definition -// output - output returned by k8s create pod operaton and/or error -func (f *FilesystemOperation) Mount(name string, path string) (string, error) { - result, err := f.k8sh.Kubectl("create", "-f", "name") - - if err != nil { - return "", fmt.Errorf("Unable to mount Filesystem -- : %s", err) - } - return result, nil - -} - // Write Function to write data to file system created by rook ,i.e. write data to a pod that has filesystem mounted // Input parameters - // name - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier @@ -116,22 +86,8 @@ func (f *FilesystemOperation) Mount(name string, path string) (string, error) { // filename - file where data is written to // namespace - optional param - namespace of the pod // Output - k8s exec pod operation output and/or error -func (f *FilesystemOperation) Write(name string, mountpath string, data string, filename string, namespace string) (string, error) { - wt := "echo \"" + data + "\">" + mountpath + "/" + filename - args := []string{"exec", name} - - if namespace != "" { - args = append(args, "-n", namespace) - } - args = append(args, "--", "sh", "-c", wt) - - result, err := f.k8sh.Kubectl(args...) - if err != nil { - return "", fmt.Errorf("Unable to write data to pod -- : %s", err) - - } - return result, nil - +func (f *FilesystemOperation) Write(name string, data string, filename string, namespace string) (string, error) { + return writeToPod(f.k8sh, name, data, filename, namespace) } // Read Function to write read from file system created by rook ,i.e. Read data from a pod that filesystem mounted @@ -141,35 +97,6 @@ func (f *FilesystemOperation) Write(name string, mountpath string, data string, // filename - file to be read // namespace - optional param - namespace of the pod // Output - k8s exec pod operation output and/or error -func (f *FilesystemOperation) Read(name string, mountpath string, filename string, namespace string) (string, error) { - rd := mountpath + "/" + filename - - args := []string{"exec", name} - - if namespace != "" { - args = append(args, "-n", namespace) - } - args = append(args, "--", "cat", rd) - - result, err := f.k8sh.Kubectl(args...) - if err != nil { - return "", fmt.Errorf("Unable to write data to pod -- : %s", err) - - } - return result, nil - -} - -// Unmount Function to UnMount a file system created by rook(delete pod) -// Input parameters - -// name - path to the yaml definition file - definition of pod to be deleted that has a file system mounted -// path - ignored in this case - mount path is defined in the path definition -// output - output returned by k8s delete pod operaton and/or error -func (f *FilesystemOperation) Unmount(name string) (string, error) { - result, err := f.k8sh.Kubectl("delete", "-f", name) - if err != nil { - return "", fmt.Errorf("Unable to unmount Filesystem -- : %s", err) - - } - return result, nil +func (f *FilesystemOperation) Read(name string, filename string, namespace string) (string, error) { + return readFromPod(f.k8sh, name, filename, namespace) } diff --git a/tests/framework/clients/object.go b/tests/framework/clients/object.go index 9625c7957..0009bf94f 100644 --- a/tests/framework/clients/object.go +++ b/tests/framework/clients/object.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/coreos/pkg/capnslog" + "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" ) @@ -29,60 +30,40 @@ var logger = capnslog.NewPackageLogger("github.com/rook/rook/tests", "clients") // ObjectOperation is wrapper for k8s rook object operations type ObjectOperation struct { - k8sh *utils.K8sHelper + k8sh *utils.K8sHelper + manifests installer.CephManifests } // CreateObjectOperation creates new rook object client -func CreateObjectOperation(k8sh *utils.K8sHelper) *ObjectOperation { - return &ObjectOperation{k8sh: k8sh} +func CreateObjectOperation(k8sh *utils.K8sHelper, manifests installer.CephManifests) *ObjectOperation { + return &ObjectOperation{k8sh, manifests} } // ObjectCreate Function to create a object store in rook -func (ro *ObjectOperation) Create(namespace, storeName string, replicaCount int32) error { +func (o *ObjectOperation) Create(namespace, storeName string, replicaCount int32) error { logger.Infof("creating the object store via CRD") - storeSpec := fmt.Sprintf(`apiVersion: ceph.rook.io/v1beta1 -kind: ObjectStore -metadata: - name: %s - namespace: %s -spec: - metadataPool: - replicated: - size: 1 - dataPool: - replicated: - size: 1 - gateway: - type: s3 - sslCertificateRef: - port: %d - securePort: - instances: %d - allNodes: false -`, storeName, namespace, rgwPort, replicaCount) - - if _, err := ro.k8sh.ResourceOperation("create", storeSpec); err != nil { + if _, err := o.k8sh.ResourceOperation("create", o.manifests.GetObjectStore(namespace, storeName, int(replicaCount), rgwPort)); err != nil { return err } - err := ro.k8sh.WaitForLabeledPodToRun(fmt.Sprintf("rook_object_store=%s", storeName), namespace) + err := o.k8sh.WaitForLabeledPodToRun(fmt.Sprintf("rook_object_store=%s", storeName), namespace) if err != nil { return fmt.Errorf("rgw did not start via crd. %+v", err) } // create the external service - return ro.k8sh.CreateExternalRGWService(namespace, storeName) + return o.k8sh.CreateExternalRGWService(namespace, storeName) } -func (ro *ObjectOperation) Delete(namespace, storeName string) error { +func (o *ObjectOperation) Delete(namespace, storeName string) error { logger.Infof("Deleting the object store via CRD") - if _, err := ro.k8sh.DeleteResource("-n", namespace, "ObjectStore", storeName); err != nil { + if _, err := o.k8sh.DeleteResource("-n", namespace, "ObjectStore", storeName); err != nil { return err } - if !ro.k8sh.WaitUntilPodWithLabelDeleted(fmt.Sprintf("rook_object_store=%s", storeName), namespace) { + if !o.k8sh.WaitUntilPodWithLabelDeleted(fmt.Sprintf("rook_object_store=%s", storeName), namespace) { return fmt.Errorf("rgw did not stop via crd") } return nil diff --git a/tests/framework/clients/pool.go b/tests/framework/clients/pool.go index b4597a7a2..55981c062 100644 --- a/tests/framework/clients/pool.go +++ b/tests/framework/clients/pool.go @@ -22,7 +22,6 @@ import ( cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" "github.com/rook/rook/pkg/daemon/ceph/client" - "github.com/rook/rook/pkg/daemon/ceph/model" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" "k8s.io/apimachinery/pkg/api/errors" @@ -31,24 +30,30 @@ import ( //PoolOperation is a wrapper for rook pool operations type PoolOperation struct { - k8sh *utils.K8sHelper + k8sh *utils.K8sHelper + manifests installer.CephManifests } // CreatePoolOperation creates a new pool client -func CreatePoolOperation(k8sh *utils.K8sHelper) *PoolOperation { - return &PoolOperation{k8sh: k8sh} +func CreatePoolOperation(k8sh *utils.K8sHelper, manifests installer.CephManifests) *PoolOperation { + return &PoolOperation{k8sh, manifests} } -func (p *PoolOperation) Create(pool model.Pool, namespace string) (string, error) { - return p.createOrUpdatePool(pool, namespace, "create") +func (p *PoolOperation) Create(name, namespace string, replicas int) (string, error) { + return p.createOrUpdatePool(name, namespace, "create", replicas) } -func (p *PoolOperation) Update(pool model.Pool, namespace string) (string, error) { - return p.createOrUpdatePool(pool, namespace, "apply") +func (p *PoolOperation) Delete(name string, namespace string) error { + _, err := p.k8sh.ResourceOperation("delete", p.manifests.GetBlockPoolDef(name, namespace, "1")) + return err } -func (p *PoolOperation) createOrUpdatePool(pool model.Pool, namespace, action string) (string, error) { - return installer.BlockResourceOperation(p.k8sh, installer.GetBlockPoolDef(pool.Name, namespace, strconv.Itoa(int(pool.ReplicatedConfig.Size))), action) +func (p *PoolOperation) Update(name, namespace string, replicas int) (string, error) { + return p.createOrUpdatePool(name, namespace, "apply", replicas) +} + +func (p *PoolOperation) createOrUpdatePool(name, namespace, action string, replicas int) (string, error) { + return p.k8sh.ResourceOperation(action, p.manifests.GetBlockPoolDef(name, namespace, strconv.Itoa(replicas))) } func (p *PoolOperation) ListCephPools(namespace string) ([]client.CephStoragePoolSummary, error) { @@ -104,3 +109,21 @@ func (p *PoolOperation) CephPoolExists(namespace, name string) (bool, error) { } return false, nil } + +func (p *PoolOperation) CreateStorageClassAndPvc(namespace, poolName, storageClassName, blockName, mode string) (string, error) { + return p.k8sh.ResourceOperation("create", p.manifests.GetBlockPoolStorageClassAndPvcDef(namespace, poolName, storageClassName, blockName, mode)) +} + +func (p *PoolOperation) DeleteStorageClass(namespace, poolName, storageClassName string) error { + _, err := p.k8sh.ResourceOperation("delete", p.manifests.GetBlockPoolStorageClass(namespace, poolName, storageClassName)) + return err +} + +func (p *PoolOperation) DeleteStorageClassAndPvc(namespace, poolName, storageClassName, blockName, mode string) error { + _, err := p.k8sh.ResourceOperation("delete", p.manifests.GetBlockPoolStorageClassAndPvcDef(namespace, poolName, storageClassName, blockName, mode)) + return err +} + +func (p *PoolOperation) CreateStorageClass(namespace, poolName, storageClassName string) (string, error) { + return p.k8sh.ResourceOperation("create", p.manifests.GetBlockPoolStorageClass(namespace, poolName, storageClassName)) +} diff --git a/tests/framework/clients/test_client.go b/tests/framework/clients/test_client.go index 3fd4cdfd6..db87010b8 100644 --- a/tests/framework/clients/test_client.go +++ b/tests/framework/clients/test_client.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" ) @@ -41,15 +42,15 @@ const ( ) //CreateTestClient creates new instance of test client for a platform -func CreateTestClient(k8sHelper *utils.K8sHelper, namespace string) (*TestClient, error) { +func CreateTestClient(k8sHelper *utils.K8sHelper, manifests installer.CephManifests) *TestClient { return &TestClient{ - CreateK8BlockOperation(k8sHelper), - CreateK8sFilesystemOperation(k8sHelper), - CreateObjectOperation(k8sHelper), - CreatePoolOperation(k8sHelper), + CreateBlockOperation(k8sHelper, manifests), + CreateFilesystemOperation(k8sHelper, manifests), + CreateObjectOperation(k8sHelper, manifests), + CreatePoolOperation(k8sHelper, manifests), k8sHelper, - }, nil + } } //Status returns rook status details diff --git a/tests/framework/contracts/operations.go b/tests/framework/contracts/operations.go deleted file mode 100644 index 66da97474..000000000 --- a/tests/framework/contracts/operations.go +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package contracts - -type Setup interface { - SetUp() - TearDown() -} diff --git a/tests/framework/installer/install_helper.go b/tests/framework/installer/ceph_installer.go similarity index 64% rename from tests/framework/installer/install_helper.go rename to tests/framework/installer/ceph_installer.go index 60c7b0226..2d78c109c 100644 --- a/tests/framework/installer/install_helper.go +++ b/tests/framework/installer/ceph_installer.go @@ -22,24 +22,17 @@ import ( "math/rand" "os" "path" - "runtime" - "strings" "testing" - "time" "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/kubelet/apis" "flag" - "github.com/coreos/pkg/capnslog" cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" - "github.com/rook/rook/pkg/util/exec" - "github.com/rook/rook/pkg/util/sys" - "github.com/rook/rook/tests/framework/objects" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/api/errors" @@ -49,71 +42,43 @@ import ( const ( rookOperatorCreatedCrd = "clusters.ceph.rook.io" - cockroachDBCRD = "clusters.cockroachdb.rook.io" - nfsServerCRD = "nfsservers.nfs.rook.io" helmChartName = "local/rook-ceph" helmDeployName = "rook-ceph" ) -var ( - // ** Variables that might need to be changed depending on the dev environment. The init function below will modify some of them automatically. ** - baseTestDir string - forceUseDevices = false - createBaseTestDir = true - // ** end of Variables to modify - logger = capnslog.NewPackageLogger("github.com/rook/rook", "installer") - createArgs = []string{"create", "-f"} - createFromStdinArgs = append(createArgs, "-") - deleteArgs = []string{"delete", "-f"} - deleteFromStdinArgs = append(deleteArgs, "-") -) - -func init() { - // this default will only work if running kubernetes on the local machine - baseTestDir, _ = os.Getwd() - - // The following settings could apply to any environment when the kube context is running on the host and the tests are running inside a - // VM such as minikube. This is a cheap test for this condition, we need to find a better way to automate these settings. - if runtime.GOOS == "darwin" { - createBaseTestDir = false - baseTestDir = "/data" - } -} - -//InstallHelper wraps installing and uninstalling rook on a platform -type InstallHelper struct { +//CephInstaller wraps installing and uninstalling rook on a platform +type CephInstaller struct { + Manifests CephManifests k8shelper *utils.K8sHelper - installData *InstallData hostPathToDelete string helmHelper *utils.HelmHelper - Env objects.EnvironmentManifest k8sVersion string T func() *testing.T } -func (h *InstallHelper) CreateK8sRookResources() (err error) { +func (h *CephInstaller) CreateCephCRDs() (err error) { var resources string logger.Info("Creating Rook CRDs") - resources = h.installData.GetRookCRDs() + resources = h.Manifests.GetRookCRDs() _, err = h.k8shelper.KubectlWithStdin(resources, createFromStdinArgs...) return } -//CreateK8sRookOperator creates rook-operator via kubectl -func (h *InstallHelper) CreateK8sRookOperator(namespace string) (err error) { +//CreateCephOperator creates rook-operator via kubectl +func (h *CephInstaller) CreateCephOperator(namespace string) (err error) { logger.Infof("Starting Rook Operator") //creating clusterrolebinding for kubeadm env. h.k8shelper.CreateAnonSystemClusterBinding() //creating rook resources - if err = h.CreateK8sRookResources(); err != nil { + if err = h.CreateCephCRDs(); err != nil { return err } - rookOperator := h.installData.GetRookOperator(namespace) + rookOperator := h.Manifests.GetRookOperator(namespace) _, err = h.k8shelper.KubectlWithStdin(rookOperator, createFromStdinArgs...) if err != nil { @@ -130,7 +95,7 @@ func (h *InstallHelper) CreateK8sRookOperator(namespace string) (err error) { } //CreateK8sRookOperatorViaHelm creates rook operator via Helm chart named local/rook present in local repo -func (h *InstallHelper) CreateK8sRookOperatorViaHelm(namespace string) error { +func (h *CephInstaller) CreateK8sRookOperatorViaHelm(namespace string) error { //creating clusterrolebinding for kubeadm env. h.k8shelper.CreateAnonSystemClusterBinding() @@ -154,10 +119,10 @@ func (h *InstallHelper) CreateK8sRookOperatorViaHelm(namespace string) error { } //CreateK8sRookToolbox creates rook-ceph-tools via kubectl -func (h *InstallHelper) CreateK8sRookToolbox(namespace string) (err error) { +func (h *CephInstaller) CreateK8sRookToolbox(namespace string) (err error) { logger.Infof("Starting Rook toolbox") - rookToolbox := h.installData.GetRookToolBox(namespace) + rookToolbox := h.Manifests.GetRookToolBox(namespace) _, err = h.k8shelper.KubectlWithStdin(rookToolbox, createFromStdinArgs...) @@ -173,13 +138,13 @@ func (h *InstallHelper) CreateK8sRookToolbox(namespace string) (err error) { return nil } -func (h *InstallHelper) CreateK8sRookCluster(namespace, systemNamespace string, storeType string) (err error) { +func (h *CephInstaller) CreateK8sRookCluster(namespace, systemNamespace string, storeType string) (err error) { return h.CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType, false, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, true /* startWithAllNodes */) } //CreateK8sRookCluster creates rook cluster via kubectl -func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType string, +func (h *CephInstaller) CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType string, useAllDevices bool, mon cephv1beta1.MonSpec, startWithAllNodes bool) error { dataDirHostPath, err := h.initTestDir(namespace) @@ -197,7 +162,7 @@ func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, sy } logger.Infof("Creating cluster roles") - roles := h.installData.GetClusterRoles(namespace, systemNamespace) + roles := h.Manifests.GetClusterRoles(namespace, systemNamespace) if _, err := h.k8shelper.KubectlWithStdin(roles, createFromStdinArgs...); err != nil { return fmt.Errorf("Failed to create cluster roles. %+v", err) } @@ -262,7 +227,7 @@ func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, sy } } else { logger.Infof("Starting Rook Cluster with yaml") - rookCluster := h.installData.GetRookCluster(namespace, storeType, dataDirHostPath, useAllDevices, mon.Count) + rookCluster := h.Manifests.GetRookCluster(&ClusterSettings{namespace, storeType, dataDirHostPath, useAllDevices, mon.Count}) if _, err := h.k8shelper.KubectlWithStdin(rookCluster, createFromStdinArgs...); err != nil { return fmt.Errorf("Failed to create rook cluster : %v ", err) } @@ -281,7 +246,7 @@ func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, sy return err } -func (h *InstallHelper) initTestDir(namespace string) (string, error) { +func (h *CephInstaller) initTestDir(namespace string) (string, error) { h.hostPathToDelete = path.Join(baseTestDir, "rook-test") testDir := path.Join(h.hostPathToDelete, namespace) @@ -303,7 +268,7 @@ func (h *InstallHelper) initTestDir(namespace string) (string, error) { return testDir, nil } -func (h *InstallHelper) GetNodeHostnames() ([]string, error) { +func (h *CephInstaller) GetNodeHostnames() ([]string, error) { nodes, err := h.k8shelper.Clientset.CoreV1().Nodes().List(metav1.ListOptions{}) if err != nil { return nil, fmt.Errorf("failed to get k8s nodes. %+v", err) @@ -316,17 +281,13 @@ func (h *InstallHelper) GetNodeHostnames() ([]string, error) { return names, nil } -func SystemNamespace(namespace string) string { - return fmt.Sprintf("%s-system", namespace) -} - //InstallRookOnK8sWithHostPathAndDevices installs rook on k8s -func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeType string, +func (h *CephInstaller) InstallRookOnK8sWithHostPathAndDevices(namespace, storeType string, helmInstalled, useDevices bool, mon cephv1beta1.MonSpec, startWithAllNodes bool) (bool, error) { var err error //flag used for local debuggin purpose, when rook is pre-installed - if h.Env.SkipInstallRook { + if Env.SkipInstallRook { return true, nil } @@ -346,7 +307,7 @@ func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeT } else { onamespace = SystemNamespace(namespace) - err := h.CreateK8sRookOperator(onamespace) + err := h.CreateCephOperator(onamespace) if err != nil { logger.Errorf("Rook Operator not installed ,error -> %v", err) return false, err @@ -354,7 +315,7 @@ func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeT } if !h.k8shelper.IsPodInExpectedState("rook-ceph-operator", onamespace, "Running") { logger.Error("rook-ceph-operator is not running") - h.k8shelper.GetRookLogs("rook-ceph-operator", h.Env.HostType, onamespace, "test-setup") + h.k8shelper.GetRookLogs("rook-ceph-operator", Env.HostType, onamespace, "test-setup") logger.Error("rook-ceph-operator is not Running, abort!") return false, err } @@ -387,14 +348,14 @@ func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeT } //UninstallRookFromK8s uninstalls rook from k8s -func (h *InstallHelper) UninstallRook(helmInstalled bool, namespace string) { +func (h *CephInstaller) UninstallRook(helmInstalled bool, namespace string) { h.UninstallRookFromMultipleNS(helmInstalled, SystemNamespace(namespace), namespace) } //UninstallRookFromK8s uninstalls rook from multiple namespaces in k8s -func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNamespace string, namespaces ...string) { +func (h *CephInstaller) UninstallRookFromMultipleNS(helmInstalled bool, systemNamespace string, namespaces ...string) { //flag used for local debugging purpose, when rook is pre-installed - if h.Env.SkipInstallRook { + if Env.SkipInstallRook { return } @@ -404,43 +365,43 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa if !h.k8shelper.VersionAtLeast("v1.8.0") { _, err = h.k8shelper.DeleteResource("-n", namespace, "serviceaccount", "rook-ceph-cluster") - h.checkError(err, "cannot remove serviceaccount rook-ceph-cluster") + checkError(h.T(), err, "cannot remove serviceaccount rook-ceph-cluster") assert.NoError(h.T(), err, "%s err -> %v", namespace, err) err = h.k8shelper.DeleteRoleAndBindings("rook-ceph-cluster", namespace) - h.checkError(err, "rook-ceph-cluster cluster role and binding cannot be deleted") + checkError(h.T(), err, "rook-ceph-cluster cluster role and binding cannot be deleted") assert.NoError(h.T(), err, "rook-ceph-cluster cluster role and binding cannot be deleted: %+v", err) err = h.k8shelper.DeleteRoleBinding("rook-ceph-cluster-mgmt", namespace) - h.checkError(err, "rook-ceph-cluster-mgmt binding cannot be deleted") + checkError(h.T(), err, "rook-ceph-cluster-mgmt binding cannot be deleted") assert.NoError(h.T(), err, "rook-ceph-cluster-mgmt binding cannot be deleted: %+v", err) } _, err = h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cluster.ceph.rook.io", namespace) - h.checkError(err, fmt.Sprintf("cannot remove cluster %s", namespace)) + checkError(h.T(), err, fmt.Sprintf("cannot remove cluster %s", namespace)) crdCheckerFunc := func() error { _, err := h.k8shelper.RookClientset.RookV1alpha1().Clusters(namespace).Get(namespace, metav1.GetOptions{}) return err } - err = h.waitForCustomResourceDeletion(namespace, crdCheckerFunc) - h.checkError(err, fmt.Sprintf("failed to wait for crd %s deletion", namespace)) + err = h.k8shelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc) + checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace)) _, err = h.k8shelper.DeleteResourceAndWait(false, "namespace", namespace) - h.checkError(err, fmt.Sprintf("cannot delete namespace %s", namespace)) + checkError(h.T(), err, fmt.Sprintf("cannot delete namespace %s", namespace)) } logger.Infof("removing the operator from namespace %s", systemNamespace) _, err = h.k8shelper.DeleteResource("crd", "clusters.ceph.rook.io", "pools.ceph.rook.io", "objectstores.ceph.rook.io", "filesystems.ceph.rook.io", "volumes.rook.io") - h.checkError(err, "cannot delete CRDs") + checkError(h.T(), err, "cannot delete CRDs") if helmInstalled { err = h.helmHelper.DeleteLocalRookHelmChart(helmDeployName) } else { - rookOperator := h.installData.GetRookOperator(systemNamespace) + rookOperator := h.Manifests.GetRookOperator(systemNamespace) _, err = h.k8shelper.KubectlWithStdin(rookOperator, deleteFromStdinArgs...) } - h.checkError(err, "cannot uninstall rook-operator") + checkError(h.T(), err, "cannot uninstall rook-operator") h.k8shelper.Clientset.RbacV1beta1().RoleBindings(systemNamespace).Delete("rook-ceph-system", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-ceph-global", nil) @@ -454,7 +415,7 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa // removing data dir if exists if h.hostPathToDelete != "" { nodes, err := h.GetNodeHostnames() - h.checkError(err, "cannot get node names") + checkError(h.T(), err, "cannot get node names") for _, node := range nodes { err = h.cleanupDir(node, h.hostPathToDelete) logger.Infof("removing %s from node %s. err=%v", h.hostPathToDelete, node, err) @@ -462,67 +423,27 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa } } -func (h *InstallHelper) cleanupDir(node, dir string) error { - resources := h.installData.GetCleanupPod(node, dir) +func (h *CephInstaller) cleanupDir(node, dir string) error { + resources := h.Manifests.GetCleanupPod(node, dir) _, err := h.k8shelper.KubectlWithStdin(resources, createFromStdinArgs...) return err } -func (h *InstallHelper) checkError(err error, message string) { - // During cleanup the resource might not be found because the test might have failed before the test was done and never created the resource - if err == nil || errors.IsNotFound(err) { - return - } - assert.NoError(h.T(), err, "%s. %+v", message, err) -} - -func (h *InstallHelper) waitForCustomResourceDeletion(namespace string, checkerFunc func() error) error { - if !h.k8shelper.VersionAtLeast("v1.8.0") { - // v1.7 has an intermittent issue with long delay to delete resources so we will skip waiting - return nil - } - - // wait for the operator to finalize and delete the CRD - for i := 0; i < 10; i++ { - err := checkerFunc() - if err == nil { - logger.Infof("custom resource %s still exists", namespace) - time.Sleep(2 * time.Second) - continue - } - if errors.IsNotFound(err) { - logger.Infof("custom resource %s deleted", namespace) - return nil - } - return err - } - logger.Errorf("gave up deleting custom resource %s", namespace) - return nil -} - -func (h *InstallHelper) GatherAllRookLogs(namespace, systemNamespace string, testName string) { +func (h *CephInstaller) GatherAllRookLogs(namespace, systemNamespace string, testName string) { logger.Infof("Gathering all logs from Rook Cluster %s", namespace) - h.k8shelper.GetRookLogs("rook-ceph-operator", h.Env.HostType, systemNamespace, testName) - h.k8shelper.GetRookLogs("rook-ceph-agent", h.Env.HostType, systemNamespace, testName) - h.k8shelper.GetRookLogs("rook-discover", h.Env.HostType, systemNamespace, testName) - h.k8shelper.GetRookLogs("rook-ceph-mgr", h.Env.HostType, namespace, testName) - h.k8shelper.GetRookLogs("rook-ceph-mon", h.Env.HostType, namespace, testName) - h.k8shelper.GetRookLogs("rook-ceph-osd", h.Env.HostType, namespace, testName) - h.k8shelper.GetRookLogs("rook-ceph-osd-prepare", h.Env.HostType, namespace, testName) - h.k8shelper.GetRookLogs("rook-ceph-rgw", h.Env.HostType, namespace, testName) - h.k8shelper.GetRookLogs("rook-ceph-mds", h.Env.HostType, namespace, testName) + h.k8shelper.GetRookLogs("rook-ceph-operator", Env.HostType, systemNamespace, testName) + h.k8shelper.GetRookLogs("rook-ceph-agent", Env.HostType, systemNamespace, testName) + h.k8shelper.GetRookLogs("rook-discover", Env.HostType, systemNamespace, testName) + h.k8shelper.GetRookLogs("rook-ceph-mgr", Env.HostType, namespace, testName) + h.k8shelper.GetRookLogs("rook-ceph-mon", Env.HostType, namespace, testName) + h.k8shelper.GetRookLogs("rook-ceph-osd", Env.HostType, namespace, testName) + h.k8shelper.GetRookLogs("rook-ceph-osd-prepare", Env.HostType, namespace, testName) + h.k8shelper.GetRookLogs("rook-ceph-rgw", Env.HostType, namespace, testName) + h.k8shelper.GetRookLogs("rook-ceph-mds", Env.HostType, namespace, testName) } -// GatherCRDObjectDebuggingInfo gathers all the descriptions for pods, pvs and pvcs -func (h *InstallHelper) GatherCRDObjectDebuggingInfo(namespace string) { - h.k8shelper.PrintPodDescribeForNamespace(namespace) - h.k8shelper.PrintPVs(true /*detailed*/) - h.k8shelper.PrintPVCs(namespace, true /*detailed*/) - h.k8shelper.PrintStorageClasses(true /*detailed*/) -} - -//NewK8sRookhelper creates new instance of InstallHelper -func NewK8sRookhelper(clientset *kubernetes.Clientset, t func() *testing.T) *InstallHelper { +// NewCephInstaller creates new instance of CephInstaller +func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, rookVersion string) *CephInstaller { // All e2e tests should run ceph commands in the toolbox since we are not inside a container client.RunAllCephCommandsInToolbox = true @@ -536,58 +457,13 @@ func NewK8sRookhelper(clientset *kubernetes.Clientset, t func() *testing.T) *Ins if err != nil { panic("failed to get kubectl client :" + err.Error()) } - ih := &InstallHelper{ - k8shelper: k8shelp, - installData: NewK8sInstallData(), - helmHelper: utils.NewHelmHelper(), - Env: objects.Env, - k8sVersion: version.String(), - T: t, + h := &CephInstaller{ + Manifests: NewCephManifests(rookVersion), + k8shelper: k8shelp, + helmHelper: utils.NewHelmHelper(), + k8sVersion: version.String(), + T: t, } flag.Parse() - return ih -} - -func IsAdditionalDeviceAvailableOnCluster() bool { - executor := &exec.CommandExecutor{} - devices, err := sys.ListDevices(executor) - if err != nil { - return false - } - disks := 0 - logger.Infof("devices : %v", devices) - for _, device := range devices { - if strings.Contains(device, "loop") { - continue - } - props, _ := sys.GetDeviceProperties(device, executor) - if props["TYPE"] != "disk" { - continue - } - - ownPartitions, fs, err := sys.CheckIfDeviceAvailable(executor, device) - if err != nil { - logger.Warningf("failed to detect device %s availability. %+v", device, err) - continue - } - if !ownPartitions { - logger.Infof("skipping device %s since don't own partitions", device) - continue - } - if fs != "" { - logger.Infof("skipping device %s since it has file system %s", device, fs) - continue - } - if strings.HasPrefix(device, "rbd") { - logger.Infof("skipping unexpected rbd device: %s", device) - continue - } - logger.Infof("available device: %s", device) - disks++ - } - if disks > 0 { - return true - } - logger.Info("No additional disks found on cluster") - return false + return h } diff --git a/tests/framework/installer/install_data.go b/tests/framework/installer/ceph_manifests.go similarity index 64% rename from tests/framework/installer/install_data.go rename to tests/framework/installer/ceph_manifests.go index 351b10091..001059336 100644 --- a/tests/framework/installer/install_data.go +++ b/tests/framework/installer/ceph_manifests.go @@ -17,21 +17,51 @@ limitations under the License. package installer import ( + "fmt" "strconv" "github.com/google/uuid" ) -//InstallData wraps rook yaml definitions -type InstallData struct { +type CephManifests interface { + GetRookCRDs() string + GetRookOperator(namespace string) string + GetClusterRoles(namespace, systemNamespace string) string + GetRookCluster(settings *ClusterSettings) string + GetRookToolBox(namespace string) string + GetCleanupPod(node, removalDir string) string + GetBlockPoolDef(poolName string, namespace string, replicaSize string) string + GetBlockStorageClassDef(poolName string, storageClassName string, namespace string, varClusterName bool) string + GetBlockPvcDef(claimName string, storageClassName string, accessModes string) string + GetBlockPoolStorageClassAndPvcDef(namespace string, poolName string, storageClassName string, blockName string, accessMode string) string + GetBlockPoolStorageClass(namespace string, poolName string, storageClassName string) string + GetFilesystem(namepace, name string) string + GetObjectStore(namespace, name string, replicaCount, port int) string } -//NewK8sInstallData creates new instance of InstallData struct -func NewK8sInstallData() *InstallData { - return &InstallData{} +type ClusterSettings struct { + Namespace string + StoreType string + DataDirHostPath string + UseAllDevices bool + Mons int } -func (i *InstallData) GetRookCRDs() string { +//CephManifestsMaster wraps rook yaml definitions +type CephManifestsMaster struct { + imageTag string +} + +// NewCephManifests gets the manifest type depending on the Rook version desired +func NewCephManifests(version string) CephManifests { + switch version { + case VersionMaster: + return &CephManifestsMaster{imageTag: VersionMaster} + } + panic(fmt.Errorf("unrecognized ceph manifest version: %s", version)) +} + +func (m *CephManifestsMaster) GetRookCRDs() string { return `apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -104,7 +134,7 @@ spec: } //GetRookOperator returns rook Operator manifest -func (i *InstallData) GetRookOperator(namespace string) string { +func (m *CephManifestsMaster) GetRookOperator(namespace string) string { return `kind: Namespace apiVersion: v1 @@ -307,7 +337,7 @@ spec: serviceAccountName: rook-ceph-system containers: - name: rook-ceph-operator - image: rook/ceph:master + image: rook/ceph:` + m.imageTag + ` args: ["ceph", "operator"] env: - name: ROOK_LOG_LEVEL @@ -331,7 +361,7 @@ spec: } //GetRookCluster returns rook-cluster manifest -func (i *InstallData) GetClusterRoles(namespace, systemNamespace string) string { +func (m *CephManifestsMaster) GetClusterRoles(namespace, systemNamespace string) string { return `apiVersion: v1 kind: ServiceAccount metadata: @@ -380,36 +410,36 @@ subjects: } //GetRookCluster returns rook-cluster manifest -func (i *InstallData) GetRookCluster(namespace, storeType, dataDirHostPath string, useAllDevices bool, mons int) string { +func (m *CephManifestsMaster) GetRookCluster(settings *ClusterSettings) string { return `apiVersion: ceph.rook.io/v1beta1 kind: Cluster metadata: - name: ` + namespace + ` - namespace: ` + namespace + ` + name: ` + settings.Namespace + ` + namespace: ` + settings.Namespace + ` spec: serviceAccount: rook-ceph-cluster - dataDirHostPath: ` + dataDirHostPath + ` + dataDirHostPath: ` + settings.DataDirHostPath + ` network: hostNetwork: false mon: - count: ` + strconv.Itoa(mons) + ` + count: ` + strconv.Itoa(settings.Mons) + ` allowMultiplePerNode: true dashboard: enabled: true metadataDevice: storage: useAllNodes: true - useAllDevices: ` + strconv.FormatBool(useAllDevices) + ` + useAllDevices: ` + strconv.FormatBool(settings.UseAllDevices) + ` deviceFilter: location: config: - storeType: "` + storeType + `" + storeType: "` + settings.StoreType + `" databaseSizeMB: "1024" journalSizeMB: "1024"` } //GetRookToolBox returns rook-toolbox manifest -func (i *InstallData) GetRookToolBox(namespace string) string { +func (m *CephManifestsMaster) GetRookToolBox(namespace string) string { return `apiVersion: v1 kind: Pod metadata: @@ -419,7 +449,7 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: rook-ceph-tools - image: rook/ceph-toolbox:master + image: rook/ceph-toolbox:` + m.imageTag + ` imagePullPolicy: IfNotPresent env: - name: ROOK_ADMIN_SECRET @@ -458,7 +488,7 @@ spec: } //GetCleanupPod gets a cleanup Pod manifest -func (i *InstallData) GetCleanupPod(node, removalDir string) string { +func (m *CephManifestsMaster) GetCleanupPod(node, removalDir string) string { return `apiVersion: batch/v1 kind: Job metadata: @@ -469,7 +499,7 @@ spec: restartPolicy: Never containers: - name: rook-cleaner - image: rook/rook:master + image: rook/ceph:` + m.imageTag + ` securityContext: privileged: true volumeMounts: @@ -486,3 +516,94 @@ spec: hostPath: path: ` + removalDir } + +func (m *CephManifestsMaster) GetBlockPoolDef(poolName string, namespace string, replicaSize string) string { + return `apiVersion: ceph.rook.io/v1beta1 +kind: Pool +metadata: + name: ` + poolName + ` + namespace: ` + namespace + ` +spec: + replicated: + size: ` + replicaSize +} + +func (m *CephManifestsMaster) GetBlockStorageClassDef(poolName string, storageClassName string, namespace string, varClusterName bool) string { + namespaceParameter := "clusterNamespace" + if varClusterName { + namespaceParameter = "clusterName" + } + return `apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ` + storageClassName + ` +provisioner: ceph.rook.io/block +parameters: + pool: ` + poolName + ` + ` + namespaceParameter + `: ` + namespace +} + +func (m *CephManifestsMaster) GetBlockPvcDef(claimName string, storageClassName string, accessModes string) string { + return `apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ` + claimName + ` + annotations: + volume.beta.kubernetes.io/storage-class: ` + storageClassName + ` +spec: + accessModes: + - ` + accessModes + ` + resources: + requests: + storage: 1M` +} + +func (m *CephManifestsMaster) GetBlockPoolStorageClassAndPvcDef(namespace string, poolName string, storageClassName string, blockName string, accessMode string) string { + return concatYaml(m.GetBlockPoolDef(poolName, namespace, "1"), + concatYaml(m.GetBlockStorageClassDef(poolName, storageClassName, namespace, false), m.GetBlockPvcDef(blockName, storageClassName, accessMode))) +} + +func (m *CephManifestsMaster) GetBlockPoolStorageClass(namespace string, poolName string, storageClassName string) string { + return concatYaml(m.GetBlockPoolDef(poolName, namespace, "1"), m.GetBlockStorageClassDef(poolName, storageClassName, namespace, false)) +} + +func (m *CephManifestsMaster) GetFilesystem(namespace, name string) string { + return `apiVersion: ceph.rook.io/v1beta1 +kind: Filesystem +metadata: + name: ` + name + ` + namespace: ` + namespace + ` +spec: + metadataPool: + replicated: + size: 1 + dataPools: + - replicated: + size: 1 + metadataServer: + activeCount: 1 + activeStandby: true` +} + +func (m *CephManifestsMaster) GetObjectStore(namespace, name string, replicaCount, port int) string { + return `apiVersion: ceph.rook.io/v1beta1 +kind: ObjectStore +metadata: + name: ` + name + ` + namespace: ` + namespace + ` +spec: + metadataPool: + replicated: + size: 1 + dataPool: + replicated: + size: 1 + gateway: + type: s3 + sslCertificateRef: + port: ` + strconv.Itoa(port) + ` + securePort: + instances: ` + strconv.Itoa(replicaCount) + ` + allNodes: false +` +} diff --git a/tests/framework/installer/cockroachdb.go b/tests/framework/installer/cockroachdb.go deleted file mode 100644 index 0dda4132b..000000000 --- a/tests/framework/installer/cockroachdb.go +++ /dev/null @@ -1,270 +0,0 @@ -/* -Copyright 2018 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package installer - -import ( - "fmt" - "strconv" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func (h *InstallHelper) InstallCockroachDB(systemNamespace, namespace string, count int) error { - h.k8shelper.CreateAnonSystemClusterBinding() - - // install hostpath provisioner if there isn't already a default storage class - defaultExists, err := h.k8shelper.IsDefaultStorageClassPresent() - if err != nil { - return err - } else if !defaultExists { - if err := h.InstallHostPathProvisioner(); err != nil { - return err - } - } else { - logger.Info("skipping install of host path provisioner because a default storage class already exists") - } - - // install cockroachdb operator - if err := h.CreateCockroachDBOperator(systemNamespace); err != nil { - return err - } - - // install cockroachdb cluster instance - if err := h.CreateCockroachDBCluster(namespace, count); err != nil { - return err - } - - return nil -} - -func (h *InstallHelper) CreateCockroachDBOperator(namespace string) error { - logger.Infof("starting cockroachDB operator") - - logger.Info("creating cockroachDB CRDs") - if _, err := h.k8shelper.KubectlWithStdin(h.installData.GetCockroachDBCRDs(), createFromStdinArgs...); err != nil { - return err - } - - cockroachDBOperator := h.installData.GetCockroachDBOperator(namespace) - _, err := h.k8shelper.KubectlWithStdin(cockroachDBOperator, createFromStdinArgs...) - if err != nil { - return fmt.Errorf("failed to create rook-cockroachdb-operator pod: %+v ", err) - } - - if !h.k8shelper.IsCRDPresent(cockroachDBCRD) { - return fmt.Errorf("failed to find cockroachdb CRD %s", cockroachDBCRD) - } - - if !h.k8shelper.IsPodInExpectedState("rook-cockroachdb-operator", namespace, "Running") { - return fmt.Errorf("rook-cockroachdb-operator is not running, aborting") - } - - logger.Infof("cockroachdb operator started") - return nil -} - -func (h *InstallHelper) CreateCockroachDBCluster(namespace string, count int) error { - if err := h.k8shelper.CreateNamespace(namespace); err != nil { - return err - } - - logger.Infof("starting cockroachdb cluster with kubectl and yaml") - cockroachDBCluster := h.installData.GetCockroachDBCluster(namespace, count) - if _, err := h.k8shelper.KubectlWithStdin(cockroachDBCluster, createFromStdinArgs...); err != nil { - return fmt.Errorf("Failed to create cockroachdb cluster: %+v ", err) - } - - if err := h.k8shelper.WaitForPodCount("app=rook-cockroachdb", namespace, count); err != nil { - logger.Errorf("cockroachdb cluster pods in namespace %s not found", namespace) - return err - } - - err := h.k8shelper.WaitForLabeledPodToRun("app=rook-cockroachdb", namespace) - if err != nil { - logger.Errorf("cockroachdb cluster pods in namespace %s are not running", namespace) - return err - } - - logger.Infof("cockroachdb cluster started") - return nil -} - -func (h *InstallHelper) UninstallCockroachDB(systemNamespace, namespace string) { - logger.Infof("uninstalling cockroachdb from namespace %s", namespace) - - _, err := h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cluster.cockroachdb.rook.io", namespace) - h.checkError(err, fmt.Sprintf("cannot remove cluster %s", namespace)) - - crdCheckerFunc := func() error { - _, err := h.k8shelper.RookClientset.CockroachdbV1alpha1().Clusters(namespace).Get(namespace, metav1.GetOptions{}) - return err - } - err = h.waitForCustomResourceDeletion(namespace, crdCheckerFunc) - h.checkError(err, fmt.Sprintf("failed to wait for crd %s deletion", namespace)) - - _, err = h.k8shelper.DeleteResourceAndWait(false, "namespace", namespace) - h.checkError(err, fmt.Sprintf("cannot delete namespace %s", namespace)) - - logger.Infof("removing the operator from namespace %s", systemNamespace) - _, err = h.k8shelper.DeleteResource("crd", "clusters.cockroachdb.rook.io") - h.checkError(err, "cannot delete CRDs") - - cockroachDBOperator := h.installData.GetCockroachDBOperator(systemNamespace) - _, err = h.k8shelper.KubectlWithStdin(cockroachDBOperator, deleteFromStdinArgs...) - h.checkError(err, "cannot uninstall rook-cockroachdb-operator") - - err = h.UninstallHostPathProvisioner() - h.checkError(err, "cannot uninstall hostpath provisioner") - - h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil) - logger.Infof("done removing the operator from namespace %s", systemNamespace) -} - -func (h *InstallHelper) GatherAllCockroachDBLogs(systemNamespace, namespace, testName string) { - logger.Infof("Gathering all logs from cockroachdb cluster %s", namespace) - h.k8shelper.GetRookLogs("rook-cockroachdb-operator", h.Env.HostType, systemNamespace, testName) - h.k8shelper.GetRookLogs("rook-cockroachdb", h.Env.HostType, namespace, testName) -} - -func (i *InstallData) GetCockroachDBCRDs() string { - return `apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: clusters.cockroachdb.rook.io -spec: - group: cockroachdb.rook.io - names: - kind: Cluster - listKind: ClusterList - plural: clusters - singular: cluster - scope: Namespaced - version: v1alpha1 -` -} - -func (i *InstallData) GetCockroachDBOperator(namespace string) string { - - return `kind: Namespace -apiVersion: v1 -metadata: - name: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rook-cockroachdb-operator -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - create - - update -- apiGroups: - - apps - resources: - - statefulsets - verbs: - - create -- apiGroups: - - policy - resources: - - poddisruptionbudgets - verbs: - - create -- apiGroups: - - cockroachdb.rook.io - resources: - - "*" - verbs: - - "*" ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-cockroachdb-operator - namespace: ` + namespace + ` ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-cockroachdb-operator - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-cockroachdb-operator -subjects: -- kind: ServiceAccount - name: rook-cockroachdb-operator - namespace: ` + namespace + ` ---- -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: rook-cockroachdb-operator - namespace: ` + namespace + ` -spec: - replicas: 1 - template: - metadata: - labels: - app: rook-cockroachdb-operator - spec: - serviceAccountName: rook-cockroachdb-operator - containers: - - name: rook-cockroachdb-operator - image: rook/cockroachdb:master - args: ["cockroachdb", "operator"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace -` -} - -func (i *InstallData) GetCockroachDBCluster(namespace string, count int) string { - return `apiVersion: cockroachdb.rook.io/v1alpha1 -kind: Cluster -metadata: - name: ` + namespace + ` - namespace: ` + namespace + ` -spec: - scope: - nodeCount: ` + strconv.Itoa(count) + ` - secure: false - volumeSize: 1Gi - cachePercent: 25 - maxSQLMemoryPercent: 25 -` -} diff --git a/tests/framework/installer/cockroachdb_installer.go b/tests/framework/installer/cockroachdb_installer.go new file mode 100644 index 000000000..636f4b1e3 --- /dev/null +++ b/tests/framework/installer/cockroachdb_installer.go @@ -0,0 +1,156 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installer + +import ( + "fmt" + "testing" + + "github.com/rook/rook/tests/framework/utils" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + cockroachDBCRD = "clusters.cockroachdb.rook.io" +) + +type CockroachDBInstaller struct { + k8shelper *utils.K8sHelper + manifests *CockroachDBManifests + T func() *testing.T +} + +func NewCockroachDBInstaller(k8shelper *utils.K8sHelper, t func() *testing.T) *CockroachDBInstaller { + return &CockroachDBInstaller{k8shelper, &CockroachDBManifests{}, t} +} + +func (h *CockroachDBInstaller) InstallCockroachDB(systemNamespace, namespace string, count int) error { + h.k8shelper.CreateAnonSystemClusterBinding() + + // install hostpath provisioner if there isn't already a default storage class + defaultExists, err := h.k8shelper.IsDefaultStorageClassPresent() + if err != nil { + return err + } else if !defaultExists { + if err := InstallHostPathProvisioner(h.k8shelper); err != nil { + return err + } + } else { + logger.Info("skipping install of host path provisioner because a default storage class already exists") + } + + // install cockroachdb operator + if err := h.CreateCockroachDBOperator(systemNamespace); err != nil { + return err + } + + // install cockroachdb cluster instance + if err := h.CreateCockroachDBCluster(namespace, count); err != nil { + return err + } + + return nil +} + +func (h *CockroachDBInstaller) CreateCockroachDBOperator(namespace string) error { + logger.Infof("starting cockroachDB operator") + + logger.Info("creating cockroachDB CRDs") + if _, err := h.k8shelper.KubectlWithStdin(h.manifests.GetCockroachDBCRDs(), createFromStdinArgs...); err != nil { + return err + } + + cockroachDBOperator := h.manifests.GetCockroachDBOperator(namespace) + _, err := h.k8shelper.KubectlWithStdin(cockroachDBOperator, createFromStdinArgs...) + if err != nil { + return fmt.Errorf("failed to create rook-cockroachdb-operator pod: %+v ", err) + } + + if !h.k8shelper.IsCRDPresent(cockroachDBCRD) { + return fmt.Errorf("failed to find cockroachdb CRD %s", cockroachDBCRD) + } + + if !h.k8shelper.IsPodInExpectedState("rook-cockroachdb-operator", namespace, "Running") { + return fmt.Errorf("rook-cockroachdb-operator is not running, aborting") + } + + logger.Infof("cockroachdb operator started") + return nil +} + +func (h *CockroachDBInstaller) CreateCockroachDBCluster(namespace string, count int) error { + if err := h.k8shelper.CreateNamespace(namespace); err != nil { + return err + } + + logger.Infof("starting cockroachdb cluster with kubectl and yaml") + cockroachDBCluster := h.manifests.GetCockroachDBCluster(namespace, count) + if _, err := h.k8shelper.KubectlWithStdin(cockroachDBCluster, createFromStdinArgs...); err != nil { + return fmt.Errorf("Failed to create cockroachdb cluster: %+v ", err) + } + + if err := h.k8shelper.WaitForPodCount("app=rook-cockroachdb", namespace, count); err != nil { + logger.Errorf("cockroachdb cluster pods in namespace %s not found", namespace) + return err + } + + err := h.k8shelper.WaitForLabeledPodToRun("app=rook-cockroachdb", namespace) + if err != nil { + logger.Errorf("cockroachdb cluster pods in namespace %s are not running", namespace) + return err + } + + logger.Infof("cockroachdb cluster started") + return nil +} + +func (h *CockroachDBInstaller) UninstallCockroachDB(systemNamespace, namespace string) { + logger.Infof("uninstalling cockroachdb from namespace %s", namespace) + + _, err := h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cluster.cockroachdb.rook.io", namespace) + checkError(h.T(), err, fmt.Sprintf("cannot remove cluster %s", namespace)) + + crdCheckerFunc := func() error { + _, err := h.k8shelper.RookClientset.CockroachdbV1alpha1().Clusters(namespace).Get(namespace, metav1.GetOptions{}) + return err + } + err = h.k8shelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc) + checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace)) + + _, err = h.k8shelper.DeleteResourceAndWait(false, "namespace", namespace) + checkError(h.T(), err, fmt.Sprintf("cannot delete namespace %s", namespace)) + + logger.Infof("removing the operator from namespace %s", systemNamespace) + _, err = h.k8shelper.DeleteResource("crd", "clusters.cockroachdb.rook.io") + checkError(h.T(), err, "cannot delete CRDs") + + cockroachDBOperator := h.manifests.GetCockroachDBOperator(systemNamespace) + _, err = h.k8shelper.KubectlWithStdin(cockroachDBOperator, deleteFromStdinArgs...) + checkError(h.T(), err, "cannot uninstall rook-cockroachdb-operator") + + err = UninstallHostPathProvisioner(h.k8shelper) + checkError(h.T(), err, "cannot uninstall hostpath provisioner") + + h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil) + logger.Infof("done removing the operator from namespace %s", systemNamespace) +} + +func (h *CockroachDBInstaller) GatherAllCockroachDBLogs(systemNamespace, namespace, testName string) { + logger.Infof("Gathering all logs from cockroachdb cluster %s", namespace) + h.k8shelper.GetRookLogs("rook-cockroachdb-operator", Env.HostType, systemNamespace, testName) + h.k8shelper.GetRookLogs("rook-cockroachdb", Env.HostType, namespace, testName) +} diff --git a/tests/framework/installer/cockroachdb_manifests.go b/tests/framework/installer/cockroachdb_manifests.go new file mode 100644 index 000000000..e8327af36 --- /dev/null +++ b/tests/framework/installer/cockroachdb_manifests.go @@ -0,0 +1,153 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installer + +import ( + "strconv" +) + +type CockroachDBManifests struct { +} + +func (i *CockroachDBManifests) GetCockroachDBCRDs() string { + return `apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusters.cockroachdb.rook.io +spec: + group: cockroachdb.rook.io + names: + kind: Cluster + listKind: ClusterList + plural: clusters + singular: cluster + scope: Namespaced + version: v1alpha1 +` +} + +func (i *CockroachDBManifests) GetCockroachDBOperator(namespace string) string { + + return `kind: Namespace +apiVersion: v1 +metadata: + name: ` + namespace + ` +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-cockroachdb-operator +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - create + - update +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - create +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - create +- apiGroups: + - cockroachdb.rook.io + resources: + - "*" + verbs: + - "*" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-cockroachdb-operator + namespace: ` + namespace + ` +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-cockroachdb-operator + namespace: ` + namespace + ` +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-cockroachdb-operator +subjects: +- kind: ServiceAccount + name: rook-cockroachdb-operator + namespace: ` + namespace + ` +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: rook-cockroachdb-operator + namespace: ` + namespace + ` +spec: + replicas: 1 + template: + metadata: + labels: + app: rook-cockroachdb-operator + spec: + serviceAccountName: rook-cockroachdb-operator + containers: + - name: rook-cockroachdb-operator + image: rook/cockroachdb:master + args: ["cockroachdb", "operator"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +` +} + +func (i *CockroachDBManifests) GetCockroachDBCluster(namespace string, count int) string { + return `apiVersion: cockroachdb.rook.io/v1alpha1 +kind: Cluster +metadata: + name: ` + namespace + ` + namespace: ` + namespace + ` +spec: + scope: + nodeCount: ` + strconv.Itoa(count) + ` + secure: false + volumeSize: 1Gi + cachePercent: 25 + maxSQLMemoryPercent: 25 +` +} diff --git a/tests/framework/installer/device.go b/tests/framework/installer/device.go new file mode 100644 index 000000000..bf3e510cf --- /dev/null +++ b/tests/framework/installer/device.go @@ -0,0 +1,68 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installer + +import ( + "strings" + + "github.com/rook/rook/pkg/util/exec" + "github.com/rook/rook/pkg/util/sys" +) + +func IsAdditionalDeviceAvailableOnCluster() bool { + executor := &exec.CommandExecutor{} + devices, err := sys.ListDevices(executor) + if err != nil { + return false + } + disks := 0 + logger.Infof("devices : %v", devices) + for _, device := range devices { + if strings.Contains(device, "loop") { + continue + } + props, _ := sys.GetDeviceProperties(device, executor) + if props["TYPE"] != "disk" { + continue + } + + ownPartitions, fs, err := sys.CheckIfDeviceAvailable(executor, device) + if err != nil { + logger.Warningf("failed to detect device %s availability. %+v", device, err) + continue + } + if !ownPartitions { + logger.Infof("skipping device %s since don't own partitions", device) + continue + } + if fs != "" { + logger.Infof("skipping device %s since it has file system %s", device, fs) + continue + } + if strings.HasPrefix(device, "rbd") { + logger.Infof("skipping unexpected rbd device: %s", device) + continue + } + logger.Infof("available device: %s", device) + disks++ + } + if disks > 0 { + return true + } + logger.Info("No additional disks found on cluster") + return false +} diff --git a/tests/framework/objects/environment_manifest.go b/tests/framework/installer/environment.go similarity index 97% rename from tests/framework/objects/environment_manifest.go rename to tests/framework/installer/environment.go index c4a983cbf..63f76a3d6 100644 --- a/tests/framework/objects/environment_manifest.go +++ b/tests/framework/installer/environment.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package objects +package installer import ( "flag" @@ -37,7 +37,7 @@ var Env EnvironmentManifest func init() { Env = EnvironmentManifest{} - flag.StringVar(&Env.HostType, "host_type", "localhost", "Host were tests are run eg - localhost,GCE or AWS") + flag.StringVar(&Env.HostType, "host_type", "localhost", "Host where tests are run eg - localhost,GCE or AWS") flag.StringVar(&Env.RookImageName, "rook_image", "rook/ceph", "Docker image name for the rook container to install, must be in docker hub or local environment") flag.StringVar(&Env.ToolboxImageName, "toolbox_image", "rook/ceph-toolbox", "Docker image name of the toolbox container to install, must be in docker hub or local environment") flag.BoolVar(&Env.SkipInstallRook, "skip_install_rook", false, "Indicate if Rook need to installed - false if tests are being running at Rook that is pre-installed") diff --git a/tests/framework/installer/installer.go b/tests/framework/installer/installer.go new file mode 100644 index 000000000..4cea903e7 --- /dev/null +++ b/tests/framework/installer/installer.go @@ -0,0 +1,90 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installer + +import ( + "fmt" + "os" + "runtime" + "testing" + + "github.com/coreos/pkg/capnslog" + "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/api/errors" +) + +const ( + // Version tag for the latest manifests + VersionMaster = "master" +) + +var ( + // ** Variables that might need to be changed depending on the dev environment. The init function below will modify some of them automatically. ** + baseTestDir string + forceUseDevices = false + createBaseTestDir = true + // ** end of Variables to modify + logger = capnslog.NewPackageLogger("github.com/rook/rook", "installer") + createArgs = []string{"create", "-f"} + createFromStdinArgs = append(createArgs, "-") + deleteArgs = []string{"delete", "-f"} + deleteFromStdinArgs = append(deleteArgs, "-") +) + +type TestSuite interface { + Setup() + Teardown() +} + +func init() { + // this default will only work if running kubernetes on the local machine + baseTestDir, _ = os.Getwd() + + // The following settings could apply to any environment when the kube context is running on the host and the tests are running inside a + // VM such as minikube. This is a cheap test for this condition, we need to find a better way to automate these settings. + if runtime.GOOS == "darwin" { + createBaseTestDir = false + baseTestDir = "/data" + } +} + +func SystemNamespace(namespace string) string { + return fmt.Sprintf("%s-system", namespace) +} + +func checkError(t *testing.T, err error, message string) { + // During cleanup the resource might not be found because the test might have failed before the test was done and never created the resource + if err == nil || errors.IsNotFound(err) { + return + } + assert.NoError(t, err, "%s. %+v", message, err) +} + +func concatYaml(first, second string) string { + return first + ` +--- +` + second +} + +// GatherCRDObjectDebuggingInfo gathers all the descriptions for pods, pvs and pvcs +func GatherCRDObjectDebuggingInfo(k8shelper *utils.K8sHelper, namespace string) { + k8shelper.PrintPodDescribeForNamespace(namespace) + k8shelper.PrintPVs(true /*detailed*/) + k8shelper.PrintPVCs(namespace, true /*detailed*/) + k8shelper.PrintStorageClasses(true /*detailed*/) +} diff --git a/tests/framework/installer/nfs.go b/tests/framework/installer/nfs_installer.go similarity index 68% rename from tests/framework/installer/nfs.go rename to tests/framework/installer/nfs_installer.go index 1c3da00eb..d97df1a72 100644 --- a/tests/framework/installer/nfs.go +++ b/tests/framework/installer/nfs_installer.go @@ -18,12 +18,28 @@ package installer import ( "fmt" + "testing" + "github.com/rook/rook/tests/framework/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + nfsServerCRD = "nfsservers.nfs.rook.io" +) + +type NFSInstaller struct { + k8shelper *utils.K8sHelper + manifests *NFSManifests + T func() *testing.T +} + +func NewNFSInstaller(k8shelper *utils.K8sHelper, t func() *testing.T) *NFSInstaller { + return &NFSInstaller{k8shelper, &NFSManifests{}, t} +} + // InstallNFSServer installs NFS operator, NFS CRD instance and NFS volume -func (h *InstallHelper) InstallNFSServer(systemNamespace, namespace string, count int) error { +func (h *NFSInstaller) InstallNFSServer(systemNamespace, namespace string, count int) error { h.k8shelper.CreateAnonSystemClusterBinding() // install hostpath provisioner if there isn't already a default storage class @@ -32,7 +48,7 @@ func (h *InstallHelper) InstallNFSServer(systemNamespace, namespace string, coun if err != nil { return err } else if !defaultExists { - if err := h.InstallHostPathProvisioner(); err != nil { + if err := InstallHostPathProvisioner(h.k8shelper); err != nil { return err } storageClassName = "hostpath" @@ -59,15 +75,15 @@ func (h *InstallHelper) InstallNFSServer(systemNamespace, namespace string, coun } // CreateNFSServerOperator creates nfs server in the provided namespace -func (h *InstallHelper) CreateNFSServerOperator(namespace string) error { +func (h *NFSInstaller) CreateNFSServerOperator(namespace string) error { logger.Infof("starting nfsserver operator") logger.Info("creating nfsserver CRDs") - if _, err := h.k8shelper.KubectlWithStdin(h.installData.GetNFSServerCRDs(), createFromStdinArgs...); err != nil { + if _, err := h.k8shelper.KubectlWithStdin(h.manifests.GetNFSServerCRDs(), createFromStdinArgs...); err != nil { return err } - nfsOperator := h.installData.GetNFSServerOperator(namespace) + nfsOperator := h.manifests.GetNFSServerOperator(namespace) _, err := h.k8shelper.KubectlWithStdin(nfsOperator, createFromStdinArgs...) if err != nil { return fmt.Errorf("failed to create rook-nfs-operator pod: %+v ", err) @@ -86,13 +102,13 @@ func (h *InstallHelper) CreateNFSServerOperator(namespace string) error { } // CreateNFSServer creates the NFS Server CRD instance -func (h *InstallHelper) CreateNFSServer(namespace string, count int, storageClassName string) error { +func (h *NFSInstaller) CreateNFSServer(namespace string, count int, storageClassName string) error { if err := h.k8shelper.CreateNamespace(namespace); err != nil { return err } logger.Infof("starting nfs server with kubectl and yaml") - nfsServer := h.installData.GetNFSServer(namespace, count, storageClassName) + nfsServer := h.manifests.GetNFSServer(namespace, count, storageClassName) if _, err := h.k8shelper.KubectlWithStdin(nfsServer, createFromStdinArgs...); err != nil { return fmt.Errorf("Failed to create nfs server: %+v ", err) } @@ -113,15 +129,15 @@ func (h *InstallHelper) CreateNFSServer(namespace string, count int, storageClas } // CreateNFSServerVolume creates NFS export PV and PVC -func (h *InstallHelper) CreateNFSServerVolume(namespace string) error { +func (h *NFSInstaller) CreateNFSServerVolume(namespace string) error { logger.Info("creating volume from nfs server in namespace %s", namespace) clusterIP, err := h.GetNFSServerClusterIP(namespace) if err != nil { return err } - nfsServerPV := h.installData.GetNFSServerPV(namespace, clusterIP) - nfsServerPVC := h.installData.GetNFSServerPVC() + nfsServerPV := h.manifests.GetNFSServerPV(namespace, clusterIP) + nfsServerPVC := h.manifests.GetNFSServerPVC() logger.Info("creating nfs server pv") if _, err := h.k8shelper.KubectlWithStdin(nfsServerPV, createFromStdinArgs...); err != nil { @@ -137,52 +153,52 @@ func (h *InstallHelper) CreateNFSServerVolume(namespace string) error { } // UninstallNFSServer uninstalls the NFS Server from the given namespace -func (h *InstallHelper) UninstallNFSServer(systemNamespace, namespace string) { +func (h *NFSInstaller) UninstallNFSServer(systemNamespace, namespace string) { logger.Infof("uninstalling nfsserver from namespace %s", namespace) _, err := h.k8shelper.DeleteResource("pvc", "nfs-pv-claim") - h.checkError(err, fmt.Sprintf("cannot remove nfs pvc")) + checkError(h.T(), err, fmt.Sprintf("cannot remove nfs pvc")) _, err = h.k8shelper.DeleteResource("pv", "nfs-pv") - h.checkError(err, fmt.Sprintf("cannot remove nfs pv")) + checkError(h.T(), err, fmt.Sprintf("cannot remove nfs pv")) _, err = h.k8shelper.DeleteResource("-n", namespace, "nfsservers.nfs.rook.io", namespace) - h.checkError(err, fmt.Sprintf("cannot remove nfsserver %s", namespace)) + checkError(h.T(), err, fmt.Sprintf("cannot remove nfsserver %s", namespace)) crdCheckerFunc := func() error { _, err := h.k8shelper.RookClientset.NfsV1alpha1().NFSServers(namespace).Get(namespace, metav1.GetOptions{}) return err } - err = h.waitForCustomResourceDeletion(namespace, crdCheckerFunc) - h.checkError(err, fmt.Sprintf("failed to wait for crd %s deletion", namespace)) + err = h.k8shelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc) + checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace)) _, err = h.k8shelper.DeleteResource("namespace", namespace) - h.checkError(err, fmt.Sprintf("cannot delete namespace %s", namespace)) + checkError(h.T(), err, fmt.Sprintf("cannot delete namespace %s", namespace)) logger.Infof("removing the operator from namespace %s", systemNamespace) _, err = h.k8shelper.DeleteResource("crd", "nfsservers.nfs.rook.io") - h.checkError(err, "cannot delete CRDs") + checkError(h.T(), err, "cannot delete CRDs") - nfsOperator := h.installData.GetNFSServerOperator(systemNamespace) + nfsOperator := h.manifests.GetNFSServerOperator(systemNamespace) _, err = h.k8shelper.KubectlWithStdin(nfsOperator, deleteFromStdinArgs...) - h.checkError(err, "cannot uninstall rook-nfs-operator") + checkError(h.T(), err, "cannot uninstall rook-nfs-operator") - err = h.UninstallHostPathProvisioner() - h.checkError(err, "cannot uninstall hostpath provisioner") + err = UninstallHostPathProvisioner(h.k8shelper) + checkError(h.T(), err, "cannot uninstall hostpath provisioner") h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil) logger.Infof("done removing the operator from namespace %s", systemNamespace) } // GatherAllNFSServerLogs gathers all NFS Server logs -func (h *InstallHelper) GatherAllNFSServerLogs(systemNamespace, namespace, testName string) { +func (h *NFSInstaller) GatherAllNFSServerLogs(systemNamespace, namespace, testName string) { logger.Infof("Gathering all logs from NFSServer %s", namespace) - h.k8shelper.GetRookLogs("rook-nfs-operator", h.Env.HostType, systemNamespace, testName) - h.k8shelper.GetRookLogs("rook-nfs", h.Env.HostType, namespace, testName) + h.k8shelper.GetRookLogs("rook-nfs-operator", Env.HostType, systemNamespace, testName) + h.k8shelper.GetRookLogs("rook-nfs", Env.HostType, namespace, testName) } // GetNFSServerClusterIP gets the nfs server cluster ip on which it serves -func (h *InstallHelper) GetNFSServerClusterIP(namespace string) (string, error) { +func (h *NFSInstaller) GetNFSServerClusterIP(namespace string) (string, error) { clusterIP := "" service, err := h.k8shelper.GetService("rook-nfs", namespace) if err != nil { diff --git a/tests/framework/installer/nfs_install_data.go b/tests/framework/installer/nfs_manifests.go similarity index 90% rename from tests/framework/installer/nfs_install_data.go rename to tests/framework/installer/nfs_manifests.go index 7a6a981c8..9ceb54298 100644 --- a/tests/framework/installer/nfs_install_data.go +++ b/tests/framework/installer/nfs_manifests.go @@ -20,8 +20,11 @@ import ( "strconv" ) +type NFSManifests struct { +} + // GetNFSServerCRDs returns NFSServer CRD definition -func (i *InstallData) GetNFSServerCRDs() string { +func (i *NFSManifests) GetNFSServerCRDs() string { return `apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -39,7 +42,7 @@ spec: } // GetNFSServerOperator returns the NFSServer operator definition -func (i *InstallData) GetNFSServerOperator(namespace string) string { +func (i *NFSManifests) GetNFSServerOperator(namespace string) string { return `kind: Namespace apiVersion: v1 metadata: @@ -131,7 +134,7 @@ spec: } // GetNFSServerPV returns NFSServer PV definition -func (i *InstallData) GetNFSServerPV(namespace string, clusterIP string) string { +func (i *NFSManifests) GetNFSServerPV(namespace string, clusterIP string) string { return `apiVersion: v1 kind: PersistentVolume metadata: @@ -152,7 +155,7 @@ spec: } // GetNFSServerPVC returns NFSServer PVC definition -func (i *InstallData) GetNFSServerPVC() string { +func (i *NFSManifests) GetNFSServerPVC() string { return `apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -168,7 +171,7 @@ spec: } // GetNFSServer returns NFSServer CRD instance definition -func (i *InstallData) GetNFSServer(namespace string, count int, storageClassName string) string { +func (i *NFSManifests) GetNFSServer(namespace string, count int, storageClassName string) string { return `apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/tests/framework/installer/provisioners.go b/tests/framework/installer/provisioners.go index 0355384e5..e80848686 100644 --- a/tests/framework/installer/provisioners.go +++ b/tests/framework/installer/provisioners.go @@ -34,68 +34,68 @@ const ( // ************************************************************************************************ // HostPath provisioner functions // ************************************************************************************************ -func (h *InstallHelper) InstallHostPathProvisioner() error { +func InstallHostPathProvisioner(k8shelper *utils.K8sHelper) error { logger.Info("installing host path provisioner") rbacResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerRBAC) args := append(createArgs, rbacResourceURL) - out, err := h.k8shelper.Kubectl(args...) + out, err := k8shelper.Kubectl(args...) if err != nil && !errors.IsAlreadyExists(err) { return fmt.Errorf("failed to create hostpath provisioner RBAC: %+v. %s", err, out) } deploymentResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerDeployment) args = append(createArgs, deploymentResourceURL) - out, err = h.k8shelper.Kubectl(args...) + out, err = k8shelper.Kubectl(args...) if err != nil && !errors.IsAlreadyExists(err) { return fmt.Errorf("failed to create hostpath provisioner deployment: %+v. %s", err, out) } storageClassResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerStorageClass) args = append(createArgs, storageClassResourceURL) - out, err = h.k8shelper.Kubectl(args...) + out, err = k8shelper.Kubectl(args...) if err != nil && !errors.IsAlreadyExists(err) { return fmt.Errorf("failed to create hostpath provisioner StorageClass: %+v. %s", err, out) } - err = h.k8shelper.WaitForLabeledPodToRun("k8s-app=hostpath-provisioner", "kube-system") + err = k8shelper.WaitForLabeledPodToRun("k8s-app=hostpath-provisioner", "kube-system") if err != nil { logger.Errorf("hostpath provisioner pod is not running: %+v", err) - h.k8shelper.PrintPodDescribeForNamespace("kube-system") - h.k8shelper.PrintStorageClasses(true /*detailed*/) + k8shelper.PrintPodDescribeForNamespace("kube-system") + k8shelper.PrintStorageClasses(true /*detailed*/) return err } - err = h.k8shelper.IsStorageClassPresent(hostPathStorageClassName) + err = k8shelper.IsStorageClassPresent(hostPathStorageClassName) if err != nil { logger.Errorf("storageClass %s not found: %+v", hostPathStorageClassName, err) - h.k8shelper.PrintStorageClasses(true /*detailed*/) + k8shelper.PrintStorageClasses(true /*detailed*/) return err } return nil } -func (h *InstallHelper) UninstallHostPathProvisioner() error { +func UninstallHostPathProvisioner(k8shelper *utils.K8sHelper) error { logger.Info("uninstalling host path provisioner") storageClassResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerStorageClass) args := append(deleteArgs, storageClassResourceURL) - out, err := h.k8shelper.Kubectl(args...) + out, err := k8shelper.Kubectl(args...) if err != nil && !utils.IsKubectlErrorNotFound(out, err) { return fmt.Errorf("failed to delete hostpath provisioner StorageClass: %+v. %s", err, out) } deploymentResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerDeployment) args = append(deleteArgs, deploymentResourceURL) - out, err = h.k8shelper.Kubectl(args...) + out, err = k8shelper.Kubectl(args...) if err != nil && !utils.IsKubectlErrorNotFound(out, err) { return fmt.Errorf("failed to delete hostpath provisioner deployment: %+v. %s", err, out) } rbacResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerRBAC) args = append(deleteArgs, rbacResourceURL) - out, err = h.k8shelper.Kubectl(args...) + out, err = k8shelper.Kubectl(args...) if err != nil && !utils.IsKubectlErrorNotFound(out, err) { return fmt.Errorf("failed to delete hostpath provisioner RBAC: %+v. %s", err, out) } diff --git a/tests/framework/installer/test_data.go b/tests/framework/installer/test_data.go deleted file mode 100644 index f25b73e7c..000000000 --- a/tests/framework/installer/test_data.go +++ /dev/null @@ -1,65 +0,0 @@ -package installer - -import ( - "github.com/rook/rook/tests/framework/utils" -) - -func BlockResourceOperation(k8sh *utils.K8sHelper, yaml string, action string) (string, error) { - return k8sh.ResourceOperation(action, yaml) -} - -func GetBlockPoolDef(poolName string, namespace string, replicaSize string) string { - return `apiVersion: ceph.rook.io/v1beta1 -kind: Pool -metadata: - name: ` + poolName + ` - namespace: ` + namespace + ` -spec: - replicated: - size: ` + replicaSize -} - -func GetBlockStorageClassDef(poolName string, storageClassName string, namespace string, varClusterName bool) string { - namespaceParameter := "clusterNamespace" - if varClusterName { - namespaceParameter = "clusterName" - } - return `apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ` + storageClassName + ` -provisioner: ceph.rook.io/block -parameters: - pool: ` + poolName + ` - ` + namespaceParameter + `: ` + namespace -} - -func GetBlockPvcDef(claimName string, storageClassName string, accessModes string) string { - return `apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ` + claimName + ` - annotations: - volume.beta.kubernetes.io/storage-class: ` + storageClassName + ` -spec: - accessModes: - - ` + accessModes + ` - resources: - requests: - storage: 1M` -} - -func concatYaml(first, second string) string { - return first + ` ---- -` + second -} - -func GetBlockPoolStorageClassAndPvcDef(namespace string, poolName string, storageClassName string, blockName string, accessMode string) string { - return concatYaml(GetBlockPoolDef(poolName, namespace, "1"), - concatYaml(GetBlockStorageClassDef(poolName, storageClassName, namespace, false), GetBlockPvcDef(blockName, storageClassName, accessMode))) -} - -func GetBlockPoolStorageClass(namespace string, poolName string, storageClassName string) string { - return concatYaml(GetBlockPoolDef(poolName, namespace, "1"), GetBlockStorageClassDef(poolName, storageClassName, namespace, false)) -} diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index 2d39cc86a..048b3ec0a 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -290,11 +290,53 @@ func (k8sh *K8sHelper) ResourceOperation(action string, podDefinition string) (s return "", fmt.Errorf("Could Not create resource in args : %v -- %v", args, err) } -//DeleteResource performs a kubectl delete on give args +//DeletePod performs a kubectl delete pod on the given pod +func (k8sh *K8sHelper) DeletePod(namespace, name string) (string, error) { + args := append([]string{"--grace-period=0", "pod"}, name) + if namespace != "" { + args = append(args, []string{"-n", namespace}...) + } + return k8sh.DeleteResourceAndWait(true, args...) +} + +//DeletePods performs a kubectl delete pod on the given pods +func (k8sh *K8sHelper) DeletePods(pods ...string) (msg string, err error) { + for _, pod := range pods { + msg, err = k8sh.DeletePod("", pod) + } + return +} + +//DeleteResource performs a kubectl delete on the given args func (k8sh *K8sHelper) DeleteResource(args ...string) (string, error) { return k8sh.DeleteResourceAndWait(true, args...) } +//WaitForCustomResourceDeletion waits for the CRD deletion +func (k8sh *K8sHelper) WaitForCustomResourceDeletion(namespace string, checkerFunc func() error) error { + if !k8sh.VersionAtLeast("v1.8.0") { + // v1.7 has an intermittent issue with long delay to delete resources so we will skip waiting + return nil + } + + // wait for the operator to finalize and delete the CRD + for i := 0; i < 10; i++ { + err := checkerFunc() + if err == nil { + logger.Infof("custom resource %s still exists", namespace) + time.Sleep(2 * time.Second) + continue + } + if errors.IsNotFound(err) { + logger.Infof("custom resource %s deleted", namespace) + return nil + } + return err + } + logger.Errorf("gave up deleting custom resource %s", namespace) + return nil +} + // DeleteResource performs a kubectl delete on give args. // If wait is false, a flag will be passed to indicate the delete should return immediately func (k8sh *K8sHelper) DeleteResourceAndWait(wait bool, args ...string) (string, error) { diff --git a/tests/integration/base_block_test.go b/tests/integration/base_block_test.go index e9b199c9a..51c99b4fc 100644 --- a/tests/integration/base_block_test.go +++ b/tests/integration/base_block_test.go @@ -21,6 +21,7 @@ import ( "strconv" "time" + "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" @@ -33,18 +34,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var ( - blockMountPath = "/tmp/rook1" - blockPodName = "block-test" -) - // Smoke Test for Block Storage - Test check the following operations on Block Storage in order //Create,Mount,Write,Read,Unmount and Delete. func runBlockE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string) { + podName := "block-test" poolName := "replicapool" storageClassName := "rook-ceph-block" blockName := "block-pv-claim" - podName := "block-test" defer blockTestDataCleanUp(helper, k8sh, namespace, poolName, storageClassName, blockName, podName) logger.Infof("Block Storage End to End Integration Test - create, mount, write to, read from, and unmount") @@ -54,28 +50,22 @@ func runBlockE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite. initBlockImages, _ := helper.BlockClient.List(namespace) logger.Infof("step 1: Create block storage") - _, cbErr := installer.BlockResourceOperation(k8sh, installer.GetBlockPoolStorageClassAndPvcDef(namespace, poolName, storageClassName, blockName, "ReadWriteOnce"), "create") + _, cbErr := helper.PoolClient.CreateStorageClassAndPvc(namespace, poolName, storageClassName, blockName, "ReadWriteOnce") require.Nil(s.T(), cbErr) require.True(s.T(), retryBlockImageCountCheck(helper, len(initBlockImages), 1, namespace), "Make sure a new block is created") logger.Infof("Block Storage created successfully") require.True(s.T(), k8sh.WaitUntilPVCIsBound(defaultNamespace, blockName), "Make sure PVC is Bound") logger.Infof("step 2: Mount block storage") - _, mtErr := helper.BlockClient.BlockMap(getBlockPodDefintion(podName, blockName, false), blockMountPath) - require.Nil(s.T(), mtErr) - crdName, err := k8sh.GetVolumeResourceName(defaultNamespace, blockName) - require.Nil(s.T(), err) - require.True(s.T(), k8sh.IsVolumeResourcePresent(installer.SystemNamespace(namespace), crdName), fmt.Sprintf("make sure Volume %s is created", crdName)) - require.True(s.T(), k8sh.IsPodRunning(blockPodName, defaultNamespace), "make sure block-test pod is in running state") - logger.Infof("Block Storage Mounted successfully") + crdName := createPodWithBlock(helper, k8sh, s, namespace, blockName, podName) logger.Infof("step 3: Write to block storage") - _, wtErr := helper.BlockClient.Write(blockPodName, blockMountPath, "Smoke Test Data form Block storage", "bsFile1", "") + _, wtErr := helper.BlockClient.Write(podName, "Smoke Test Data form Block storage", "bsFile1", "") require.Nil(s.T(), wtErr) logger.Infof("Write to Block storage successfully") logger.Infof("step 4: Read from block storage") - read, rErr := helper.BlockClient.Read(blockPodName, blockMountPath, "bsFile1", "") + read, rErr := helper.BlockClient.Read(podName, "bsFile1", "") require.Nil(s.T(), rErr) assert.Contains(s.T(), read, "Smoke Test Data form Block storage", "make sure content of the files is unchanged") logger.Infof("Read from Block storage successfully") @@ -84,38 +74,49 @@ func runBlockE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite. restartOSDPods(k8sh, s, namespace) logger.Infof("step 6: Read from block storage again") - read, rErr = helper.BlockClient.Read(blockPodName, blockMountPath, "bsFile1", "") + read, rErr = helper.BlockClient.Read(podName, "bsFile1", "") require.Nil(s.T(), rErr) assert.Contains(s.T(), read, "Smoke Test Data form Block storage", "make sure content of the files is unchanged") logger.Infof("Read from Block storage successfully") logger.Infof("step 7: Mount same block storage on a different pod. Should not be allowed") otherPod := "block-test2" - _, mtErr = helper.BlockClient.BlockMap(getBlockPodDefintion(otherPod, blockName, false), blockMountPath) + _, mtErr := helper.BlockClient.BlockMap(getBlockPodDefintion(otherPod, blockName, false)) require.Nil(s.T(), mtErr) require.True(s.T(), k8sh.IsPodInError(otherPod, defaultNamespace, "FailedMount", "Volume is already attached by pod"), "make sure block-test2 pod errors out while mounting the volume") logger.Infof("Block Storage successfully fenced") logger.Infof("step 8: Delete fenced pod") - _, unmtErr := helper.BlockClient.BlockUnmap(getBlockPodDefintion(otherPod, blockName, false), blockMountPath) + _, unmtErr := k8sh.DeletePod(k8sutil.DefaultNamespace, otherPod) require.Nil(s.T(), unmtErr) require.True(s.T(), k8sh.IsPodTerminated(otherPod, defaultNamespace), "make sure block-test2 pod is terminated") logger.Infof("Fenced pod deleted successfully") logger.Infof("step 9: Unmount block storage") - _, unmtErr = helper.BlockClient.BlockUnmap(getBlockPodDefintion(podName, blockName, false), blockMountPath) + _, unmtErr = k8sh.DeletePod(k8sutil.DefaultNamespace, podName) require.Nil(s.T(), unmtErr) require.True(s.T(), k8sh.IsVolumeResourceAbsent(installer.SystemNamespace(namespace), crdName), fmt.Sprintf("make sure Volume %s is deleted", crdName)) - require.True(s.T(), k8sh.IsPodTerminated(blockPodName, defaultNamespace), "make sure block-test pod is terminated") + require.True(s.T(), k8sh.IsPodTerminated(podName, defaultNamespace), "make sure block-test pod is terminated") logger.Infof("Block Storage unmounted successfully") logger.Infof("step 10: Deleting block storage") - _, dbErr := installer.BlockResourceOperation(k8sh, installer.GetBlockPoolStorageClassAndPvcDef(namespace, poolName, storageClassName, blockName, "ReadWriteOnce"), "delete") + dbErr := helper.PoolClient.DeleteStorageClassAndPvc(namespace, poolName, storageClassName, blockName, "ReadWriteOnce") require.Nil(s.T(), dbErr) require.True(s.T(), retryBlockImageCountCheck(helper, len(initBlockImages), 0, namespace), "Make sure a block is deleted") logger.Infof("Block Storage deleted successfully") } +func createPodWithBlock(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, blockName, podName string) string { + _, mtErr := helper.BlockClient.BlockMap(getBlockPodDefintion(podName, blockName, false)) + require.Nil(s.T(), mtErr) + crdName, err := k8sh.GetVolumeResourceName(defaultNamespace, blockName) + require.Nil(s.T(), err) + require.True(s.T(), k8sh.IsVolumeResourcePresent(installer.SystemNamespace(namespace), crdName), fmt.Sprintf("make sure Volume %s is created", crdName)) + require.True(s.T(), k8sh.IsPodRunning(podName, defaultNamespace), "make sure block-test pod is in running state") + logger.Infof("Block Storage Mounted successfully") + return crdName +} + func restartOSDPods(k8sh *utils.K8sHelper, s suite.Suite, namespace string) { osdLabel := "app=rook-ceph-osd" @@ -142,38 +143,51 @@ func runBlockE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s su logger.Infof("Block Storage End to End Integration Test - create storageclass,pool and pvc") logger.Infof("Running on Rook Cluster %s", clusterNamespace) poolName := "rookpool" + storageClassName := "rook-ceph-block-lite" + blockName := "test-block-claim-lite" + podName := "test-pod-lite" + defer blockTestDataCleanUp(helper, k8sh, clusterNamespace, poolName, storageClassName, blockName, podName) + initBlockCount := setupBlockLite(helper, k8sh, s, clusterNamespace, poolName, storageClassName, blockName, podName) + deleteBlockLite(helper, k8sh, s, clusterNamespace, poolName, storageClassName, blockName, podName, initBlockCount) +} + +func setupBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, + clusterNamespace, poolName, storageClassName, blockName, podName string) int { //Check initial number of blocks - defer blockTestDataCleanUp(helper, k8sh, clusterNamespace, poolName, "rook-ceph-block", "test-block", "block-test") - bc := helper.BlockClient - initialBlocks, err := bc.List(clusterNamespace) + initialBlocks, err := helper.BlockClient.List(clusterNamespace) require.Nil(s.T(), err) initBlockCount := len(initialBlocks) logger.Infof("step : Create Pool,StorageClass and PVC") - volumeDef := installer.GetBlockPoolStorageClassAndPvcDef(clusterNamespace, poolName, "rook-ceph-block", "test-block-claim", "ReadWriteOnce") - res1, err := installer.BlockResourceOperation(k8sh, volumeDef, "create") - checkOrderedSubstrings(s.T(), res1, poolName, "created", "rook-ceph-block", "created", "test-block-claim", "created") + res1, err := helper.PoolClient.CreateStorageClassAndPvc(clusterNamespace, poolName, storageClassName, blockName, "ReadWriteOnce") + checkOrderedSubstrings(s.T(), res1, poolName, "created", storageClassName, "created", blockName, "created") require.NoError(s.T(), err) - require.True(s.T(), k8sh.WaitUntilPVCIsBound(defaultNamespace, "test-block-claim")) + require.True(s.T(), k8sh.WaitUntilPVCIsBound(defaultNamespace, blockName)) - //Make sure new block is created - b, _ := bc.List(clusterNamespace) + //Make sure new block is created + b, _ := helper.BlockClient.List(clusterNamespace) assert.Equal(s.T(), initBlockCount+1, len(b), "Make sure new block image is created") poolExists, err := helper.PoolClient.CephPoolExists(clusterNamespace, poolName) assert.Nil(s.T(), err) assert.True(s.T(), poolExists) + return initBlockCount +} +func deleteBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, + clusterNamespace, poolName, storageClassName, blockName, podName string, initBlockCount int) { + + logger.Infof("deleteBlockLite: cleaning up after test") //Delete pvc and storageclass - _, err = installer.BlockResourceOperation(k8sh, volumeDef, "delete") + err := helper.PoolClient.DeleteStorageClassAndPvc(clusterNamespace, poolName, storageClassName, blockName, "ReadWriteOnce") assert.NoError(s.T(), err) - assert.True(s.T(), k8sh.WaitUntilPVCIsDeleted(defaultNamespace, "test-block-claim")) + assert.True(s.T(), k8sh.WaitUntilPVCIsDeleted(defaultNamespace, blockName)) require.True(s.T(), retryBlockImageCountCheck(helper, initBlockCount, 0, clusterNamespace), "Make sure a new block is deleted") - b, _ = bc.List(clusterNamespace) + b, _ := helper.BlockClient.List(clusterNamespace) assert.Equal(s.T(), initBlockCount, len(b), "Make sure new block image is deleted") checkPoolDeleted(helper, s, clusterNamespace, poolName) @@ -199,8 +213,8 @@ func checkPoolDeleted(helper *clients.TestClient, s suite.Suite, namespace, name func blockTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, poolname, storageclassname, blockname, podname string) { logger.Infof("Cleaning up block storage") - helper.BlockClient.BlockUnmap(getBlockPodDefintion(podname, blockname, false), blockMountPath) - installer.BlockResourceOperation(k8sh, installer.GetBlockPoolStorageClassAndPvcDef(namespace, poolname, storageclassname, blockname, "ReadWriteOnce"), "delete") + k8sh.DeletePod(k8sutil.DefaultNamespace, podname) + helper.PoolClient.DeleteStorageClassAndPvc(namespace, poolname, storageclassname, blockname, "ReadWriteOnce") cleanupDynamicBlockStorage(helper, namespace) } @@ -227,7 +241,6 @@ func cleanupDynamicBlockStorage(helper *clients.TestClient, namespace string) { for _, blockImage := range blockImagesList { helper.BlockClient.DeleteBlockImage(blockImage, namespace) } - } func getBlockPodDefintion(podname, blockName string, readOnly bool) string { @@ -245,7 +258,7 @@ spec: imagePullPolicy: IfNotPresent volumeMounts: - name: block-persistent-storage - mountPath: /tmp/rook1 + mountPath: ` + clients.TestMountPath + ` volumes: - name: block-persistent-storage persistentVolumeClaim: diff --git a/tests/integration/base_deploy_test.go b/tests/integration/base_deploy_test.go index 90d9a7ba7..908afd273 100644 --- a/tests/integration/base_deploy_test.go +++ b/tests/integration/base_deploy_test.go @@ -25,7 +25,6 @@ import ( "github.com/coreos/pkg/capnslog" cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" "github.com/rook/rook/tests/framework/clients" - "github.com/rook/rook/tests/framework/contracts" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/assert" @@ -33,10 +32,12 @@ import ( "github.com/stretchr/testify/suite" ) +const ( + defaultNamespace = "default" +) + var ( logger = capnslog.NewPackageLogger("github.com/rook/rook", "integrationTest") - - defaultNamespace = "default" ) //Test to make sure all rook components are installed and Running @@ -76,30 +77,18 @@ func checkIfRookClusterIsHealthy(s suite.Suite, testClient *clients.TestClient, require.Nil(s.T(), err) } -func HandlePanics(r interface{}, op contracts.Setup, t func() *testing.T) { +func HandlePanics(r interface{}, op installer.TestSuite, t func() *testing.T) { if r != nil { logger.Infof("unexpected panic occurred during test %s, --> %v", t().Name(), r) t().Fail() - op.TearDown() + op.Teardown() t().FailNow() } } -//GetTestClient sets up SetTestClient for rook -func GetTestClient(kh *utils.K8sHelper, namespace string, op contracts.Setup, t func() *testing.T) *clients.TestClient { - helper, err := clients.CreateTestClient(kh, namespace) - if err != nil { - logger.Errorf("Cannot create rook test client, er -> %v", err) - t().Fail() - op.TearDown() - t().FailNow() - } - return helper -} - -//BaseTestOperations struct for handling panic and test suite tear down -type BaseTestOperations struct { - installer *installer.InstallHelper +//TestCluster struct for handling panic and test suite tear down +type TestCluster struct { + installer *installer.CephInstaller kh *utils.K8sHelper helper *clients.TestClient T func() *testing.T @@ -110,20 +99,20 @@ type BaseTestOperations struct { mons int } -// StartBaseTestOperations creates new instance of BaseTestOperations struct -func StartBaseTestOperations(t func() *testing.T, namespace, storeType string, helmInstalled, useDevices bool, mons int) (BaseTestOperations, *utils.K8sHelper) { +// StartTestCluster creates new instance of TestCluster struct +func StartTestCluster(t func() *testing.T, namespace, storeType string, helmInstalled, useDevices bool, mons int, rookVersion string) (*TestCluster, *utils.K8sHelper) { kh, err := utils.CreateK8sHelper(t) require.NoError(t(), err) - i := installer.NewK8sRookhelper(kh.Clientset, t) + i := installer.NewCephInstaller(t, kh.Clientset, rookVersion) - op := BaseTestOperations{i, kh, nil, t, namespace, storeType, helmInstalled, useDevices, mons} - op.SetUp() + op := &TestCluster{i, kh, nil, t, namespace, storeType, helmInstalled, useDevices, mons} + op.Setup() return op, kh } //SetUpRook is a wrapper for setting up rook -func (op BaseTestOperations) SetUp() { +func (op *TestCluster) Setup() { isRookInstalled, err := op.installer.InstallRookOnK8sWithHostPathAndDevices(op.namespace, op.storeType, op.helmInstalled, op.useDevices, cephv1beta1.MonSpec{Count: op.mons, AllowMultiplePerNode: true}, false /* startWithAllNodes */) @@ -133,13 +122,18 @@ func (op BaseTestOperations) SetUp() { op.installer.GatherAllRookLogs(op.namespace, installer.SystemNamespace(op.namespace), op.installer.T().Name()) } op.T().Fail() - op.TearDown() + op.Teardown() op.T().FailNow() } } +// SetInstallData updates the installer helper based on the version of Rook desired +func (op *TestCluster) SetInstallData(version string) { + +} + //TearDownRook is a wrapper for tearDown after Suite -func (op BaseTestOperations) TearDown() { +func (op *TestCluster) Teardown() { if op.installer.T().Failed() { op.installer.GatherAllRookLogs(op.namespace, installer.SystemNamespace(op.namespace), op.installer.T().Name()) } diff --git a/tests/integration/base_file_test.go b/tests/integration/base_file_test.go index fba30687a..edea2d819 100644 --- a/tests/integration/base_file_test.go +++ b/tests/integration/base_file_test.go @@ -23,13 +23,12 @@ import ( "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "k8s.io/kubernetes/pkg/util/version" ) -var ( +const ( fileMountPath = "/tmp/rookfs" filePodName = "file-test" ) @@ -37,21 +36,18 @@ var ( // Smoke Test for File System Storage - Test check the following operations on Filesystem Storage in order //Create,Mount,Write,Read,Unmount and Delete. func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) { - defer fileTestDataCleanUp(helper, k8sh, s, filePodName, namespace, filesystemName, fileMountPath) + defer fileTestDataCleanUp(helper, k8sh, s, filePodName, namespace, filesystemName) logger.Infof("Running on Rook Cluster %s", namespace) logger.Infof("File Storage End To End Integration Test - create, mount, write to, read from, and unmount") - logger.Infof("Step 1: Create file System") - fscErr := helper.FSClient.Create(filesystemName, namespace) - require.Nil(s.T(), fscErr) - filesystemList, _ := helper.FSClient.List(namespace) - require.Equal(s.T(), 1, len(filesystemList), "There should one shared file system present") - filesystemData := filesystemList[0] - require.Equal(s.T(), filesystemName, filesystemData.Name, "make sure filesystem name matches") - logger.Infof("File system created") + createFilesystem(helper, k8sh, s, namespace, filesystemName) + createFilesystemConsumerPod(helper, k8sh, s, namespace, filesystemName) + writeAndReadToFilesystem(helper, k8sh, s, namespace, "test_file") + cleanupFilesystemConsumer(helper, k8sh, s, namespace, filesystemName) +} - logger.Infof("Step 2: Mount file System") - mtfsErr := podWithFilesystem(k8sh, s, filePodName, namespace, filesystemName, fileMountPath, "create") +func createFilesystemConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) { + mtfsErr := podWithFilesystem(k8sh, s, filePodName, namespace, filesystemName, "create") require.Nil(s.T(), mtfsErr) filePodRunning := k8sh.IsPodRunning(filePodName, namespace) if !filePodRunning { @@ -61,25 +57,31 @@ func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S } require.True(s.T(), filePodRunning, "make sure file-test pod is in running state") logger.Infof("File system mounted successfully") +} - logger.Infof("Step 3: Write to file system") - _, wfsErr := helper.FSClient.Write(filePodName, fileMountPath, "Smoke Test Data for file system storage", "fsFile1", namespace) +func writeAndReadToFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filename string) { + + logger.Infof("Write to file system") + message := "Test Data for file system storage" + _, wfsErr := helper.FSClient.Write(filePodName, message, filename, namespace) require.Nil(s.T(), wfsErr) logger.Infof("Write to file system successful") - logger.Infof("Step 4: Read from file system") - read, rdErr := helper.FSClient.Read(filePodName, fileMountPath, "fsFile1", namespace) + logger.Infof("Read from file system") + read, rdErr := helper.FSClient.Read(filePodName, filename, namespace) require.Nil(s.T(), rdErr) - assert.Contains(s.T(), read, "Smoke Test Data for file system storage", "make sure content of the files is unchanged") + require.Contains(s.T(), read, message) logger.Infof("Read from file system successful") +} - logger.Infof("Step 5: UnMount file System") - umtfsErr := podWithFilesystem(k8sh, s, filePodName, namespace, filesystemName, fileMountPath, "delete") - require.Nil(s.T(), umtfsErr) +func cleanupFilesystemConsumer(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) { + logger.Infof("Unmount file System") + _, err := k8sh.DeleteResource("pod", filePodName, "-n", namespace) + require.Nil(s.T(), err) require.True(s.T(), k8sh.IsPodTerminated(filePodName, namespace), "make sure file-test pod is terminated") - logger.Infof("File system mounted successfully") + logger.Infof("File system unmounted successfully") - logger.Infof("Step 6: Deleting file storage") + logger.Infof("Deleting file storage") helper.FSClient.Delete(filesystemName, namespace) //Delete is not deleting filesystem - known issue //require.Nil(suite.T(), fsd_err) @@ -90,22 +92,26 @@ func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S func runFileE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) { logger.Infof("File Storage End to End Integration Test - create Filesystem and make sure mds pod is running") logger.Infof("Running on Rook Cluster %s", namespace) - - logger.Infof("Step 1: Create file System") - fscErr := helper.FSClient.Create(filesystemName, namespace) - require.Nil(s.T(), fscErr) - filesystemList, _ := helper.FSClient.List(namespace) - logger.Infof("Retrieved file systems: %+v", filesystemList) - require.Equal(s.T(), 1, len(filesystemList), "There should one shared file system present") + createFilesystem(helper, k8sh, s, namespace, filesystemName) } -func fileTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string, fileMountPath string) { +func createFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName string) { + logger.Infof("Create file System") + fscErr := helper.FSClient.Create(filesystemName, namespace) + require.Nil(s.T(), fscErr) + logger.Infof("File system %s created", filesystemName) + + filesystemList, _ := helper.FSClient.List(namespace) + require.Equal(s.T(), 1, len(filesystemList), "There should be one shared file system present") +} + +func fileTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string) { logger.Infof("Cleaning up file system") - podWithFilesystem(k8sh, s, podname, namespace, filesystemName, fileMountPath, "delete") + podWithFilesystem(k8sh, s, podname, namespace, filesystemName, "delete") helper.FSClient.Delete(filesystemName, namespace) } -func podWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string, fileMountPath string, action string) error { +func podWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string, action string) error { driverName := installer.SystemNamespace(namespace) v := version.MustParseSemantic(k8sh.GetK8sServerVersion()) if v.LessThan(version.MustParseSemantic("1.10.0")) { @@ -113,7 +119,7 @@ func podWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, nam driverName = flexvolume.FlexDriverName } - testPod := getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath, driverName) + testPod := getFilesystemTestPod(podname, namespace, filesystemName, driverName) logger.Infof("creating test pod: %s", testPod) _, err := k8sh.ResourceOperation(action, testPod) if err != nil { @@ -122,7 +128,7 @@ func podWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, nam return nil } -func getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath, driverName string) string { +func getFilesystemTestPod(podname, namespace, filesystemName, driverName string) string { return `apiVersion: v1 kind: Pod metadata: @@ -138,7 +144,7 @@ spec: imagePullPolicy: IfNotPresent env: volumeMounts: - - mountPath: "` + fileMountPath + `" + - mountPath: "` + clients.TestMountPath + `" name: ` + filesystemName + ` volumes: - name: ` + filesystemName + ` diff --git a/tests/integration/block_createBlock_k8s_test.go b/tests/integration/block_create_test.go similarity index 81% rename from tests/integration/block_createBlock_k8s_test.go rename to tests/integration/block_create_test.go index e35ab1281..c9bac3aa1 100644 --- a/tests/integration/block_createBlock_k8s_test.go +++ b/tests/integration/block_create_test.go @@ -20,9 +20,7 @@ import ( "fmt" "testing" - "github.com/rook/rook/pkg/daemon/ceph/model" "github.com/rook/rook/tests/framework/clients" - "github.com/rook/rook/tests/framework/contracts" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/assert" @@ -48,16 +46,16 @@ type BlockCreateSuite struct { kh *utils.K8sHelper initBlockCount int namespace string - installer *installer.InstallHelper - op contracts.Setup + installer *installer.CephInstaller + op *TestCluster } func (s *BlockCreateSuite) SetupSuite() { var err error s.namespace = "block-k8s-ns" - s.op, s.kh = StartBaseTestOperations(s.T, s.namespace, "bluestore", false, false, 1) - s.testClient = GetTestClient(s.kh, s.namespace, s.op, s.T) + s.op, s.kh = StartTestCluster(s.T, s.namespace, "bluestore", false, false, 1, installer.VersionMaster) + s.testClient = clients.CreateTestClient(s.kh, s.op.installer.Manifests) initialBlocks, err := s.testClient.BlockClient.List(s.namespace) assert.Nil(s.T(), err) s.initBlockCount = len(initialBlocks) @@ -73,7 +71,7 @@ func (s *BlockCreateSuite) TestCreatePVCWhenNoStorageClassExists() { storageClassName := "rook-ceph-block" defer s.tearDownTest(claimName, poolName, storageClassName, "ReadWriteOnce") - result, err := installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(claimName, storageClassName, "ReadWriteOnce"), "create") + result, err := s.testClient.BlockClient.CreatePvc(claimName, storageClassName, "ReadWriteOnce") checkOrderedSubstrings(s.T(), result, "persistentvolumeclaim", claimName, "created") require.NoError(s.T(), err) @@ -105,12 +103,11 @@ func (s *BlockCreateSuite) TestCreateSamePVCTwice() { s.testClient.BlockClient.List(s.namespace) logger.Infof("create pool and storageclass") - pool := model.Pool{Name: poolName, ReplicatedConfig: model.ReplicatedPoolConfig{Size: 1}} - result0, err0 := s.testClient.PoolClient.Create(pool, s.namespace) + result0, err0 := s.testClient.PoolClient.Create(poolName, s.namespace, 1) checkOrderedSubstrings(s.T(), result0, poolName, "created") require.NoError(s.T(), err0) - result1, err1 := installer.BlockResourceOperation(s.kh, installer.GetBlockStorageClassDef(poolName, storageClassName, s.namespace, true), "create") + result1, err1 := s.testClient.BlockClient.CreateStorageClass(poolName, storageClassName, s.namespace, true) checkOrderedSubstrings(s.T(), result1, storageClassName, "created") require.NoError(s.T(), err1) @@ -119,7 +116,7 @@ func (s *BlockCreateSuite) TestCreateSamePVCTwice() { require.Nil(s.T(), err) logger.Infof("create pvc") - result2, err2 := installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(claimName, storageClassName, "ReadWriteOnce"), "create") + result2, err2 := s.testClient.BlockClient.CreatePvc(claimName, storageClassName, "ReadWriteOnce") checkOrderedSubstrings(s.T(), result2, claimName, "created") require.NoError(s.T(), err2) @@ -131,7 +128,7 @@ func (s *BlockCreateSuite) TestCreateSamePVCTwice() { assert.Equal(s.T(), s.initBlockCount+1, len(b1), "Make sure new block image is created") logger.Infof("Create same pvc again") - _, err3 := installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(claimName, storageClassName, "ReadWriteOnce"), "create") + _, err3 := s.testClient.BlockClient.CreatePvc(claimName, storageClassName, "ReadWriteOnce") checkOrderedSubstrings(s.T(), err3.Error(), claimName, "already exists") logger.Infof("check status of PVC") @@ -152,7 +149,8 @@ func (s *BlockCreateSuite) TestBlockStorageMountUnMountForStatefulSets() { defer s.statefulSetDataCleanup(defaultNamespace, poolName, storageClassName, statefulSetName, statefulPodsName) logger.Infof("Test case when block persistent volumes are scaled up and down along with StatefulSet") logger.Info("Step 1: Create pool and storageClass") - _, cbErr := installer.BlockResourceOperation(s.kh, installer.GetBlockPoolStorageClass(s.namespace, poolName, storageClassName), "create") + + _, cbErr := s.testClient.PoolClient.CreateStorageClass(s.namespace, poolName, storageClassName) assert.Nil(s.T(), cbErr) logger.Info("Step 2 : Deploy statefulSet with 1X replication") service, statefulset := getBlockStatefulSetAndServiceDefinition(defaultNamespace, statefulSetName, statefulPodsName, storageClassName) @@ -198,19 +196,17 @@ func (s *BlockCreateSuite) statefulSetDataCleanup(namespace, poolName, storageCl //Delete all PVCs s.kh.DeletePvcWithLabel(defaultNamespace, statefulSetName) //Delete storageclass and pool - installer.BlockResourceOperation(s.kh, installer.GetBlockPoolStorageClass(s.namespace, poolName, storageClassName), "delete") - + s.testClient.PoolClient.DeleteStorageClass(s.namespace, poolName, storageClassName) } func (s *BlockCreateSuite) tearDownTest(claimName string, poolName string, storageClassName string, accessMode string) { - installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(claimName, storageClassName, accessMode), "delete") - installer.BlockResourceOperation(s.kh, installer.GetBlockPoolDef(poolName, s.namespace, "1"), "delete") - installer.BlockResourceOperation(s.kh, installer.GetBlockStorageClassDef(poolName, storageClassName, s.namespace, true), "delete") - + s.testClient.BlockClient.DeletePvc(claimName, storageClassName, accessMode) + s.testClient.PoolClient.Delete(poolName, s.namespace) + s.testClient.BlockClient.DeleteStorageClass(poolName, storageClassName, s.namespace) } func (s *BlockCreateSuite) TearDownSuite() { - s.op.TearDown() + s.op.Teardown() } func (s *BlockCreateSuite) CheckCreatingPVC(pvcName, pvcAccessMode string) { @@ -221,10 +217,10 @@ func (s *BlockCreateSuite) CheckCreatingPVC(pvcName, pvcAccessMode string) { defer s.tearDownTest(claimName, poolName, storageClassName, pvcAccessMode) //create pool and storageclass - result0, err0 := installer.BlockResourceOperation(s.kh, installer.GetBlockPoolDef(poolName, s.namespace, "1"), "create") + result0, err0 := s.testClient.PoolClient.Create(poolName, s.namespace, 1) checkOrderedSubstrings(s.T(), result0, poolName, "created") require.NoError(s.T(), err0) - result1, err1 := installer.BlockResourceOperation(s.kh, installer.GetBlockStorageClassDef(poolName, storageClassName, s.namespace, true), "create") + result1, err1 := s.testClient.BlockClient.CreateStorageClass(poolName, storageClassName, s.namespace, true) checkOrderedSubstrings(s.T(), result1, storageClassName, "created") require.NoError(s.T(), err1) @@ -233,7 +229,7 @@ func (s *BlockCreateSuite) CheckCreatingPVC(pvcName, pvcAccessMode string) { require.Nil(s.T(), err) //create pvc - result2, err2 := installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(claimName, storageClassName, pvcAccessMode), "create") + result2, err2 := s.testClient.BlockClient.CreatePvc(claimName, storageClassName, pvcAccessMode) checkOrderedSubstrings(s.T(), result2, claimName, "created") require.NoError(s.T(), err2) diff --git a/tests/integration/cockroachdb_test.go b/tests/integration/cockroachdb_test.go index b2f8e2800..6097cfc06 100644 --- a/tests/integration/cockroachdb_test.go +++ b/tests/integration/cockroachdb_test.go @@ -49,66 +49,66 @@ func TestCockroachDBSuite(t *testing.T) { type CockroachDBSuite struct { suite.Suite - k8sHelper *utils.K8sHelper - installHelper *installer.InstallHelper + k8shelper *utils.K8sHelper + installer *installer.CockroachDBInstaller namespace string systemNamespace string instanceCount int } func (suite *CockroachDBSuite) SetupSuite() { - suite.SetUp() + suite.Setup() } func (suite *CockroachDBSuite) TearDownSuite() { - suite.TearDown() + suite.Teardown() } -func (suite *CockroachDBSuite) SetUp() { +func (suite *CockroachDBSuite) Setup() { suite.namespace = "cockroachdb-ns" suite.systemNamespace = installer.SystemNamespace(suite.namespace) suite.instanceCount = 1 - k8sHelper, err := utils.CreateK8sHelper(suite.T) + k8shelper, err := utils.CreateK8sHelper(suite.T) require.NoError(suite.T(), err) - suite.k8sHelper = k8sHelper + suite.k8shelper = k8shelper - k8sversion := suite.k8sHelper.GetK8sServerVersion() + k8sversion := suite.k8shelper.GetK8sServerVersion() logger.Infof("Installing cockroachdb on k8s %s", k8sversion) - suite.installHelper = installer.NewK8sRookhelper(suite.k8sHelper.Clientset, suite.T) + suite.installer = installer.NewCockroachDBInstaller(suite.k8shelper, suite.T) - err = suite.installHelper.InstallCockroachDB(suite.systemNamespace, suite.namespace, suite.instanceCount) + err = suite.installer.InstallCockroachDB(suite.systemNamespace, suite.namespace, suite.instanceCount) if err != nil { logger.Errorf("cockroachdb was not installed successfully: %+v", err) suite.T().Fail() - suite.TearDown() + suite.Teardown() suite.T().FailNow() } } -func (suite *CockroachDBSuite) TearDown() { +func (suite *CockroachDBSuite) Teardown() { if suite.T().Failed() { - suite.installHelper.GatherCRDObjectDebuggingInfo(suite.systemNamespace) - suite.installHelper.GatherCRDObjectDebuggingInfo(suite.namespace) - suite.installHelper.GatherAllCockroachDBLogs(suite.systemNamespace, suite.namespace, suite.T().Name()) + installer.GatherCRDObjectDebuggingInfo(suite.k8shelper, suite.systemNamespace) + installer.GatherCRDObjectDebuggingInfo(suite.k8shelper, suite.namespace) + suite.installer.GatherAllCockroachDBLogs(suite.systemNamespace, suite.namespace, suite.T().Name()) } - suite.installHelper.UninstallCockroachDB(suite.systemNamespace, suite.namespace) + suite.installer.UninstallCockroachDB(suite.systemNamespace, suite.namespace) } func (suite *CockroachDBSuite) TestCockroachDBClusterInstallation() { logger.Infof("Verifying that all expected pods in cockroachdb cluster %s are running", suite.namespace) // verify cockroachdb operator is running OK - assert.True(suite.T(), suite.k8sHelper.CheckPodCountAndState("rook-cockroachdb-operator", suite.systemNamespace, 1, "Running"), + assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-cockroachdb-operator", suite.systemNamespace, 1, "Running"), "1 rook-cockroachdb-operator must be in Running state") // verify cockroachdb cluster instances are running OK - assert.True(suite.T(), suite.k8sHelper.CheckPodCountAndState("rook-cockroachdb", suite.namespace, suite.instanceCount, "Running"), + assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-cockroachdb", suite.namespace, suite.instanceCount, "Running"), fmt.Sprintf("%d rook-cockroachdb pods must be in Running state", suite.instanceCount)) // determine the cockroachdb operator pod name - podNames, err := suite.k8sHelper.GetPodNamesForApp("rook-cockroachdb-operator", suite.systemNamespace) + podNames, err := suite.k8shelper.GetPodNamesForApp("rook-cockroachdb-operator", suite.systemNamespace) require.NoError(suite.T(), err) require.Equal(suite.T(), 1, len(podNames)) operatorPodName := podNames[0] @@ -123,7 +123,7 @@ func (suite *CockroachDBSuite) TestCockroachDBClusterInstallation() { inc := 0 var result string for inc < utils.RetryLoop { - result, err = suite.k8sHelper.Exec(suite.systemNamespace, operatorPodName, command, commandArgs) + result, err = suite.k8shelper.Exec(suite.systemNamespace, operatorPodName, command, commandArgs) logger.Infof("cockroachdb sql command exited, err: %+v. result: %s", err, result) if err == nil { break diff --git a/tests/integration/helm_test.go b/tests/integration/helm_test.go index ae30f857e..b7b9c4e62 100644 --- a/tests/integration/helm_test.go +++ b/tests/integration/helm_test.go @@ -20,7 +20,7 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/rook/rook/tests/framework/clients" - "github.com/rook/rook/tests/framework/contracts" + "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/suite" ) @@ -57,18 +57,18 @@ type HelmSuite struct { suite.Suite helper *clients.TestClient kh *utils.K8sHelper - op contracts.Setup + op *TestCluster namespace string } func (hs *HelmSuite) SetupSuite() { hs.namespace = "helm-ns" - hs.op, hs.kh = StartBaseTestOperations(hs.T, hs.namespace, "bluestore", true, false, 1) - hs.helper = GetTestClient(hs.kh, hs.namespace, hs.op, hs.T) + hs.op, hs.kh = StartTestCluster(hs.T, hs.namespace, "bluestore", true, false, 1, installer.VersionMaster) + hs.helper = clients.CreateTestClient(hs.kh, hs.op.installer.Manifests) } func (hs *HelmSuite) TearDownSuite() { - hs.op.TearDown() + hs.op.Teardown() } //Test to make sure all rook components are installed and Running diff --git a/tests/integration/multi_cluster_deploy_test.go b/tests/integration/multi_cluster_test.go similarity index 76% rename from tests/integration/multi_cluster_deploy_test.go rename to tests/integration/multi_cluster_test.go index 9cc804fc4..2a87dd9b3 100644 --- a/tests/integration/multi_cluster_deploy_test.go +++ b/tests/integration/multi_cluster_test.go @@ -22,7 +22,6 @@ import ( "time" "github.com/rook/rook/tests/framework/clients" - "github.com/rook/rook/tests/framework/contracts" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/require" @@ -55,12 +54,11 @@ func TestMultiClusterDeploySuite(t *testing.T) { type MultiClusterDeploySuite struct { suite.Suite - helper1 *clients.TestClient - helper2 *clients.TestClient + testClient *clients.TestClient k8sh *utils.K8sHelper namespace1 string namespace2 string - op contracts.Setup + op *MCTestOperations } //Deploy Multiple Rook clusters @@ -70,89 +68,84 @@ func (mrc *MultiClusterDeploySuite) SetupSuite() { mrc.namespace2 = "mrc-n2" mrc.op, mrc.k8sh = NewMCTestOperations(mrc.T, mrc.namespace1, mrc.namespace2) - mrc.helper1 = GetTestClient(mrc.k8sh, mrc.namespace1, mrc.op, mrc.T) - mrc.helper2 = GetTestClient(mrc.k8sh, mrc.namespace2, mrc.op, mrc.T) + mrc.testClient = clients.CreateTestClient(mrc.k8sh, mrc.op.installer.Manifests) mrc.createPools() - } + func (mrc *MultiClusterDeploySuite) createPools() { // create a test pool in each cluster so that we get some PGs poolName := "multi-cluster-pool1" logger.Infof("Creating pool %s", poolName) - result, err := installer.BlockResourceOperation(mrc.k8sh, installer.GetBlockPoolDef(poolName, mrc.namespace1, "1"), "create") + result, err := mrc.testClient.PoolClient.Create(poolName, mrc.namespace1, 1) checkOrderedSubstrings(mrc.T(), result, poolName, "created") require.Nil(mrc.T(), err) poolName = "multi-cluster-pool2" logger.Infof("Creating pool %s", poolName) - result, err = installer.BlockResourceOperation(mrc.k8sh, installer.GetBlockPoolDef(poolName, mrc.namespace2, "1"), "create") + result, err = mrc.testClient.PoolClient.Create(poolName, mrc.namespace2, 1) checkOrderedSubstrings(mrc.T(), result, poolName, "created") require.Nil(mrc.T(), err) } func (mrc *MultiClusterDeploySuite) TearDownSuite() { - mrc.op.TearDown() + mrc.op.Teardown() } //Test to make sure all rook components are installed and Running func (mrc *MultiClusterDeploySuite) TestInstallingMultipleRookClusters() { //Check if Rook cluster 1 is deployed successfully checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace1, 1) - checkIfRookClusterIsHealthy(mrc.Suite, mrc.helper1, mrc.namespace1) + checkIfRookClusterIsHealthy(mrc.Suite, mrc.testClient, mrc.namespace1) //Check if Rook cluster 2 is deployed successfully checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace2, 1) - checkIfRookClusterIsHealthy(mrc.Suite, mrc.helper2, mrc.namespace2) + checkIfRookClusterIsHealthy(mrc.Suite, mrc.testClient, mrc.namespace2) } //Test Block Store Creation on multiple rook clusters func (mrc *MultiClusterDeploySuite) TestBlockStoreOnMultipleRookCluster() { - runBlockE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, mrc.namespace1) - runBlockE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, mrc.namespace2) + runBlockE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace1) + runBlockE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace2) } //Test Filesystem Creation on multiple rook clusters func (mrc *MultiClusterDeploySuite) TestFileStoreOnMultiRookCluster() { - runFileE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, mrc.namespace1, "test-fs-1") - runFileE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, mrc.namespace2, "test-fs-2") + runFileE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace1, "test-fs-1") + runFileE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace2, "test-fs-2") } //Test Object Store Creation on multiple rook clusters func (mrc *MultiClusterDeploySuite) TestObjectStoreOnMultiRookCluster() { - runObjectE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, mrc.namespace1, "default-c1", 2) - runObjectE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, mrc.namespace2, "default-c2", 1) + runObjectE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace1, "default-c1", 2) + runObjectE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace2, "default-c2", 1) } //MCTestOperations struct for handling panic and test suite tear down type MCTestOperations struct { - installer *installer.InstallHelper - installData *installer.InstallData + installer *installer.CephInstaller kh *utils.K8sHelper T func() *testing.T namespace1 string namespace2 string systemNamespace string - helper1 *clients.TestClient - helper2 *clients.TestClient } -//NewMCTestOperations creates new instance of BaseTestOperations struct -func NewMCTestOperations(t func() *testing.T, namespace1 string, namespace2 string) (MCTestOperations, *utils.K8sHelper) { +//NewMCTestOperations creates new instance of TestCluster struct +func NewMCTestOperations(t func() *testing.T, namespace1 string, namespace2 string) (*MCTestOperations, *utils.K8sHelper) { kh, err := utils.CreateK8sHelper(t) require.NoError(t(), err) - i := installer.NewK8sRookhelper(kh.Clientset, t) - d := installer.NewK8sInstallData() + i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster) - op := MCTestOperations{i, d, kh, t, namespace1, namespace2, installer.SystemNamespace(namespace1), nil, nil} - op.SetUp() + op := &MCTestOperations{i, kh, t, namespace1, namespace2, installer.SystemNamespace(namespace1)} + op.Setup() return op, kh } //SetUpRook is wrapper for setting up multiple rook clusters. -func (o MCTestOperations) SetUp() { +func (o MCTestOperations) Setup() { var err error - err = o.installer.CreateK8sRookOperator(installer.SystemNamespace(o.namespace1)) + err = o.installer.CreateCephOperator(installer.SystemNamespace(o.namespace1)) require.NoError(o.T(), err) require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-operator", o.systemNamespace, "Running"), @@ -179,7 +172,7 @@ func (o MCTestOperations) SetUp() { } //TearDownRook is a wrapper for tearDown after suite -func (o MCTestOperations) TearDown() { +func (o MCTestOperations) Teardown() { defer func() { if r := recover(); r != nil { logger.Infof("Unexpected Errors while cleaning up MultiCluster test --> %v", r) diff --git a/tests/integration/nfs_test.go b/tests/integration/nfs_test.go index 7e98d98f1..dcfadca1c 100644 --- a/tests/integration/nfs_test.go +++ b/tests/integration/nfs_test.go @@ -47,8 +47,8 @@ func TestNfsSuite(t *testing.T) { type NfsSuite struct { suite.Suite - k8sHelper *utils.K8sHelper - installHelper *installer.InstallHelper + k8shelper *utils.K8sHelper + installer *installer.NFSInstaller rwClient *clients.ReadWriteOperation namespace string systemNamespace string @@ -56,60 +56,60 @@ type NfsSuite struct { } func (suite *NfsSuite) SetupSuite() { - suite.SetUp() + suite.Setup() } func (suite *NfsSuite) TearDownSuite() { - suite.TearDown() + suite.Teardown() } -func (suite *NfsSuite) SetUp() { +func (suite *NfsSuite) Setup() { suite.namespace = "nfs-ns" suite.systemNamespace = installer.SystemNamespace(suite.namespace) suite.instanceCount = 1 - k8sHelper, err := utils.CreateK8sHelper(suite.T) + k8shelper, err := utils.CreateK8sHelper(suite.T) require.NoError(suite.T(), err) - suite.k8sHelper = k8sHelper + suite.k8shelper = k8shelper - k8sversion := suite.k8sHelper.GetK8sServerVersion() + k8sversion := suite.k8shelper.GetK8sServerVersion() logger.Infof("Installing nfs server on k8s %s", k8sversion) - suite.installHelper = installer.NewK8sRookhelper(suite.k8sHelper.Clientset, suite.T) + suite.installer = installer.NewNFSInstaller(suite.k8shelper, suite.T) - suite.rwClient = clients.CreateReadWriteOperation(suite.k8sHelper) + suite.rwClient = clients.CreateReadWriteOperation(suite.k8shelper) - err = suite.installHelper.InstallNFSServer(suite.systemNamespace, suite.namespace, suite.instanceCount) + err = suite.installer.InstallNFSServer(suite.systemNamespace, suite.namespace, suite.instanceCount) if err != nil { logger.Errorf("nfs server installation failed: %+v", err) suite.T().Fail() - suite.TearDown() + suite.Teardown() suite.T().FailNow() } } -func (suite *NfsSuite) TearDown() { +func (suite *NfsSuite) Teardown() { if suite.T().Failed() { - suite.installHelper.GatherCRDObjectDebuggingInfo(suite.systemNamespace) - suite.installHelper.GatherCRDObjectDebuggingInfo(suite.namespace) - suite.installHelper.GatherAllNFSServerLogs(suite.systemNamespace, suite.namespace, suite.T().Name()) + installer.GatherCRDObjectDebuggingInfo(suite.k8shelper, suite.systemNamespace) + installer.GatherCRDObjectDebuggingInfo(suite.k8shelper, suite.namespace) + suite.installer.GatherAllNFSServerLogs(suite.systemNamespace, suite.namespace, suite.T().Name()) } - suite.installHelper.UninstallNFSServer(suite.systemNamespace, suite.namespace) + suite.installer.UninstallNFSServer(suite.systemNamespace, suite.namespace) } func (suite *NfsSuite) TestNfsServerInstallation() { logger.Infof("Verifying that nfs server pod %s is running", suite.namespace) // verify nfs server operator is running OK - assert.True(suite.T(), suite.k8sHelper.CheckPodCountAndState("rook-nfs-operator", suite.systemNamespace, 1, "Running"), + assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-nfs-operator", suite.systemNamespace, 1, "Running"), "1 rook-nfs-operator must be in Running state") // verify nfs server instances are running OK - assert.True(suite.T(), suite.k8sHelper.CheckPodCountAndState("rook-nfs", suite.namespace, suite.instanceCount, "Running"), + assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-nfs", suite.namespace, suite.instanceCount, "Running"), fmt.Sprintf("%d rook-nfs pods must be in Running state", suite.instanceCount)) // verify nfs server storage - assert.True(suite.T(), true, suite.k8sHelper.WaitUntilPVCIsBound("default", "nfs-pv-claim")) + assert.True(suite.T(), true, suite.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim")) defer suite.rwClient.Delete() podList, err := suite.rwClient.CreateWriteClient("nfs-pv-claim") diff --git a/tests/integration/pool_test.go b/tests/integration/pool_test.go index bc1ea1eb1..e28623f43 100644 --- a/tests/integration/pool_test.go +++ b/tests/integration/pool_test.go @@ -20,7 +20,6 @@ import ( "fmt" "time" - "github.com/rook/rook/pkg/daemon/ceph/model" "github.com/stretchr/testify/require" ) @@ -29,8 +28,7 @@ func (suite *SmokeSuite) TestPoolResize() { logger.Infof("Pool Resize Smoke Test") poolName := "testpool" - pool := model.Pool{Name: "testpool", ReplicatedConfig: model.ReplicatedPoolConfig{Size: 1}} - out, err := suite.helper.PoolClient.Create(pool, suite.namespace) + out, err := suite.helper.PoolClient.Create(poolName, suite.namespace, 1) logger.Infof("poolCreate: %+v", out) require.Nil(suite.T(), err) @@ -56,8 +54,7 @@ func (suite *SmokeSuite) TestPoolResize() { require.Equal(suite.T(), true, poolFound, "pool not found") - pool.ReplicatedConfig.Size = 3 - _, err = suite.helper.PoolClient.Update(pool, suite.namespace) + _, err = suite.helper.PoolClient.Update(poolName, suite.namespace, 3) logger.Infof("poolCreate (modify): %+v", out) require.Nil(suite.T(), err) diff --git a/tests/integration/smoke_test.go b/tests/integration/smoke_test.go index 83727b045..2b70ed9ab 100644 --- a/tests/integration/smoke_test.go +++ b/tests/integration/smoke_test.go @@ -23,7 +23,6 @@ import ( "testing" "github.com/rook/rook/tests/framework/clients" - "github.com/rook/rook/tests/framework/contracts" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/assert" @@ -69,7 +68,7 @@ func TestSmokeSuite(t *testing.T) { type SmokeSuite struct { suite.Suite helper *clients.TestClient - op contracts.Setup + op *TestCluster k8sh *utils.K8sHelper namespace string } @@ -77,12 +76,12 @@ type SmokeSuite struct { func (suite *SmokeSuite) SetupSuite() { suite.namespace = "smoke-ns" useDevices := true - suite.op, suite.k8sh = StartBaseTestOperations(suite.T, suite.namespace, "bluestore", false, useDevices, 3) - suite.helper = GetTestClient(suite.k8sh, suite.namespace, suite.op, suite.T) + suite.op, suite.k8sh = StartTestCluster(suite.T, suite.namespace, "bluestore", false, useDevices, 3, installer.VersionMaster) + suite.helper = clients.CreateTestClient(suite.k8sh, suite.op.installer.Manifests) } func (suite *SmokeSuite) TearDownSuite() { - suite.op.TearDown() + suite.op.Teardown() } func (suite *SmokeSuite) TestBlockStorage_SmokeTest() { diff --git a/tests/integration/zblock_mount_unmount_test.go b/tests/integration/zblock_mount_unmount_test.go index 802d96b10..0534c33cb 100644 --- a/tests/integration/zblock_mount_unmount_test.go +++ b/tests/integration/zblock_mount_unmount_test.go @@ -22,7 +22,6 @@ import ( "fmt" "github.com/rook/rook/tests/framework/clients" - "github.com/rook/rook/tests/framework/contracts" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/assert" @@ -76,8 +75,8 @@ type BlockMountUnMountSuite struct { namespace string pvcNameRWO string pvcNameRWX string - installer *installer.InstallHelper - op contracts.Setup + installer *installer.CephInstaller + op *TestCluster } func (s *BlockMountUnMountSuite) SetupSuite() { @@ -87,8 +86,8 @@ func (s *BlockMountUnMountSuite) SetupSuite() { s.pvcNameRWX = "block-persistent-rwx" useHelm := false useDevices := true - s.op, s.kh = StartBaseTestOperations(s.T, s.namespace, "filestore", useHelm, useDevices, 1) - s.testClient = GetTestClient(s.kh, s.namespace, s.op, s.T) + s.op, s.kh = StartTestCluster(s.T, s.namespace, "filestore", useHelm, useDevices, 1, installer.VersionMaster) + s.testClient = clients.CreateTestClient(s.kh, s.op.installer.Manifests) s.bc = s.testClient.BlockClient } @@ -98,16 +97,16 @@ func (s *BlockMountUnMountSuite) setupPVCs() { storageClassNameRWO := "rook-ceph-block-rwo" // Create PVCs - _, cbErr := installer.BlockResourceOperation(s.kh, installer.GetBlockPoolStorageClassAndPvcDef(s.namespace, poolNameRWO, storageClassNameRWO, s.pvcNameRWO, "ReadWriteOnce"), "create") + _, cbErr := s.testClient.PoolClient.CreateStorageClassAndPvc(s.namespace, poolNameRWO, storageClassNameRWO, s.pvcNameRWO, "ReadWriteOnce") require.Nil(s.T(), cbErr) require.True(s.T(), s.kh.WaitUntilPVCIsBound(defaultNamespace, s.pvcNameRWO), "Make sure PVC is Bound") - _, cbErr2 := installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(s.pvcNameRWX, storageClassNameRWO, "ReadWriteMany"), "create") + _, cbErr2 := s.testClient.BlockClient.CreatePvc(s.pvcNameRWX, storageClassNameRWO, "ReadWriteMany") require.Nil(s.T(), cbErr2) require.True(s.T(), s.kh.WaitUntilPVCIsBound(defaultNamespace, s.pvcNameRWX), "Make sure PVC is Bound") // Mount PVC on a pod and write some data. - _, mtErr := s.bc.BlockMap(getBlockPodDefintion("setup-block-rwo", s.pvcNameRWO, false), blockMountPath) + _, mtErr := s.bc.BlockMap(getBlockPodDefintion("setup-block-rwo", s.pvcNameRWO, false)) require.Nil(s.T(), mtErr) crdName, err := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWO) require.Nil(s.T(), err) @@ -119,7 +118,7 @@ func (s *BlockMountUnMountSuite) setupPVCs() { } require.True(s.T(), rwoVolumePresent, fmt.Sprintf("make sure rwo Volume %s is created", crdName)) - _, mtErr1 := s.bc.BlockMap(getBlockPodDefintion("setup-block-rwx", s.pvcNameRWX, false), blockMountPath) + _, mtErr1 := s.bc.BlockMap(getBlockPodDefintion("setup-block-rwx", s.pvcNameRWX, false)) require.Nil(s.T(), mtErr1) crdName1, err1 := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWX) require.Nil(s.T(), err1) @@ -128,37 +127,29 @@ func (s *BlockMountUnMountSuite) setupPVCs() { require.True(s.T(), s.kh.IsPodRunning("setup-block-rwx", defaultNamespace), "make sure setup-block-rwx pod is in running state") // Write Data to Pod - _, wtErr1 := s.bc.Write("setup-block-rwo", blockMountPath, "Persisted message one", "bsFile1", "") + _, wtErr1 := s.bc.Write("setup-block-rwo", "Persisted message one", "bsFile1", "") require.Nil(s.T(), wtErr1) - _, wtErr2 := s.bc.Write("setup-block-rwx", blockMountPath, "Persisted message one", "bsFile1", "") + _, wtErr2 := s.bc.Write("setup-block-rwx", "Persisted message one", "bsFile1", "") require.Nil(s.T(), wtErr2) // Unmount pod - _, unmtErr1 := s.bc.BlockUnmap(getBlockPodDefintion("setup-block-rwo", s.pvcNameRWO, false), blockMountPath) - _, unmtErr2 := s.bc.BlockUnmap(getBlockPodDefintion("setup-block-rwx", s.pvcNameRWO, false), blockMountPath) + _, unmtErr1 := s.kh.DeletePods("setup-block-rwo", "setup-block-rwx") require.Nil(s.T(), unmtErr1) - require.Nil(s.T(), unmtErr2) require.True(s.T(), s.kh.IsPodTerminated("setup-block-rwo", defaultNamespace), "make sure setup-block-rwo pod is terminated") require.True(s.T(), s.kh.IsPodTerminated("setup-block-rwx", defaultNamespace), "make sure setup-block-rwx pod is terminated") } func (s *BlockMountUnMountSuite) TearDownSuite() { logger.Infof("Cleaning up block storage") - s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("setup-block-rwo", s.pvcNameRWO, false), blockMountPath) - s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("setup-block-rwx", s.pvcNameRWX, false), blockMountPath) - s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwo-block-rw-one", s.pvcNameRWO, false), blockMountPath) - s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwo-block-rw-two", s.pvcNameRWO, false), blockMountPath) - s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true), blockMountPath) - s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwo-block-ro-two", s.pvcNameRWO, true), blockMountPath) - s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwx-block-rw-one", s.pvcNameRWX, false), blockMountPath) - s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwx-block-rw-two", s.pvcNameRWX, false), blockMountPath) - s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true), blockMountPath) - s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwx-block-ro-two", s.pvcNameRWX, true), blockMountPath) - installer.BlockResourceOperation(s.kh, installer.GetBlockPoolStorageClassAndPvcDef(s.namespace, "block-pool-rwo", "rook-ceph-block-rwo", s.pvcNameRWO, "ReadWriteOnce"), "delete") - installer.BlockResourceOperation(s.kh, installer.GetBlockPoolStorageClassAndPvcDef(s.namespace, "block-pool-rwx", "rook-ceph-block-rwx", s.pvcNameRWX, "ReadWriteMany"), "delete") + + s.kh.DeletePods( + "setup-block-rwo", "setup-block-rwx", "rwo-block-rw-one", "rwo-block-rw-two", "rwo-block-ro-one", + "rwo-block-ro-two", "rwx-block-rw-one", "rwx-block-rw-two", "rwx-block-ro-one", "rwx-block-ro-two") + s.testClient.PoolClient.DeleteStorageClassAndPvc(s.namespace, "block-pool-rwo", "rook-ceph-block-rwo", s.pvcNameRWO, "ReadWriteOnce") + s.testClient.PoolClient.DeleteStorageClassAndPvc(s.namespace, "block-pool-rwx", "rook-ceph-block-rwx", s.pvcNameRWX, "ReadWriteMany") cleanupDynamicBlockStorage(s.testClient, s.namespace) - s.op.TearDown() + s.op.Teardown() } func (s *BlockMountUnMountSuite) TestBlockStorageMountUnMountForDifferentAccessModes() { @@ -167,11 +158,11 @@ func (s *BlockMountUnMountSuite) TestBlockStorageMountUnMountForDifferentAccessM logger.Infof("Test case when existing RWO PVC is mounted and unmounted on pods with various accessModes") logger.Infof("Step 1.1: Mount existing ReadWriteOnce and ReadWriteMany PVC on a Pod with RW access") //mount PVC with RWO access on a pod with readonly set to false - _, mtErr1 := s.bc.BlockMap(getBlockPodDefintion("rwo-block-rw-one", s.pvcNameRWO, false), blockMountPath) - require.Nil(s.T(), mtErr1) + _, err := s.bc.BlockMap(getBlockPodDefintion("rwo-block-rw-one", s.pvcNameRWO, false)) + require.Nil(s.T(), err) //mount PVC with RWX access on a pod with readonly set to false - _, mtErr2 := s.bc.BlockMap(getBlockPodDefintion("rwx-block-rw-one", s.pvcNameRWX, false), blockMountPath) - require.Nil(s.T(), mtErr2) + _, err = s.bc.BlockMap(getBlockPodDefintion("rwx-block-rw-one", s.pvcNameRWX, false)) + require.Nil(s.T(), err) crdName1, err1 := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWO) crdName2, err2 := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWX) assert.Nil(s.T(), err1) @@ -184,85 +175,76 @@ func (s *BlockMountUnMountSuite) TestBlockStorageMountUnMountForDifferentAccessM logger.Infof("Step 2: Check if previously persisted data is readable from ReadWriteOnce and ReadWriteMany PVC") //Read data on RWO PVC Mounted on pod with RW Access - read1, rErr1 := s.bc.Read("rwo-block-rw-one", blockMountPath, "bsFile1", "") - assert.Nil(s.T(), rErr1) + read1, err := s.bc.Read("rwo-block-rw-one", "bsFile1", "") + assert.Nil(s.T(), err) assert.Contains(s.T(), read1, "Persisted message one", "make sure previously persisted data is readable") //Read data on RWX PVC Mounted on pod with RW Access - read2, rErr2 := s.bc.Read("rwx-block-rw-one", blockMountPath, "bsFile1", "") - assert.Nil(s.T(), rErr2) + read2, err := s.bc.Read("rwx-block-rw-one", "bsFile1", "") + assert.Nil(s.T(), err) assert.Contains(s.T(), read2, "Persisted message one", "make sure previously persisted data is readable") logger.Infof("Step 3: Check if read/write works on ReadWriteOnce and ReadWriteMany PVC") //Write data on RWO PVC Mounted on pod with RW Access - _, wtErr1 := s.bc.Write("rwo-block-rw-one", blockMountPath, "Persisted message two", "bsFile2", "") - assert.Nil(s.T(), wtErr1) + _, err = s.bc.Write("rwo-block-rw-one", "Persisted message two", "bsFile2", "") + assert.Nil(s.T(), err) //Read data on RWO PVC Mounted on pod with RW Access - read1, rErr1 = s.bc.Read("rwo-block-rw-one", blockMountPath, "bsFile2", "") - assert.Nil(s.T(), rErr1) + read1, err = s.bc.Read("rwo-block-rw-one", "bsFile2", "") + assert.Nil(s.T(), err) assert.Contains(s.T(), read1, "Persisted message two", "make sure new persisted data is readable") //Write data on RWX PVC Mounted on pod with RW Access - _, wtErr2 := s.bc.Write("rwx-block-rw-one", blockMountPath, "Persisted message two", "bsFile2", "") - assert.Nil(s.T(), wtErr2) + _, err = s.bc.Write("rwx-block-rw-one", "Persisted message two", "bsFile2", "") + assert.Nil(s.T(), err) //Read data on RWX PVC Mounted on pod with RW Access - read2, rErr2 = s.bc.Read("rwx-block-rw-one", blockMountPath, "bsFile2", "") - assert.Nil(s.T(), rErr2) + read2, err = s.bc.Read("rwx-block-rw-one", "bsFile2", "") + assert.Nil(s.T(), err) assert.Contains(s.T(), read2, "Persisted message two", "make sure new persisted data is readable") //Mount another Pod with RW access on same PVC logger.Infof("Step 4: Mount existing ReadWriteOnce and ReadWriteMany PVC on a new Pod with RW access") //Mount RWO PVC on a new pod with ReadOnly set to false - _, mtErr1 = s.bc.BlockMap(getBlockPodDefintion("rwo-block-rw-two", s.pvcNameRWO, false), blockMountPath) - assert.Nil(s.T(), mtErr1) + _, err = s.bc.BlockMap(getBlockPodDefintion("rwo-block-rw-two", s.pvcNameRWO, false)) + assert.Nil(s.T(), err) //Mount RWX PVC on a new pod with ReadOnly set to false - _, mtErr2 = s.bc.BlockMap(getBlockPodDefintion("rwx-block-rw-two", s.pvcNameRWX, false), blockMountPath) - assert.Nil(s.T(), mtErr2) + _, err = s.bc.BlockMap(getBlockPodDefintion("rwx-block-rw-two", s.pvcNameRWX, false)) + assert.Nil(s.T(), err) assert.True(s.T(), s.kh.IsPodInError("rwo-block-rw-two", defaultNamespace, "FailedMount", "Volume is already attached by pod"), "make sure rwo-block-rw-two pod errors out while mounting the volume") assert.True(s.T(), s.kh.IsPodInError("rwx-block-rw-two", defaultNamespace, "FailedMount", "Volume is already attached by pod"), "make sure rwx-block-rw-two pod errors out while mounting the volume") - _, unmtErr1 := s.bc.BlockUnmap(getBlockPodDefintion("rwo-block-rw-two", s.pvcNameRWO, false), blockMountPath) - assert.Nil(s.T(), unmtErr1) - - _, unmtErr2 := s.bc.BlockUnmap(getBlockPodDefintion("rwx-block-rw-two", s.pvcNameRWX, false), blockMountPath) - assert.Nil(s.T(), unmtErr2) + _, err = s.kh.DeletePods("rwo-block-rw-two", "rwx-block-rw-two") + assert.Nil(s.T(), err) assert.True(s.T(), s.kh.IsPodTerminated("rwo-block-rw-two", defaultNamespace), "make sure rwo-block-rw-two pod is terminated") assert.True(s.T(), s.kh.IsPodTerminated("rwx-block-rw-two", defaultNamespace), "make sure rwx-block-rw-two pod is terminated") logger.Infof("Step 5: Mount existing ReadWriteOnce and ReadWriteMany PVC on a new Pod with RO access") //Mount RWO PVC on a new pod with ReadOnly set to true - _, mtErr1 = s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true), blockMountPath) - assert.Nil(s.T(), mtErr1) + _, err = s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true)) + assert.Nil(s.T(), err) //Mount RWX PVC on a new pod with ReadOnly set to true - _, mtErr2 = s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true), blockMountPath) - assert.Nil(s.T(), mtErr2) + _, err = s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true)) + assert.Nil(s.T(), err) assert.True(s.T(), s.kh.IsPodInError("rwo-block-ro-one", defaultNamespace, "FailedMount", "Volume is already attached by pod"), "make sure rwo-block-ro-one pod errors out while mounting the volume") assert.True(s.T(), s.kh.IsPodInError("rwx-block-ro-one", defaultNamespace, "FailedMount", "Volume is already attached by pod"), "make sure rwx-block-ro-one pod errors out while mounting the volume") - _, unmtErr1 = s.bc.BlockUnmap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true), blockMountPath) - assert.Nil(s.T(), unmtErr1) - _, unmtErr2 = s.bc.BlockUnmap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true), blockMountPath) - assert.Nil(s.T(), unmtErr2) + _, err = s.kh.DeletePods("rwo-block-ro-one", "rwx-block-ro-one") + assert.Nil(s.T(), err) assert.True(s.T(), s.kh.IsPodTerminated("rwo-block-ro-one", defaultNamespace), "make sure rwo-block-ro-one pod is terminated") assert.True(s.T(), s.kh.IsPodTerminated("rwx-block-ro-one", defaultNamespace), "make sure rwx-block-ro-one pod is terminated") logger.Infof("Step 6: UnMount Pod with RWX and RWO access") - //UnMount RWO PVC - _, unmtErr1 = s.bc.BlockUnmap(getBlockPodDefintion("rwo-block-rw-one", s.pvcNameRWO, false), blockMountPath) - assert.Nil(s.T(), unmtErr1) - //UnMount RWX PVC - _, unmtErr2 = s.bc.BlockUnmap(getBlockPodDefintion("rwx-block-rw-one", s.pvcNameRWX, false), blockMountPath) - assert.Nil(s.T(), unmtErr2) + _, err = s.kh.DeletePods("rwo-block-rw-one", "rwx-block-rw-one") + assert.Nil(s.T(), err) assert.True(s.T(), s.kh.IsPodTerminated("rwo-block-rw-one", defaultNamespace), "make sure rwo-block-rw-one pod is terminated") assert.True(s.T(), s.kh.IsPodTerminated("rwx-block-rw-one", defaultNamespace), "make sure rwx-lock-rw-one pod is terminated") logger.Infof("Step 7: Mount ReadWriteOnce and ReadWriteMany PVC on two different pods with ReadOnlyMany with Readonly Access") //Mount RWO PVC on 2 pods with ReadOnly set to True - _, mtErr1_1 := s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true), blockMountPath) - _, mtErr2_1 := s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-two", s.pvcNameRWO, true), blockMountPath) - assert.Nil(s.T(), mtErr1_1) - assert.Nil(s.T(), mtErr2_1) + _, err1 = s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true)) + _, err2 = s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-two", s.pvcNameRWO, true)) + assert.Nil(s.T(), err1) + assert.Nil(s.T(), err2) //Mount RWX PVC on 2 pods with ReadOnly set to True - _, mtErr1_2 := s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true), blockMountPath) - _, mtErr2_2 := s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-two", s.pvcNameRWX, true), blockMountPath) - assert.Nil(s.T(), mtErr1_2) - assert.Nil(s.T(), mtErr2_2) + _, err1 = s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true)) + _, err2 = s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-two", s.pvcNameRWX, true)) + assert.Nil(s.T(), err1) + assert.Nil(s.T(), err2) assert.True(s.T(), s.kh.IsPodRunning("rwo-block-ro-one", defaultNamespace), "make sure rwo-block-ro-one pod is in running state") assert.True(s.T(), s.kh.IsPodRunning("rwo-block-ro-two", defaultNamespace), "make sure rwo-block-ro-two pod is in running state") assert.True(s.T(), s.kh.IsPodRunning("rwx-block-ro-one", defaultNamespace), "make sure rwx-block-ro-one pod is in running state") @@ -270,39 +252,32 @@ func (s *BlockMountUnMountSuite) TestBlockStorageMountUnMountForDifferentAccessM logger.Infof("Step 8: Read Data from both ReadyOnlyMany and ReadWriteOnce pods with ReadOnly Access") //Read data from RWO PVC via both ReadOnly pods - read1_1, rErr1_1 := s.bc.Read("rwo-block-ro-one", blockMountPath, "bsFile1", "") - read2_1, rErr2_1 := s.bc.Read("rwo-block-ro-two", blockMountPath, "bsFile1", "") - assert.Nil(s.T(), rErr1_1) - assert.Nil(s.T(), rErr2_1) + read1_1, err1 := s.bc.Read("rwo-block-ro-one", "bsFile1", "") + read2_1, err2 := s.bc.Read("rwo-block-ro-two", "bsFile1", "") + assert.Nil(s.T(), err1) + assert.Nil(s.T(), err2) assert.Contains(s.T(), read1_1, "Persisted message one", "make sure previously persisted data is readable from ReadOnlyMany access Pod") assert.Contains(s.T(), read2_1, "Persisted message one", "make sure previously persisted data is readable from ReadOnlyMany access Pod") //Read data from RWX PVC via both ReadOnly pods - read1_2, rErr1_2 := s.bc.Read("rwx-block-ro-one", blockMountPath, "bsFile1", "") - read2_2, rErr2_2 := s.bc.Read("rwx-block-ro-two", blockMountPath, "bsFile1", "") - assert.Nil(s.T(), rErr1_2) - assert.Nil(s.T(), rErr2_2) + read1_2, err1 := s.bc.Read("rwx-block-ro-one", "bsFile1", "") + read2_2, err2 := s.bc.Read("rwx-block-ro-two", "bsFile1", "") + assert.Nil(s.T(), err1) + assert.Nil(s.T(), err2) assert.Contains(s.T(), read1_2, "Persisted message one", "make sure previously persisted data is readable from ReadOnlyMany access Pod") assert.Contains(s.T(), read2_2, "Persisted message one", "make sure previously persisted data is readable from ReadOnlyMany access Pod") logger.Infof("Step 9: Write Data to Pod with ReadOnlyMany and ReadWriteOnce PVC mounted with ReadOnly access") //Write data to RWO PVC via pod with ReadOnly Set to true - _, wtErr1 = s.bc.Write("rwo-block-ro-one", blockMountPath, "Persisted message three", "bsFile3", "") - assert.Contains(s.T(), wtErr1.Error(), "Unable to write data to pod") + _, err = s.bc.Write("rwo-block-ro-one", "Persisted message three", "bsFile3", "") + assert.Contains(s.T(), err.Error(), "Unable to write data to pod") //Write data to RWx PVC via pod with ReadOnly Set to true - _, wtErr2 = s.bc.Write("rwx-block-ro-one", blockMountPath, "Persisted message three", "bsFile3", "") - assert.Contains(s.T(), wtErr2.Error(), "Unable to write data to pod") + _, err = s.bc.Write("rwx-block-ro-one", "Persisted message three", "bsFile3", "") + assert.Contains(s.T(), err.Error(), "Unable to write data to pod") logger.Infof("Step 10: UnMount Pod with ReadOnlyMany and ReadWriteOnce PVCs") //UnMount RWO PVC from both ReadOnly Pods - _, unmtErr1_1 := s.bc.BlockUnmap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true), blockMountPath) - _, unmtErr2_1 := s.bc.BlockUnmap(getBlockPodDefintion("rwo-block-ro-two", s.pvcNameRWO, true), blockMountPath) - assert.Nil(s.T(), unmtErr1_1) - assert.Nil(s.T(), unmtErr2_1) - //UnMount RWX PVC from both ReadOnly Pods - _, unmtErr1_2 := s.bc.BlockUnmap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true), blockMountPath) - _, unmtErr2_2 := s.bc.BlockUnmap(getBlockPodDefintion("rwx-block-ro-two", s.pvcNameRWX, true), blockMountPath) - assert.Nil(s.T(), unmtErr1_2) - assert.Nil(s.T(), unmtErr2_2) + _, err = s.kh.DeletePods("rwo-block-ro-one", "rwo-block-ro-two", "rwx-block-ro-one", "rwx-block-ro-two") + assert.Nil(s.T(), err) assert.True(s.T(), s.kh.IsPodTerminated("rwo-block-ro-one", defaultNamespace), "make sure rwo-block-ro-one pod is terminated") assert.True(s.T(), s.kh.IsPodTerminated("rwo-block-ro-two", defaultNamespace), "make sure rwo-block-ro-two pod is terminated") assert.True(s.T(), s.kh.IsPodTerminated("rwx-block-ro-one", defaultNamespace), "make sure rwx-lock-ro-one pod is terminated") diff --git a/tests/longhaul/base_block_test.go b/tests/longhaul/base_block_test.go index cbd508a9d..ac66a1112 100644 --- a/tests/longhaul/base_block_test.go +++ b/tests/longhaul/base_block_test.go @@ -18,13 +18,13 @@ var ( ) // Create StorageClass and poll if needed -func createStorageClassAndPool(t func() *testing.T, kh *utils.K8sHelper, namespace string, storageClassName string, poolName string) { +func createStorageClassAndPool(t func() *testing.T, testClient *clients.TestClient, kh *utils.K8sHelper, namespace string, storageClassName string, poolName string) { //create storage class if err := kh.IsStorageClassPresent(storageClassName); err != nil { logger.Infof("Install pool and storage class for rook block") - _, err := installer.BlockResourceOperation(kh, installer.GetBlockPoolDef(poolName, namespace, "3"), "create") + _, err := testClient.PoolClient.Create(poolName, namespace, 3) require.NoError(t(), err) - _, err = installer.BlockResourceOperation(kh, installer.GetBlockStorageClassDef(poolName, storageClassName, namespace, false), "create") + _, err = testClient.BlockClient.CreateStorageClass(poolName, storageClassName, namespace, false) require.NoError(t(), err) //make sure storageclass is created @@ -90,7 +90,7 @@ func mountUnmountPVCOnPod(k8sh *utils.K8sHelper, podName string, pvcName string, return result, err } -func performBlockOperations(installer *installer.InstallHelper, db *utils.MySQLHelper) { +func performBlockOperations(db *utils.MySQLHelper) { var wg sync.WaitGroup for i := 1; i <= installer.Env.LoadConcurrentRuns; i++ { wg.Add(1) @@ -133,29 +133,29 @@ func dbOperation(db *utils.MySQLHelper, wg *sync.WaitGroup, runtime int, loadSiz } -//BaseTestOperations struct for handling panic and test suite tear down -type BaseLoadTestOperations struct { - installer *installer.InstallHelper +// LoadTestCluster struct for handling panic and test suite tear down +type LoadTestCluster struct { + installer *installer.CephInstaller kh *utils.K8sHelper helper *clients.TestClient T func() *testing.T namespace string } -// StartBaseTestOperations creates new instance of BaseTestOperations struct -func StartBaseLoadTestOperations(t func() *testing.T, namespace string) (BaseLoadTestOperations, *utils.K8sHelper, *installer.InstallHelper) { +// StartLoadTestCluster creates new instance of TestCluster struct +func StartLoadTestCluster(t func() *testing.T, namespace string) (LoadTestCluster, *utils.K8sHelper, *installer.CephInstaller) { kh, err := utils.CreateK8sHelper(t) require.NoError(t(), err) - i := installer.NewK8sRookhelper(kh.Clientset, t) + i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster) - op := BaseLoadTestOperations{i, kh, nil, t, namespace} - op.SetUp() + op := LoadTestCluster{i, kh, nil, t, namespace} + op.Setup() return op, kh, i } -//SetUpRook is a wrapper for setting up rook -func (o BaseLoadTestOperations) SetUp() { +// Setup is a wrapper for setting up rook +func (o LoadTestCluster) Setup() { if !o.kh.IsRookInstalled(o.namespace) { isRookInstalled, err := o.installer.InstallRookOnK8sWithHostPathAndDevices(o.namespace, "bluestore", @@ -166,14 +166,14 @@ func (o BaseLoadTestOperations) SetUp() { } // Enable chaos monkey if enable_chaos flag is present - if o.installer.Env.EnableChaos { + if installer.Env.EnableChaos { c := NewChaosHelper(o.namespace, o.kh) go c.Monkey() } } //TearDownRook is a wrapper for tearDown after suite -func (o BaseLoadTestOperations) TearDown() { +func (o LoadTestCluster) Teardown() { // No Clean up for load test } @@ -262,7 +262,7 @@ spec: imagePullPolicy: IfNotPresent volumeMounts: - name: block-persistent-storage - mountPath: /tmp/rook1 + mountPath: ` + clients.TestMountPath + ` volumes: - name: block-persistent-storage persistentVolumeClaim: diff --git a/tests/longhaul/base_object_test.go b/tests/longhaul/base_object_test.go index 876d0ccd9..d6fba6c40 100644 --- a/tests/longhaul/base_object_test.go +++ b/tests/longhaul/base_object_test.go @@ -69,7 +69,7 @@ func isObjectStorePresent(kh *utils.K8sHelper, namespace string, storeName strin return false } -func performObjectStoreOperations(installer *installer.InstallHelper, s3 *utils.S3Helper, bucketName string) { +func performObjectStoreOperations(s3 *utils.S3Helper, bucketName string) { var wg sync.WaitGroup for i := 1; i <= installer.Env.LoadConcurrentRuns; i++ { wg.Add(1) diff --git a/tests/longhaul/block_test.go b/tests/longhaul/block_test.go index b72e3b570..17394d4f6 100644 --- a/tests/longhaul/block_test.go +++ b/tests/longhaul/block_test.go @@ -21,9 +21,10 @@ import ( "sync" "testing" + "github.com/rook/rook/tests/framework/clients" + "time" - "github.com/rook/rook/tests/framework/contracts" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/suite" @@ -44,18 +45,20 @@ func TestBlockLongHaul(t *testing.T) { type BlockLongHaulSuite struct { suite.Suite - kh *utils.K8sHelper - installer *installer.InstallHelper - namespace string - op contracts.Setup + kh *utils.K8sHelper + installer *installer.CephInstaller + namespace string + op installer.TestSuite + testClient *clients.TestClient } //Test set up - does the following in order //create pool and storage class, create a PVC, Create a MySQL app/service that uses pvc func (s *BlockLongHaulSuite) SetupSuite() { s.namespace = "longhaul-ns" - s.op, s.kh, s.installer = StartBaseLoadTestOperations(s.T, s.namespace) - createStorageClassAndPool(s.T, s.kh, s.namespace, "rook-ceph-block", "rook-pool") + s.op, s.kh, s.installer = StartLoadTestCluster(s.T, s.namespace) + s.testClient = clients.CreateTestClient(s.kh, s.installer.Manifests) + createStorageClassAndPool(s.T, s.testClient, s.kh, s.namespace, "rook-ceph-block", "rook-pool") } //create a n number ofPVC, Create a MySQL app/service that uses pvc @@ -63,8 +66,8 @@ func (s *BlockLongHaulSuite) SetupSuite() { //All other PVC and mounts are created and deleted/unmounted per test func (s *BlockLongHaulSuite) TestBlockLonghaulRun() { var wg sync.WaitGroup - wg.Add(s.installer.Env.LoadVolumeNumber) - for i := 1; i <= s.installer.Env.LoadVolumeNumber; i++ { + wg.Add(installer.Env.LoadVolumeNumber) + for i := 1; i <= installer.Env.LoadVolumeNumber; i++ { if i == 1 { go BlockVolumeOperations(s, &wg, "rook-ceph-block", "mysqlapp-persist", "mysqldb", "mysql-persist-claim", false) } else { @@ -79,7 +82,7 @@ func (s *BlockLongHaulSuite) TestBlockLonghaulRun() { func BlockVolumeOperations(s *BlockLongHaulSuite, wg *sync.WaitGroup, storageClassName string, appName string, appLabel string, pvcName string, cleanup bool) { defer wg.Done() db := createPVCAndMountMysqlPod(s.T, s.kh, storageClassName, appName, appLabel, pvcName) - performBlockOperations(s.installer, db) + performBlockOperations(db) if cleanup { mySqlPodOperation(s.kh, storageClassName, appName, appLabel, pvcName, "delete") s.kh.WaitUntilPodWithLabelDeleted(appLabel, "default") @@ -91,7 +94,7 @@ func BlockVolumeOperations(s *BlockLongHaulSuite, wg *sync.WaitGroup, storageCla func (s *BlockLongHaulSuite) TearDownSuite() { //clean up all ephemeral block storage, index 1 is persistent block storage which is going to be used among different test runs. - for i := 2; i <= s.installer.Env.LoadVolumeNumber; i++ { + for i := 2; i <= installer.Env.LoadVolumeNumber; i++ { mySqlPodOperation(s.kh, "rook-ceph-block", "mysqlapp-ephemeral-"+strconv.Itoa(i), "mysqldbeph"+strconv.Itoa(i), "mysql-ephemeral-claim"+strconv.Itoa(i), "delete") } s.kh = nil diff --git a/tests/longhaul/block_with_fencing_test.go b/tests/longhaul/block_with_fencing_test.go index c4f4b7c11..6fd4ba094 100644 --- a/tests/longhaul/block_with_fencing_test.go +++ b/tests/longhaul/block_with_fencing_test.go @@ -24,7 +24,6 @@ import ( "time" "github.com/rook/rook/tests/framework/clients" - "github.com/rook/rook/tests/framework/contracts" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/assert" @@ -46,42 +45,40 @@ func TestBlockWithFencingLongHaul(t *testing.T) { type BlockLongHaulSuiteWithFencing struct { suite.Suite kh *utils.K8sHelper - installer *installer.InstallHelper + installer *installer.CephInstaller testClient *clients.TestClient namespace string - op contracts.Setup + op installer.TestSuite } //Test set up - does the following in order //create pool and storage class, Create a PVC and mount a pod with ReadOnly=false. //Write some data to the pvc and unmount the pod func (s *BlockLongHaulSuiteWithFencing) SetupSuite() { - var err error s.namespace = "longhaul-ns" - s.op, s.kh, s.installer = StartBaseLoadTestOperations(s.T, s.namespace) - createStorageClassAndPool(s.T, s.kh, s.namespace, "rook-ceph-block", "rook-pool") - s.testClient, err = clients.CreateTestClient(s.kh, s.namespace) - require.Nil(s.T(), err) + s.op, s.kh, s.installer = StartLoadTestCluster(s.T, s.namespace) + s.testClient = clients.CreateTestClient(s.kh, s.installer.Manifests) + createStorageClassAndPool(s.T, s.testClient, s.kh, s.namespace, "rook-ceph-block", "rook-pool") if _, err := s.kh.GetPVCStatus(defaultNamespace, "block-pv-one"); err != nil { logger.Infof("Creating PVC and mounting it to pod with readOnly set to false") - installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef("block-pv-one", "rook-ceph-block", "ReadWriteOnce"), "create") + _, err = s.testClient.BlockClient.CreatePvc("block-pv-one", "rook-ceph-block", "ReadWriteOnce") + require.Nil(s.T(), err) mountUnmountPVCOnPod(s.kh, "block-rw", "block-pv-one", "false", "create") require.True(s.T(), s.kh.IsPodRunning("block-rw", defaultNamespace)) - s.testClient.BlockClient.Write("block-rw", "/tmp/rook1", "this is long running test", "longhaul", defaultNamespace) + s.testClient.BlockClient.Write("block-rw", "this is long running test", "longhaul", defaultNamespace) mountUnmountPVCOnPod(s.kh, "block-rw", "block-pv-one", "false", "delete") require.True(s.T(), s.kh.IsPodTerminated("block-rw", defaultNamespace)) time.Sleep(5 * time.Second) } - } //Mount n number of pods on the same PVC created earlier with readOnly=True. func (s *BlockLongHaulSuiteWithFencing) TestBlockWithFencingLonghaulRun() { var wg sync.WaitGroup - wg.Add(s.installer.Env.LoadVolumeNumber) - for i := 1; i <= s.installer.Env.LoadVolumeNumber; i++ { + wg.Add(installer.Env.LoadVolumeNumber) + for i := 1; i <= installer.Env.LoadVolumeNumber; i++ { go blockVolumeFencingOperations(s, &wg, "block-read"+strconv.Itoa(i), "block-pv-one") } wg.Wait() @@ -91,7 +88,7 @@ func blockVolumeFencingOperations(s *BlockLongHaulSuiteWithFencing, wg *sync.Wai defer wg.Done() mountUnmountPVCOnPod(s.kh, podName, pvcName, "true", "create") require.True(s.T(), s.kh.IsPodRunning(podName, defaultNamespace)) - read, rErr := s.testClient.BlockClient.Read(podName, "/tmp/rook1", "longhaul", "default") + read, rErr := s.testClient.BlockClient.Read(podName, "longhaul", "default") require.Nil(s.T(), rErr) assert.Contains(s.T(), read, "this is long running test") mountUnmountPVCOnPod(s.kh, podName, pvcName, "true", "delete") diff --git a/tests/longhaul/object_test.go b/tests/longhaul/object_test.go index 406a54174..1b9fcbe2b 100644 --- a/tests/longhaul/object_test.go +++ b/tests/longhaul/object_test.go @@ -22,10 +22,8 @@ import ( "testing" "github.com/rook/rook/tests/framework/clients" - "github.com/rook/rook/tests/framework/contracts" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -47,25 +45,23 @@ func TestObjectLongHaul(t *testing.T) { type ObjectLongHaulSuite struct { suite.Suite kh *utils.K8sHelper - installer *installer.InstallHelper + installer *installer.CephInstaller tc *clients.TestClient namespace string - op contracts.Setup + op installer.TestSuite } func (s *ObjectLongHaulSuite) SetupSuite() { - var err error s.namespace = "longhaul-ns" - s.op, s.kh, s.installer = StartBaseLoadTestOperations(s.T, s.namespace) - s.tc, err = clients.CreateTestClient(s.kh, s.namespace) - require.Nil(s.T(), err) + s.op, s.kh, s.installer = StartLoadTestCluster(s.T, s.namespace) + s.tc = clients.CreateTestClient(s.kh, s.installer.Manifests) } func (s *ObjectLongHaulSuite) TestObjectLonghaulRun() { var wg sync.WaitGroup storeName := "longhaulstore" - wg.Add(s.installer.Env.LoadVolumeNumber) - for i := 1; i <= s.installer.Env.LoadVolumeNumber; i++ { + wg.Add(installer.Env.LoadVolumeNumber) + for i := 1; i <= installer.Env.LoadVolumeNumber; i++ { if i == 1 { go ObjectStoreOperations(s, &wg, s.namespace, storeName+strconv.Itoa(i), false) } else { @@ -90,7 +86,7 @@ func ObjectStoreOperations(s *ObjectLongHaulSuite, wg *sync.WaitGroup, namespace s.T().Fail() } - performObjectStoreOperations(s.installer, s3, bucketName) + performObjectStoreOperations(s3, bucketName) if deleteStore { delOpts := metav1.DeleteOptions{} s.tc.ObjectClient.Delete(namespace, storeName)