tests: refactor for adding an upgrade test

Signed-off-by: travisn <tnielsen@redhat.com>
This commit is contained in:
travisn committed 2018-08-23 11:40:24 -06:00
1 parent e2fd1eb8e6
commit b2c039fc72
35 files changed
+1214 -1133

No files matched your search

+49 -37
View File
@@ -18,14 +18,21 @@ package clients
import (
"fmt"
"path"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
)
const (
TestMountPath = "/tmp/testrook"
)
//BlockOperation is wrapper for k8s rook block operations
type BlockOperation struct {
k8sClient *utils.K8sHelper
manifests installer.CephManifests
}
type BlockImage struct {
@@ -41,9 +48,9 @@ var (
readDataFromBlockPod = []string{"cat", "READ_DATA_CMD"}
)
// CreateK8BlockOperation - Constructor to create BlockOperation - client to perform rook Block operations on k8s
func CreateK8BlockOperation(k8shelp *utils.K8sHelper) *BlockOperation {
return &BlockOperation{k8sClient: k8shelp}
// CreateBlockOperation - Constructor to create BlockOperation - client to perform rook Block operations on k8s
func CreateBlockOperation(k8shelp *utils.K8sHelper, manifests installer.CephManifests) *BlockOperation {
return &BlockOperation{k8shelp, manifests}
}
// BlockCreate Function to create a Block using Rook
@@ -62,6 +69,24 @@ func (b *BlockOperation) Create(manifest string, size int) (string, error) {
}
func (b *BlockOperation) CreatePvc(claimName, storageClassName, mode string) (string, error) {
return b.k8sClient.ResourceOperation("create", b.manifests.GetBlockPvcDef(claimName, storageClassName, mode))
}
func (b *BlockOperation) CreateStorageClass(poolName, storageClassName, namespace string, varClusterName bool) (string, error) {
return b.k8sClient.ResourceOperation("create", b.manifests.GetBlockStorageClassDef(poolName, storageClassName, namespace, varClusterName))
}
func (b *BlockOperation) DeletePvc(claimName, storageClassName, mode string) error {
_, err := b.k8sClient.ResourceOperation("delete", b.manifests.GetBlockPvcDef(claimName, storageClassName, mode))
return err
}
func (b *BlockOperation) DeleteStorageClass(poolName, storageClassName, namespace string) error {
_, err := b.k8sClient.ResourceOperation("delete", b.manifests.GetBlockStorageClassDef(poolName, storageClassName, namespace, false))
return err
}
// BlockDelete Function to delete a Block using Rook
// Input parameters -
// manifest - pod definition where pvc is described - delete is run on the the yaml definition
@@ -117,9 +142,8 @@ func (b *BlockOperation) DeleteBlockImage(image BlockImage, namespace string) er
// BlockMap Function to map a Block using Rook
// Input parameters -
// manifest - Pod definition - pod should be defined to use a pvc that was created earlier
// mountpath - not used in this impl since mountpath is defined in the pod definition
// Output - k8s create pod operation output and/or error
func (b *BlockOperation) BlockMap(manifest string, mountpath string) (string, error) {
func (b *BlockOperation) BlockMap(manifest string) (string, error) {
args := []string{"create", "-f", "-"}
result, err := b.k8sClient.KubectlWithStdin(manifest, args...)
if err != nil {
@@ -133,13 +157,26 @@ func (b *BlockOperation) BlockMap(manifest string, mountpath string) (string, er
// Write Function to write data to block created by rook ,i.e. write data to a pod that is using a pvc
// Input parameters -
// manifest - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier
// mountpath - folder on the pod were data is supposed to be written(should match the mountpath described in the pod definition)
// data - data to be written
// filename - file where data is written to
// namespace - optional param - namespace of the pod
// Output - k8s exec pod operation output and/or error
func (b *BlockOperation) Write(name string, mountpath string, data string, filename string, namespace string) (string, error) {
wt := "echo \"" + data + "\">" + mountpath + "/" + filename
func (b *BlockOperation) Write(name string, data string, filename string, namespace string) (string, error) {
return writeToPod(b.k8sClient, name, data, filename, namespace)
}
// Read Function to read from block created by rook ,i.e. Read data from a pod that is using a pvc
// Input parameters -
// manifest - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier
// filename - file to be read
// namespace - optional param - namespace of the pod
// Output - k8s exec pod operation output and/or error
func (b *BlockOperation) Read(name string, filename string, namespace string) (string, error) {
return readFromPod(b.k8sClient, name, filename, namespace)
}
func writeToPod(k8sh *utils.K8sHelper, name string, data string, filename string, namespace string) (string, error) {
wt := "echo \"" + data + "\">" + path.Join(TestMountPath, filename)
args := []string{"exec", name}
if namespace != "" {
@@ -147,24 +184,16 @@ func (b *BlockOperation) Write(name string, mountpath string, data string, filen
}
args = append(args, "--", "sh", "-c", wt)
result, err := b.k8sClient.Kubectl(args...)
result, err := k8sh.Kubectl(args...)
if err != nil {
return "", fmt.Errorf("Unable to write data to pod --: %s", err)
}
return result, nil
}
// Read Function to read from block created by rook ,i.e. Read data from a pod that is using a pvc
// Input parameters -
// manifest - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier
// mountpath - folder on the pod were data is supposed to be written(should match the mountpath described in the pod definition)
// filename - file to be read
// namespace - optional param - namespace of the pod
// Output - k8s exec pod operation output and/or error
func (b *BlockOperation) Read(name string, mountpath string, filename string, namespace string) (string, error) {
rd := mountpath + "/" + filename
func readFromPod(k8sh *utils.K8sHelper, name string, filename string, namespace string) (string, error) {
rd := path.Join(TestMountPath, filename)
args := []string{"exec", name}
if namespace != "" {
@@ -172,27 +201,10 @@ func (b *BlockOperation) Read(name string, mountpath string, filename string, na
}
args = append(args, "--", "cat", rd)
result, err := b.k8sClient.Kubectl(args...)
result, err := k8sh.Kubectl(args...)
if err != nil {
return "", fmt.Errorf("Unable to read data to pod -- : %s", err)
}
return result, nil
}
// BlockUnmap Function to unmap a Block using Rook
// Input parameters -
// manifest - pod definition - the pod described in yaml file is deleted
// mountpath - not used in this impl since mountpath is defined in the pod definition
// Output - k8s delete pod operation output and/or error
func (b *BlockOperation) BlockUnmap(name string, mountpath string) (string, error) {
args := []string{"delete", "-f", "-"}
result, err := b.k8sClient.KubectlWithStdin(name, args...)
if err != nil {
return "", fmt.Errorf("Unable to unmap block -- : %s", err)
}
return result, nil
}
+11 -84
View File
@@ -20,6 +20,7 @@ import (
"fmt"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -27,12 +28,13 @@ import (
//FilesystemOperation is a wrapper for k8s rook file operations
type FilesystemOperation struct {
k8sh *utils.K8sHelper
k8sh *utils.K8sHelper
manifests installer.CephManifests
}
// CreateK8sFilesystemOperation Constructor to create FilesystemOperation - client to perform rook file system operations on k8s
func CreateK8sFilesystemOperation(k8sh *utils.K8sHelper) *FilesystemOperation {
return &FilesystemOperation{k8sh: k8sh}
// CreateFilesystemOperation Constructor to create FilesystemOperation - client to perform rook file system operations on k8s
func CreateFilesystemOperation(k8sh *utils.K8sHelper, manifests installer.CephManifests) *FilesystemOperation {
return &FilesystemOperation{k8sh, manifests}
}
// FSCreate Function to create a filesystem in rook
@@ -42,24 +44,7 @@ func CreateK8sFilesystemOperation(k8sh *utils.K8sHelper) *FilesystemOperation {
func (f *FilesystemOperation) Create(name, namespace string) error {
logger.Infof("creating the filesystem via CRD")
fsSpec := fmt.Sprintf(`apiVersion: ceph.rook.io/v1beta1
kind: Filesystem
metadata:
name: %s
namespace: %s
spec:
metadataPool:
replicated:
size: 1
dataPools:
- replicated:
size: 1
metadataServer:
activeCount: 1
activeStandby: true
`, name, namespace)
if _, err := f.k8sh.ResourceOperation("create", fsSpec); err != nil {
if _, err := f.k8sh.ResourceOperation("create", f.manifests.GetFilesystem(namespace, name)); err != nil {
return err
}
@@ -93,21 +78,6 @@ func (f *FilesystemOperation) List(namespace string) ([]client.CephFilesystem, e
return filesystems, nil
}
// Mount Function to Mount a file system created by rook(on a pod)
// Input parameters -
// name - path to the yaml definition file - definition of pod to be created that mounts existing file system
// path - ignored in this case - mount path is defined in the path definition
// output - output returned by k8s create pod operaton and/or error
func (f *FilesystemOperation) Mount(name string, path string) (string, error) {
result, err := f.k8sh.Kubectl("create", "-f", "name")
if err != nil {
return "", fmt.Errorf("Unable to mount Filesystem -- : %s", err)
}
return result, nil
}
// Write Function to write data to file system created by rook ,i.e. write data to a pod that has filesystem mounted
// Input parameters -
// name - path to a yaml file that creates a pod - pod should be defined to use a pvc that was created earlier
@@ -116,22 +86,8 @@ func (f *FilesystemOperation) Mount(name string, path string) (string, error) {
// filename - file where data is written to
// namespace - optional param - namespace of the pod
// Output - k8s exec pod operation output and/or error
func (f *FilesystemOperation) Write(name string, mountpath string, data string, filename string, namespace string) (string, error) {
wt := "echo \"" + data + "\">" + mountpath + "/" + filename
args := []string{"exec", name}
if namespace != "" {
args = append(args, "-n", namespace)
}
args = append(args, "--", "sh", "-c", wt)
result, err := f.k8sh.Kubectl(args...)
if err != nil {
return "", fmt.Errorf("Unable to write data to pod -- : %s", err)
}
return result, nil
func (f *FilesystemOperation) Write(name string, data string, filename string, namespace string) (string, error) {
return writeToPod(f.k8sh, name, data, filename, namespace)
}
// Read Function to write read from file system created by rook ,i.e. Read data from a pod that filesystem mounted
@@ -141,35 +97,6 @@ func (f *FilesystemOperation) Write(name string, mountpath string, data string,
// filename - file to be read
// namespace - optional param - namespace of the pod
// Output - k8s exec pod operation output and/or error
func (f *FilesystemOperation) Read(name string, mountpath string, filename string, namespace string) (string, error) {
rd := mountpath + "/" + filename
args := []string{"exec", name}
if namespace != "" {
args = append(args, "-n", namespace)
}
args = append(args, "--", "cat", rd)
result, err := f.k8sh.Kubectl(args...)
if err != nil {
return "", fmt.Errorf("Unable to write data to pod -- : %s", err)
}
return result, nil
}
// Unmount Function to UnMount a file system created by rook(delete pod)
// Input parameters -
// name - path to the yaml definition file - definition of pod to be deleted that has a file system mounted
// path - ignored in this case - mount path is defined in the path definition
// output - output returned by k8s delete pod operaton and/or error
func (f *FilesystemOperation) Unmount(name string) (string, error) {
result, err := f.k8sh.Kubectl("delete", "-f", name)
if err != nil {
return "", fmt.Errorf("Unable to unmount Filesystem -- : %s", err)
}
return result, nil
func (f *FilesystemOperation) Read(name string, filename string, namespace string) (string, error) {
return readFromPod(f.k8sh, name, filename, namespace)
}
+12 -31
View File
@@ -20,6 +20,7 @@ import (
"fmt"
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
)
@@ -29,60 +30,40 @@ var logger = capnslog.NewPackageLogger("github.com/rook/rook/tests", "clients")
// ObjectOperation is wrapper for k8s rook object operations
type ObjectOperation struct {
k8sh *utils.K8sHelper
k8sh *utils.K8sHelper
manifests installer.CephManifests
}
// CreateObjectOperation creates new rook object client
func CreateObjectOperation(k8sh *utils.K8sHelper) *ObjectOperation {
return &ObjectOperation{k8sh: k8sh}
func CreateObjectOperation(k8sh *utils.K8sHelper, manifests installer.CephManifests) *ObjectOperation {
return &ObjectOperation{k8sh, manifests}
}
// ObjectCreate Function to create a object store in rook
func (ro *ObjectOperation) Create(namespace, storeName string, replicaCount int32) error {
func (o *ObjectOperation) Create(namespace, storeName string, replicaCount int32) error {
logger.Infof("creating the object store via CRD")
storeSpec := fmt.Sprintf(`apiVersion: ceph.rook.io/v1beta1
kind: ObjectStore
metadata:
name: %s
namespace: %s
spec:
metadataPool:
replicated:
size: 1
dataPool:
replicated:
size: 1
gateway:
type: s3
sslCertificateRef:
port: %d
securePort:
instances: %d
allNodes: false
`, storeName, namespace, rgwPort, replicaCount)
if _, err := ro.k8sh.ResourceOperation("create", storeSpec); err != nil {
if _, err := o.k8sh.ResourceOperation("create", o.manifests.GetObjectStore(namespace, storeName, int(replicaCount), rgwPort)); err != nil {
return err
}
err := ro.k8sh.WaitForLabeledPodToRun(fmt.Sprintf("rook_object_store=%s", storeName), namespace)
err := o.k8sh.WaitForLabeledPodToRun(fmt.Sprintf("rook_object_store=%s", storeName), namespace)
if err != nil {
return fmt.Errorf("rgw did not start via crd. %+v", err)
}
// create the external service
return ro.k8sh.CreateExternalRGWService(namespace, storeName)
return o.k8sh.CreateExternalRGWService(namespace, storeName)
}
func (ro *ObjectOperation) Delete(namespace, storeName string) error {
func (o *ObjectOperation) Delete(namespace, storeName string) error {
logger.Infof("Deleting the object store via CRD")
if _, err := ro.k8sh.DeleteResource("-n", namespace, "ObjectStore", storeName); err != nil {
if _, err := o.k8sh.DeleteResource("-n", namespace, "ObjectStore", storeName); err != nil {
return err
}
if !ro.k8sh.WaitUntilPodWithLabelDeleted(fmt.Sprintf("rook_object_store=%s", storeName), namespace) {
if !o.k8sh.WaitUntilPodWithLabelDeleted(fmt.Sprintf("rook_object_store=%s", storeName), namespace) {
return fmt.Errorf("rgw did not stop via crd")
}
return nil
+33 -10
View File
@@ -22,7 +22,6 @@ import (
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/daemon/ceph/model"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"k8s.io/apimachinery/pkg/api/errors"
@@ -31,24 +30,30 @@ import (
//PoolOperation is a wrapper for rook pool operations
type PoolOperation struct {
k8sh *utils.K8sHelper
k8sh *utils.K8sHelper
manifests installer.CephManifests
}
// CreatePoolOperation creates a new pool client
func CreatePoolOperation(k8sh *utils.K8sHelper) *PoolOperation {
return &PoolOperation{k8sh: k8sh}
func CreatePoolOperation(k8sh *utils.K8sHelper, manifests installer.CephManifests) *PoolOperation {
return &PoolOperation{k8sh, manifests}
}
func (p *PoolOperation) Create(pool model.Pool, namespace string) (string, error) {
return p.createOrUpdatePool(pool, namespace, "create")
func (p *PoolOperation) Create(name, namespace string, replicas int) (string, error) {
return p.createOrUpdatePool(name, namespace, "create", replicas)
}
func (p *PoolOperation) Update(pool model.Pool, namespace string) (string, error) {
return p.createOrUpdatePool(pool, namespace, "apply")
func (p *PoolOperation) Delete(name string, namespace string) error {
_, err := p.k8sh.ResourceOperation("delete", p.manifests.GetBlockPoolDef(name, namespace, "1"))
return err
}
func (p *PoolOperation) createOrUpdatePool(pool model.Pool, namespace, action string) (string, error) {
return installer.BlockResourceOperation(p.k8sh, installer.GetBlockPoolDef(pool.Name, namespace, strconv.Itoa(int(pool.ReplicatedConfig.Size))), action)
func (p *PoolOperation) Update(name, namespace string, replicas int) (string, error) {
return p.createOrUpdatePool(name, namespace, "apply", replicas)
}
func (p *PoolOperation) createOrUpdatePool(name, namespace, action string, replicas int) (string, error) {
return p.k8sh.ResourceOperation(action, p.manifests.GetBlockPoolDef(name, namespace, strconv.Itoa(replicas)))
}
func (p *PoolOperation) ListCephPools(namespace string) ([]client.CephStoragePoolSummary, error) {
@@ -104,3 +109,21 @@ func (p *PoolOperation) CephPoolExists(namespace, name string) (bool, error) {
}
return false, nil
}
func (p *PoolOperation) CreateStorageClassAndPvc(namespace, poolName, storageClassName, blockName, mode string) (string, error) {
return p.k8sh.ResourceOperation("create", p.manifests.GetBlockPoolStorageClassAndPvcDef(namespace, poolName, storageClassName, blockName, mode))
}
func (p *PoolOperation) DeleteStorageClass(namespace, poolName, storageClassName string) error {
_, err := p.k8sh.ResourceOperation("delete", p.manifests.GetBlockPoolStorageClass(namespace, poolName, storageClassName))
return err
}
func (p *PoolOperation) DeleteStorageClassAndPvc(namespace, poolName, storageClassName, blockName, mode string) error {
_, err := p.k8sh.ResourceOperation("delete", p.manifests.GetBlockPoolStorageClassAndPvcDef(namespace, poolName, storageClassName, blockName, mode))
return err
}
func (p *PoolOperation) CreateStorageClass(namespace, poolName, storageClassName string) (string, error) {
return p.k8sh.ResourceOperation("create", p.manifests.GetBlockPoolStorageClass(namespace, poolName, storageClassName))
}
+7 -6
View File
@@ -20,6 +20,7 @@ import (
"fmt"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
)
@@ -41,15 +42,15 @@ const (
)
//CreateTestClient creates new instance of test client for a platform
func CreateTestClient(k8sHelper *utils.K8sHelper, namespace string) (*TestClient, error) {
func CreateTestClient(k8sHelper *utils.K8sHelper, manifests installer.CephManifests) *TestClient {
return &TestClient{
CreateK8BlockOperation(k8sHelper),
CreateK8sFilesystemOperation(k8sHelper),
CreateObjectOperation(k8sHelper),
CreatePoolOperation(k8sHelper),
CreateBlockOperation(k8sHelper, manifests),
CreateFilesystemOperation(k8sHelper, manifests),
CreateObjectOperation(k8sHelper, manifests),
CreatePoolOperation(k8sHelper, manifests),
k8sHelper,
}, nil
}
}
//Status returns rook status details
-22
View File
@@ -1,22 +0,0 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package contracts
type Setup interface {
SetUp()
TearDown()
}
@@ -22,24 +22,17 @@ import (
"math/rand"
"os"
"path"
"runtime"
"strings"
"testing"
"time"
"k8s.io/api/core/v1"
"k8s.io/kubernetes/pkg/kubelet/apis"
"flag"
"github.com/coreos/pkg/capnslog"
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
"github.com/rook/rook/pkg/util/exec"
"github.com/rook/rook/pkg/util/sys"
"github.com/rook/rook/tests/framework/objects"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
"k8s.io/apimachinery/pkg/api/errors"
@@ -49,71 +42,43 @@ import (
const (
rookOperatorCreatedCrd = "clusters.ceph.rook.io"
cockroachDBCRD = "clusters.cockroachdb.rook.io"
nfsServerCRD = "nfsservers.nfs.rook.io"
helmChartName = "local/rook-ceph"
helmDeployName = "rook-ceph"
)
var (
// ** Variables that might need to be changed depending on the dev environment. The init function below will modify some of them automatically. **
baseTestDir string
forceUseDevices = false
createBaseTestDir = true
// ** end of Variables to modify
logger = capnslog.NewPackageLogger("github.com/rook/rook", "installer")
createArgs = []string{"create", "-f"}
createFromStdinArgs = append(createArgs, "-")
deleteArgs = []string{"delete", "-f"}
deleteFromStdinArgs = append(deleteArgs, "-")
)
func init() {
// this default will only work if running kubernetes on the local machine
baseTestDir, _ = os.Getwd()
// The following settings could apply to any environment when the kube context is running on the host and the tests are running inside a
// VM such as minikube. This is a cheap test for this condition, we need to find a better way to automate these settings.
if runtime.GOOS == "darwin" {
createBaseTestDir = false
baseTestDir = "/data"
}
}
//InstallHelper wraps installing and uninstalling rook on a platform
type InstallHelper struct {
//CephInstaller wraps installing and uninstalling rook on a platform
type CephInstaller struct {
Manifests CephManifests
k8shelper *utils.K8sHelper
installData *InstallData
hostPathToDelete string
helmHelper *utils.HelmHelper
Env objects.EnvironmentManifest
k8sVersion string
T func() *testing.T
}
func (h *InstallHelper) CreateK8sRookResources() (err error) {
func (h *CephInstaller) CreateCephCRDs() (err error) {
var resources string
logger.Info("Creating Rook CRDs")
resources = h.installData.GetRookCRDs()
resources = h.Manifests.GetRookCRDs()
_, err = h.k8shelper.KubectlWithStdin(resources, createFromStdinArgs...)
return
}
//CreateK8sRookOperator creates rook-operator via kubectl
func (h *InstallHelper) CreateK8sRookOperator(namespace string) (err error) {
//CreateCephOperator creates rook-operator via kubectl
func (h *CephInstaller) CreateCephOperator(namespace string) (err error) {
logger.Infof("Starting Rook Operator")
//creating clusterrolebinding for kubeadm env.
h.k8shelper.CreateAnonSystemClusterBinding()
//creating rook resources
if err = h.CreateK8sRookResources(); err != nil {
if err = h.CreateCephCRDs(); err != nil {
return err
}
rookOperator := h.installData.GetRookOperator(namespace)
rookOperator := h.Manifests.GetRookOperator(namespace)
_, err = h.k8shelper.KubectlWithStdin(rookOperator, createFromStdinArgs...)
if err != nil {
@@ -130,7 +95,7 @@ func (h *InstallHelper) CreateK8sRookOperator(namespace string) (err error) {
}
//CreateK8sRookOperatorViaHelm creates rook operator via Helm chart named local/rook present in local repo
func (h *InstallHelper) CreateK8sRookOperatorViaHelm(namespace string) error {
func (h *CephInstaller) CreateK8sRookOperatorViaHelm(namespace string) error {
//creating clusterrolebinding for kubeadm env.
h.k8shelper.CreateAnonSystemClusterBinding()
@@ -154,10 +119,10 @@ func (h *InstallHelper) CreateK8sRookOperatorViaHelm(namespace string) error {
}
//CreateK8sRookToolbox creates rook-ceph-tools via kubectl
func (h *InstallHelper) CreateK8sRookToolbox(namespace string) (err error) {
func (h *CephInstaller) CreateK8sRookToolbox(namespace string) (err error) {
logger.Infof("Starting Rook toolbox")
rookToolbox := h.installData.GetRookToolBox(namespace)
rookToolbox := h.Manifests.GetRookToolBox(namespace)
_, err = h.k8shelper.KubectlWithStdin(rookToolbox, createFromStdinArgs...)
@@ -173,13 +138,13 @@ func (h *InstallHelper) CreateK8sRookToolbox(namespace string) (err error) {
return nil
}
func (h *InstallHelper) CreateK8sRookCluster(namespace, systemNamespace string, storeType string) (err error) {
func (h *CephInstaller) CreateK8sRookCluster(namespace, systemNamespace string, storeType string) (err error) {
return h.CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType, false,
cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, true /* startWithAllNodes */)
}
//CreateK8sRookCluster creates rook cluster via kubectl
func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType string,
func (h *CephInstaller) CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType string,
useAllDevices bool, mon cephv1beta1.MonSpec, startWithAllNodes bool) error {
dataDirHostPath, err := h.initTestDir(namespace)
@@ -197,7 +162,7 @@ func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, sy
}
logger.Infof("Creating cluster roles")
roles := h.installData.GetClusterRoles(namespace, systemNamespace)
roles := h.Manifests.GetClusterRoles(namespace, systemNamespace)
if _, err := h.k8shelper.KubectlWithStdin(roles, createFromStdinArgs...); err != nil {
return fmt.Errorf("Failed to create cluster roles. %+v", err)
}
@@ -262,7 +227,7 @@ func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, sy
}
} else {
logger.Infof("Starting Rook Cluster with yaml")
rookCluster := h.installData.GetRookCluster(namespace, storeType, dataDirHostPath, useAllDevices, mon.Count)
rookCluster := h.Manifests.GetRookCluster(&ClusterSettings{namespace, storeType, dataDirHostPath, useAllDevices, mon.Count})
if _, err := h.k8shelper.KubectlWithStdin(rookCluster, createFromStdinArgs...); err != nil {
return fmt.Errorf("Failed to create rook cluster : %v ", err)
}
@@ -281,7 +246,7 @@ func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, sy
return err
}
func (h *InstallHelper) initTestDir(namespace string) (string, error) {
func (h *CephInstaller) initTestDir(namespace string) (string, error) {
h.hostPathToDelete = path.Join(baseTestDir, "rook-test")
testDir := path.Join(h.hostPathToDelete, namespace)
@@ -303,7 +268,7 @@ func (h *InstallHelper) initTestDir(namespace string) (string, error) {
return testDir, nil
}
func (h *InstallHelper) GetNodeHostnames() ([]string, error) {
func (h *CephInstaller) GetNodeHostnames() ([]string, error) {
nodes, err := h.k8shelper.Clientset.CoreV1().Nodes().List(metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("failed to get k8s nodes. %+v", err)
@@ -316,17 +281,13 @@ func (h *InstallHelper) GetNodeHostnames() ([]string, error) {
return names, nil
}
func SystemNamespace(namespace string) string {
return fmt.Sprintf("%s-system", namespace)
}
//InstallRookOnK8sWithHostPathAndDevices installs rook on k8s
func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeType string,
func (h *CephInstaller) InstallRookOnK8sWithHostPathAndDevices(namespace, storeType string,
helmInstalled, useDevices bool, mon cephv1beta1.MonSpec, startWithAllNodes bool) (bool, error) {
var err error
//flag used for local debuggin purpose, when rook is pre-installed
if h.Env.SkipInstallRook {
if Env.SkipInstallRook {
return true, nil
}
@@ -346,7 +307,7 @@ func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeT
} else {
onamespace = SystemNamespace(namespace)
err := h.CreateK8sRookOperator(onamespace)
err := h.CreateCephOperator(onamespace)
if err != nil {
logger.Errorf("Rook Operator not installed ,error -> %v", err)
return false, err
@@ -354,7 +315,7 @@ func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeT
}
if !h.k8shelper.IsPodInExpectedState("rook-ceph-operator", onamespace, "Running") {
logger.Error("rook-ceph-operator is not running")
h.k8shelper.GetRookLogs("rook-ceph-operator", h.Env.HostType, onamespace, "test-setup")
h.k8shelper.GetRookLogs("rook-ceph-operator", Env.HostType, onamespace, "test-setup")
logger.Error("rook-ceph-operator is not Running, abort!")
return false, err
}
@@ -387,14 +348,14 @@ func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeT
}
//UninstallRookFromK8s uninstalls rook from k8s
func (h *InstallHelper) UninstallRook(helmInstalled bool, namespace string) {
func (h *CephInstaller) UninstallRook(helmInstalled bool, namespace string) {
h.UninstallRookFromMultipleNS(helmInstalled, SystemNamespace(namespace), namespace)
}
//UninstallRookFromK8s uninstalls rook from multiple namespaces in k8s
func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNamespace string, namespaces ...string) {
func (h *CephInstaller) UninstallRookFromMultipleNS(helmInstalled bool, systemNamespace string, namespaces ...string) {
//flag used for local debugging purpose, when rook is pre-installed
if h.Env.SkipInstallRook {
if Env.SkipInstallRook {
return
}
@@ -404,43 +365,43 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa
if !h.k8shelper.VersionAtLeast("v1.8.0") {
_, err = h.k8shelper.DeleteResource("-n", namespace, "serviceaccount", "rook-ceph-cluster")
h.checkError(err, "cannot remove serviceaccount rook-ceph-cluster")
checkError(h.T(), err, "cannot remove serviceaccount rook-ceph-cluster")
assert.NoError(h.T(), err, "%s err -> %v", namespace, err)
err = h.k8shelper.DeleteRoleAndBindings("rook-ceph-cluster", namespace)
h.checkError(err, "rook-ceph-cluster cluster role and binding cannot be deleted")
checkError(h.T(), err, "rook-ceph-cluster cluster role and binding cannot be deleted")
assert.NoError(h.T(), err, "rook-ceph-cluster cluster role and binding cannot be deleted: %+v", err)
err = h.k8shelper.DeleteRoleBinding("rook-ceph-cluster-mgmt", namespace)
h.checkError(err, "rook-ceph-cluster-mgmt binding cannot be deleted")
checkError(h.T(), err, "rook-ceph-cluster-mgmt binding cannot be deleted")
assert.NoError(h.T(), err, "rook-ceph-cluster-mgmt binding cannot be deleted: %+v", err)
}
_, err = h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cluster.ceph.rook.io", namespace)
h.checkError(err, fmt.Sprintf("cannot remove cluster %s", namespace))
checkError(h.T(), err, fmt.Sprintf("cannot remove cluster %s", namespace))
crdCheckerFunc := func() error {
_, err := h.k8shelper.RookClientset.RookV1alpha1().Clusters(namespace).Get(namespace, metav1.GetOptions{})
return err
}
err = h.waitForCustomResourceDeletion(namespace, crdCheckerFunc)
h.checkError(err, fmt.Sprintf("failed to wait for crd %s deletion", namespace))
err = h.k8shelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc)
checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace))
_, err = h.k8shelper.DeleteResourceAndWait(false, "namespace", namespace)
h.checkError(err, fmt.Sprintf("cannot delete namespace %s", namespace))
checkError(h.T(), err, fmt.Sprintf("cannot delete namespace %s", namespace))
}
logger.Infof("removing the operator from namespace %s", systemNamespace)
_, err = h.k8shelper.DeleteResource("crd", "clusters.ceph.rook.io", "pools.ceph.rook.io", "objectstores.ceph.rook.io", "filesystems.ceph.rook.io", "volumes.rook.io")
h.checkError(err, "cannot delete CRDs")
checkError(h.T(), err, "cannot delete CRDs")
if helmInstalled {
err = h.helmHelper.DeleteLocalRookHelmChart(helmDeployName)
} else {
rookOperator := h.installData.GetRookOperator(systemNamespace)
rookOperator := h.Manifests.GetRookOperator(systemNamespace)
_, err = h.k8shelper.KubectlWithStdin(rookOperator, deleteFromStdinArgs...)
}
h.checkError(err, "cannot uninstall rook-operator")
checkError(h.T(), err, "cannot uninstall rook-operator")
h.k8shelper.Clientset.RbacV1beta1().RoleBindings(systemNamespace).Delete("rook-ceph-system", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-ceph-global", nil)
@@ -454,7 +415,7 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa
// removing data dir if exists
if h.hostPathToDelete != "" {
nodes, err := h.GetNodeHostnames()
h.checkError(err, "cannot get node names")
checkError(h.T(), err, "cannot get node names")
for _, node := range nodes {
err = h.cleanupDir(node, h.hostPathToDelete)
logger.Infof("removing %s from node %s. err=%v", h.hostPathToDelete, node, err)
@@ -462,67 +423,27 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa
}
}
func (h *InstallHelper) cleanupDir(node, dir string) error {
resources := h.installData.GetCleanupPod(node, dir)
func (h *CephInstaller) cleanupDir(node, dir string) error {
resources := h.Manifests.GetCleanupPod(node, dir)
_, err := h.k8shelper.KubectlWithStdin(resources, createFromStdinArgs...)
return err
}
func (h *InstallHelper) checkError(err error, message string) {
// During cleanup the resource might not be found because the test might have failed before the test was done and never created the resource
if err == nil || errors.IsNotFound(err) {
return
}
assert.NoError(h.T(), err, "%s. %+v", message, err)
}
func (h *InstallHelper) waitForCustomResourceDeletion(namespace string, checkerFunc func() error) error {
if !h.k8shelper.VersionAtLeast("v1.8.0") {
// v1.7 has an intermittent issue with long delay to delete resources so we will skip waiting
return nil
}
// wait for the operator to finalize and delete the CRD
for i := 0; i < 10; i++ {
err := checkerFunc()
if err == nil {
logger.Infof("custom resource %s still exists", namespace)
time.Sleep(2 * time.Second)
continue
}
if errors.IsNotFound(err) {
logger.Infof("custom resource %s deleted", namespace)
return nil
}
return err
}
logger.Errorf("gave up deleting custom resource %s", namespace)
return nil
}
func (h *InstallHelper) GatherAllRookLogs(namespace, systemNamespace string, testName string) {
func (h *CephInstaller) GatherAllRookLogs(namespace, systemNamespace string, testName string) {
logger.Infof("Gathering all logs from Rook Cluster %s", namespace)
h.k8shelper.GetRookLogs("rook-ceph-operator", h.Env.HostType, systemNamespace, testName)
h.k8shelper.GetRookLogs("rook-ceph-agent", h.Env.HostType, systemNamespace, testName)
h.k8shelper.GetRookLogs("rook-discover", h.Env.HostType, systemNamespace, testName)
h.k8shelper.GetRookLogs("rook-ceph-mgr", h.Env.HostType, namespace, testName)
h.k8shelper.GetRookLogs("rook-ceph-mon", h.Env.HostType, namespace, testName)
h.k8shelper.GetRookLogs("rook-ceph-osd", h.Env.HostType, namespace, testName)
h.k8shelper.GetRookLogs("rook-ceph-osd-prepare", h.Env.HostType, namespace, testName)
h.k8shelper.GetRookLogs("rook-ceph-rgw", h.Env.HostType, namespace, testName)
h.k8shelper.GetRookLogs("rook-ceph-mds", h.Env.HostType, namespace, testName)
h.k8shelper.GetRookLogs("rook-ceph-operator", Env.HostType, systemNamespace, testName)
h.k8shelper.GetRookLogs("rook-ceph-agent", Env.HostType, systemNamespace, testName)
h.k8shelper.GetRookLogs("rook-discover", Env.HostType, systemNamespace, testName)
h.k8shelper.GetRookLogs("rook-ceph-mgr", Env.HostType, namespace, testName)
h.k8shelper.GetRookLogs("rook-ceph-mon", Env.HostType, namespace, testName)
h.k8shelper.GetRookLogs("rook-ceph-osd", Env.HostType, namespace, testName)
h.k8shelper.GetRookLogs("rook-ceph-osd-prepare", Env.HostType, namespace, testName)
h.k8shelper.GetRookLogs("rook-ceph-rgw", Env.HostType, namespace, testName)
h.k8shelper.GetRookLogs("rook-ceph-mds", Env.HostType, namespace, testName)
}
// GatherCRDObjectDebuggingInfo gathers all the descriptions for pods, pvs and pvcs
func (h *InstallHelper) GatherCRDObjectDebuggingInfo(namespace string) {
h.k8shelper.PrintPodDescribeForNamespace(namespace)
h.k8shelper.PrintPVs(true /*detailed*/)
h.k8shelper.PrintPVCs(namespace, true /*detailed*/)
h.k8shelper.PrintStorageClasses(true /*detailed*/)
}
//NewK8sRookhelper creates new instance of InstallHelper
func NewK8sRookhelper(clientset *kubernetes.Clientset, t func() *testing.T) *InstallHelper {
// NewCephInstaller creates new instance of CephInstaller
func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, rookVersion string) *CephInstaller {
// All e2e tests should run ceph commands in the toolbox since we are not inside a container
client.RunAllCephCommandsInToolbox = true
@@ -536,58 +457,13 @@ func NewK8sRookhelper(clientset *kubernetes.Clientset, t func() *testing.T) *Ins
if err != nil {
panic("failed to get kubectl client :" + err.Error())
}
ih := &InstallHelper{
k8shelper: k8shelp,
installData: NewK8sInstallData(),
helmHelper: utils.NewHelmHelper(),
Env: objects.Env,
k8sVersion: version.String(),
T: t,
h := &CephInstaller{
Manifests: NewCephManifests(rookVersion),
k8shelper: k8shelp,
helmHelper: utils.NewHelmHelper(),
k8sVersion: version.String(),
T: t,
}
flag.Parse()
return ih
}
func IsAdditionalDeviceAvailableOnCluster() bool {
executor := &exec.CommandExecutor{}
devices, err := sys.ListDevices(executor)
if err != nil {
return false
}
disks := 0
logger.Infof("devices : %v", devices)
for _, device := range devices {
if strings.Contains(device, "loop") {
continue
}
props, _ := sys.GetDeviceProperties(device, executor)
if props["TYPE"] != "disk" {
continue
}
ownPartitions, fs, err := sys.CheckIfDeviceAvailable(executor, device)
if err != nil {
logger.Warningf("failed to detect device %s availability. %+v", device, err)
continue
}
if !ownPartitions {
logger.Infof("skipping device %s since don't own partitions", device)
continue
}
if fs != "" {
logger.Infof("skipping device %s since it has file system %s", device, fs)
continue
}
if strings.HasPrefix(device, "rbd") {
logger.Infof("skipping unexpected rbd device: %s", device)
continue
}
logger.Infof("available device: %s", device)
disks++
}
if disks > 0 {
return true
}
logger.Info("No additional disks found on cluster")
return false
return h
}
@@ -17,21 +17,51 @@ limitations under the License.
package installer
import (
"fmt"
"strconv"
"github.com/google/uuid"
)
//InstallData wraps rook yaml definitions
type InstallData struct {
type CephManifests interface {
GetRookCRDs() string
GetRookOperator(namespace string) string
GetClusterRoles(namespace, systemNamespace string) string
GetRookCluster(settings *ClusterSettings) string
GetRookToolBox(namespace string) string
GetCleanupPod(node, removalDir string) string
GetBlockPoolDef(poolName string, namespace string, replicaSize string) string
GetBlockStorageClassDef(poolName string, storageClassName string, namespace string, varClusterName bool) string
GetBlockPvcDef(claimName string, storageClassName string, accessModes string) string
GetBlockPoolStorageClassAndPvcDef(namespace string, poolName string, storageClassName string, blockName string, accessMode string) string
GetBlockPoolStorageClass(namespace string, poolName string, storageClassName string) string
GetFilesystem(namepace, name string) string
GetObjectStore(namespace, name string, replicaCount, port int) string
}
//NewK8sInstallData creates new instance of InstallData struct
func NewK8sInstallData() *InstallData {
return &InstallData{}
type ClusterSettings struct {
Namespace string
StoreType string
DataDirHostPath string
UseAllDevices bool
Mons int
}
func (i *InstallData) GetRookCRDs() string {
//CephManifestsMaster wraps rook yaml definitions
type CephManifestsMaster struct {
imageTag string
}
// NewCephManifests gets the manifest type depending on the Rook version desired
func NewCephManifests(version string) CephManifests {
switch version {
case VersionMaster:
return &CephManifestsMaster{imageTag: VersionMaster}
}
panic(fmt.Errorf("unrecognized ceph manifest version: %s", version))
}
func (m *CephManifestsMaster) GetRookCRDs() string {
return `apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
@@ -104,7 +134,7 @@ spec:
}
//GetRookOperator returns rook Operator manifest
func (i *InstallData) GetRookOperator(namespace string) string {
func (m *CephManifestsMaster) GetRookOperator(namespace string) string {
return `kind: Namespace
apiVersion: v1
@@ -307,7 +337,7 @@ spec:
serviceAccountName: rook-ceph-system
containers:
- name: rook-ceph-operator
image: rook/ceph:master
image: rook/ceph:` + m.imageTag + `
args: ["ceph", "operator"]
env:
- name: ROOK_LOG_LEVEL
@@ -331,7 +361,7 @@ spec:
}
//GetRookCluster returns rook-cluster manifest
func (i *InstallData) GetClusterRoles(namespace, systemNamespace string) string {
func (m *CephManifestsMaster) GetClusterRoles(namespace, systemNamespace string) string {
return `apiVersion: v1
kind: ServiceAccount
metadata:
@@ -380,36 +410,36 @@ subjects:
}
//GetRookCluster returns rook-cluster manifest
func (i *InstallData) GetRookCluster(namespace, storeType, dataDirHostPath string, useAllDevices bool, mons int) string {
func (m *CephManifestsMaster) GetRookCluster(settings *ClusterSettings) string {
return `apiVersion: ceph.rook.io/v1beta1
kind: Cluster
metadata:
name: ` + namespace + `
namespace: ` + namespace + `
name: ` + settings.Namespace + `
namespace: ` + settings.Namespace + `
spec:
serviceAccount: rook-ceph-cluster
dataDirHostPath: ` + dataDirHostPath + `
dataDirHostPath: ` + settings.DataDirHostPath + `
network:
hostNetwork: false
mon:
count: ` + strconv.Itoa(mons) + `
count: ` + strconv.Itoa(settings.Mons) + `
allowMultiplePerNode: true
dashboard:
enabled: true
metadataDevice:
storage:
useAllNodes: true
useAllDevices: ` + strconv.FormatBool(useAllDevices) + `
useAllDevices: ` + strconv.FormatBool(settings.UseAllDevices) + `
deviceFilter:
location:
config:
storeType: "` + storeType + `"
storeType: "` + settings.StoreType + `"
databaseSizeMB: "1024"
journalSizeMB: "1024"`
}
//GetRookToolBox returns rook-toolbox manifest
func (i *InstallData) GetRookToolBox(namespace string) string {
func (m *CephManifestsMaster) GetRookToolBox(namespace string) string {
return `apiVersion: v1
kind: Pod
metadata:
@@ -419,7 +449,7 @@ spec:
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: rook-ceph-tools
image: rook/ceph-toolbox:master
image: rook/ceph-toolbox:` + m.imageTag + `
imagePullPolicy: IfNotPresent
env:
- name: ROOK_ADMIN_SECRET
@@ -458,7 +488,7 @@ spec:
}
//GetCleanupPod gets a cleanup Pod manifest
func (i *InstallData) GetCleanupPod(node, removalDir string) string {
func (m *CephManifestsMaster) GetCleanupPod(node, removalDir string) string {
return `apiVersion: batch/v1
kind: Job
metadata:
@@ -469,7 +499,7 @@ spec:
restartPolicy: Never
containers:
- name: rook-cleaner
image: rook/rook:master
image: rook/ceph:` + m.imageTag + `
securityContext:
privileged: true
volumeMounts:
@@ -486,3 +516,94 @@ spec:
hostPath:
path: ` + removalDir
}
func (m *CephManifestsMaster) GetBlockPoolDef(poolName string, namespace string, replicaSize string) string {
return `apiVersion: ceph.rook.io/v1beta1
kind: Pool
metadata:
name: ` + poolName + `
namespace: ` + namespace + `
spec:
replicated:
size: ` + replicaSize
}
func (m *CephManifestsMaster) GetBlockStorageClassDef(poolName string, storageClassName string, namespace string, varClusterName bool) string {
namespaceParameter := "clusterNamespace"
if varClusterName {
namespaceParameter = "clusterName"
}
return `apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: ` + storageClassName + `
provisioner: ceph.rook.io/block
parameters:
pool: ` + poolName + `
` + namespaceParameter + `: ` + namespace
}
func (m *CephManifestsMaster) GetBlockPvcDef(claimName string, storageClassName string, accessModes string) string {
return `apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: ` + claimName + `
annotations:
volume.beta.kubernetes.io/storage-class: ` + storageClassName + `
spec:
accessModes:
- ` + accessModes + `
resources:
requests:
storage: 1M`
}
func (m *CephManifestsMaster) GetBlockPoolStorageClassAndPvcDef(namespace string, poolName string, storageClassName string, blockName string, accessMode string) string {
return concatYaml(m.GetBlockPoolDef(poolName, namespace, "1"),
concatYaml(m.GetBlockStorageClassDef(poolName, storageClassName, namespace, false), m.GetBlockPvcDef(blockName, storageClassName, accessMode)))
}
func (m *CephManifestsMaster) GetBlockPoolStorageClass(namespace string, poolName string, storageClassName string) string {
return concatYaml(m.GetBlockPoolDef(poolName, namespace, "1"), m.GetBlockStorageClassDef(poolName, storageClassName, namespace, false))
}
func (m *CephManifestsMaster) GetFilesystem(namespace, name string) string {
return `apiVersion: ceph.rook.io/v1beta1
kind: Filesystem
metadata:
name: ` + name + `
namespace: ` + namespace + `
spec:
metadataPool:
replicated:
size: 1
dataPools:
- replicated:
size: 1
metadataServer:
activeCount: 1
activeStandby: true`
}
func (m *CephManifestsMaster) GetObjectStore(namespace, name string, replicaCount, port int) string {
return `apiVersion: ceph.rook.io/v1beta1
kind: ObjectStore
metadata:
name: ` + name + `
namespace: ` + namespace + `
spec:
metadataPool:
replicated:
size: 1
dataPool:
replicated:
size: 1
gateway:
type: s3
sslCertificateRef:
port: ` + strconv.Itoa(port) + `
securePort:
instances: ` + strconv.Itoa(replicaCount) + `
allNodes: false
`
}
-270
View File
@@ -1,270 +0,0 @@
/*
Copyright 2018 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package installer
import (
"fmt"
"strconv"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func (h *InstallHelper) InstallCockroachDB(systemNamespace, namespace string, count int) error {
h.k8shelper.CreateAnonSystemClusterBinding()
// install hostpath provisioner if there isn't already a default storage class
defaultExists, err := h.k8shelper.IsDefaultStorageClassPresent()
if err != nil {
return err
} else if !defaultExists {
if err := h.InstallHostPathProvisioner(); err != nil {
return err
}
} else {
logger.Info("skipping install of host path provisioner because a default storage class already exists")
}
// install cockroachdb operator
if err := h.CreateCockroachDBOperator(systemNamespace); err != nil {
return err
}
// install cockroachdb cluster instance
if err := h.CreateCockroachDBCluster(namespace, count); err != nil {
return err
}
return nil
}
func (h *InstallHelper) CreateCockroachDBOperator(namespace string) error {
logger.Infof("starting cockroachDB operator")
logger.Info("creating cockroachDB CRDs")
if _, err := h.k8shelper.KubectlWithStdin(h.installData.GetCockroachDBCRDs(), createFromStdinArgs...); err != nil {
return err
}
cockroachDBOperator := h.installData.GetCockroachDBOperator(namespace)
_, err := h.k8shelper.KubectlWithStdin(cockroachDBOperator, createFromStdinArgs...)
if err != nil {
return fmt.Errorf("failed to create rook-cockroachdb-operator pod: %+v ", err)
}
if !h.k8shelper.IsCRDPresent(cockroachDBCRD) {
return fmt.Errorf("failed to find cockroachdb CRD %s", cockroachDBCRD)
}
if !h.k8shelper.IsPodInExpectedState("rook-cockroachdb-operator", namespace, "Running") {
return fmt.Errorf("rook-cockroachdb-operator is not running, aborting")
}
logger.Infof("cockroachdb operator started")
return nil
}
func (h *InstallHelper) CreateCockroachDBCluster(namespace string, count int) error {
if err := h.k8shelper.CreateNamespace(namespace); err != nil {
return err
}
logger.Infof("starting cockroachdb cluster with kubectl and yaml")
cockroachDBCluster := h.installData.GetCockroachDBCluster(namespace, count)
if _, err := h.k8shelper.KubectlWithStdin(cockroachDBCluster, createFromStdinArgs...); err != nil {
return fmt.Errorf("Failed to create cockroachdb cluster: %+v ", err)
}
if err := h.k8shelper.WaitForPodCount("app=rook-cockroachdb", namespace, count); err != nil {
logger.Errorf("cockroachdb cluster pods in namespace %s not found", namespace)
return err
}
err := h.k8shelper.WaitForLabeledPodToRun("app=rook-cockroachdb", namespace)
if err != nil {
logger.Errorf("cockroachdb cluster pods in namespace %s are not running", namespace)
return err
}
logger.Infof("cockroachdb cluster started")
return nil
}
func (h *InstallHelper) UninstallCockroachDB(systemNamespace, namespace string) {
logger.Infof("uninstalling cockroachdb from namespace %s", namespace)
_, err := h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cluster.cockroachdb.rook.io", namespace)
h.checkError(err, fmt.Sprintf("cannot remove cluster %s", namespace))
crdCheckerFunc := func() error {
_, err := h.k8shelper.RookClientset.CockroachdbV1alpha1().Clusters(namespace).Get(namespace, metav1.GetOptions{})
return err
}
err = h.waitForCustomResourceDeletion(namespace, crdCheckerFunc)
h.checkError(err, fmt.Sprintf("failed to wait for crd %s deletion", namespace))
_, err = h.k8shelper.DeleteResourceAndWait(false, "namespace", namespace)
h.checkError(err, fmt.Sprintf("cannot delete namespace %s", namespace))
logger.Infof("removing the operator from namespace %s", systemNamespace)
_, err = h.k8shelper.DeleteResource("crd", "clusters.cockroachdb.rook.io")
h.checkError(err, "cannot delete CRDs")
cockroachDBOperator := h.installData.GetCockroachDBOperator(systemNamespace)
_, err = h.k8shelper.KubectlWithStdin(cockroachDBOperator, deleteFromStdinArgs...)
h.checkError(err, "cannot uninstall rook-cockroachdb-operator")
err = h.UninstallHostPathProvisioner()
h.checkError(err, "cannot uninstall hostpath provisioner")
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
logger.Infof("done removing the operator from namespace %s", systemNamespace)
}
func (h *InstallHelper) GatherAllCockroachDBLogs(systemNamespace, namespace, testName string) {
logger.Infof("Gathering all logs from cockroachdb cluster %s", namespace)
h.k8shelper.GetRookLogs("rook-cockroachdb-operator", h.Env.HostType, systemNamespace, testName)
h.k8shelper.GetRookLogs("rook-cockroachdb", h.Env.HostType, namespace, testName)
}
func (i *InstallData) GetCockroachDBCRDs() string {
return `apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: clusters.cockroachdb.rook.io
spec:
group: cockroachdb.rook.io
names:
kind: Cluster
listKind: ClusterList
plural: clusters
singular: cluster
scope: Namespaced
version: v1alpha1
`
}
func (i *InstallData) GetCockroachDBOperator(namespace string) string {
return `kind: Namespace
apiVersion: v1
metadata:
name: ` + namespace + `
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-cockroachdb-operator
rules:
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- list
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- create
- update
- apiGroups:
- apps
resources:
- statefulsets
verbs:
- create
- apiGroups:
- policy
resources:
- poddisruptionbudgets
verbs:
- create
- apiGroups:
- cockroachdb.rook.io
resources:
- "*"
verbs:
- "*"
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-cockroachdb-operator
namespace: ` + namespace + `
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-cockroachdb-operator
namespace: ` + namespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-cockroachdb-operator
subjects:
- kind: ServiceAccount
name: rook-cockroachdb-operator
namespace: ` + namespace + `
---
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: rook-cockroachdb-operator
namespace: ` + namespace + `
spec:
replicas: 1
template:
metadata:
labels:
app: rook-cockroachdb-operator
spec:
serviceAccountName: rook-cockroachdb-operator
containers:
- name: rook-cockroachdb-operator
image: rook/cockroachdb:master
args: ["cockroachdb", "operator"]
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
`
}
func (i *InstallData) GetCockroachDBCluster(namespace string, count int) string {
return `apiVersion: cockroachdb.rook.io/v1alpha1
kind: Cluster
metadata:
name: ` + namespace + `
namespace: ` + namespace + `
spec:
scope:
nodeCount: ` + strconv.Itoa(count) + `
secure: false
volumeSize: 1Gi
cachePercent: 25
maxSQLMemoryPercent: 25
`
}
@@ -0,0 +1,156 @@
/*
Copyright 2018 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package installer
import (
"fmt"
"testing"
"github.com/rook/rook/tests/framework/utils"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
cockroachDBCRD = "clusters.cockroachdb.rook.io"
)
type CockroachDBInstaller struct {
k8shelper *utils.K8sHelper
manifests *CockroachDBManifests
T func() *testing.T
}
func NewCockroachDBInstaller(k8shelper *utils.K8sHelper, t func() *testing.T) *CockroachDBInstaller {
return &CockroachDBInstaller{k8shelper, &CockroachDBManifests{}, t}
}
func (h *CockroachDBInstaller) InstallCockroachDB(systemNamespace, namespace string, count int) error {
h.k8shelper.CreateAnonSystemClusterBinding()
// install hostpath provisioner if there isn't already a default storage class
defaultExists, err := h.k8shelper.IsDefaultStorageClassPresent()
if err != nil {
return err
} else if !defaultExists {
if err := InstallHostPathProvisioner(h.k8shelper); err != nil {
return err
}
} else {
logger.Info("skipping install of host path provisioner because a default storage class already exists")
}
// install cockroachdb operator
if err := h.CreateCockroachDBOperator(systemNamespace); err != nil {
return err
}
// install cockroachdb cluster instance
if err := h.CreateCockroachDBCluster(namespace, count); err != nil {
return err
}
return nil
}
func (h *CockroachDBInstaller) CreateCockroachDBOperator(namespace string) error {
logger.Infof("starting cockroachDB operator")
logger.Info("creating cockroachDB CRDs")
if _, err := h.k8shelper.KubectlWithStdin(h.manifests.GetCockroachDBCRDs(), createFromStdinArgs...); err != nil {
return err
}
cockroachDBOperator := h.manifests.GetCockroachDBOperator(namespace)
_, err := h.k8shelper.KubectlWithStdin(cockroachDBOperator, createFromStdinArgs...)
if err != nil {
return fmt.Errorf("failed to create rook-cockroachdb-operator pod: %+v ", err)
}
if !h.k8shelper.IsCRDPresent(cockroachDBCRD) {
return fmt.Errorf("failed to find cockroachdb CRD %s", cockroachDBCRD)
}
if !h.k8shelper.IsPodInExpectedState("rook-cockroachdb-operator", namespace, "Running") {
return fmt.Errorf("rook-cockroachdb-operator is not running, aborting")
}
logger.Infof("cockroachdb operator started")
return nil
}
func (h *CockroachDBInstaller) CreateCockroachDBCluster(namespace string, count int) error {
if err := h.k8shelper.CreateNamespace(namespace); err != nil {
return err
}
logger.Infof("starting cockroachdb cluster with kubectl and yaml")
cockroachDBCluster := h.manifests.GetCockroachDBCluster(namespace, count)
if _, err := h.k8shelper.KubectlWithStdin(cockroachDBCluster, createFromStdinArgs...); err != nil {
return fmt.Errorf("Failed to create cockroachdb cluster: %+v ", err)
}
if err := h.k8shelper.WaitForPodCount("app=rook-cockroachdb", namespace, count); err != nil {
logger.Errorf("cockroachdb cluster pods in namespace %s not found", namespace)
return err
}
err := h.k8shelper.WaitForLabeledPodToRun("app=rook-cockroachdb", namespace)
if err != nil {
logger.Errorf("cockroachdb cluster pods in namespace %s are not running", namespace)
return err
}
logger.Infof("cockroachdb cluster started")
return nil
}
func (h *CockroachDBInstaller) UninstallCockroachDB(systemNamespace, namespace string) {
logger.Infof("uninstalling cockroachdb from namespace %s", namespace)
_, err := h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cluster.cockroachdb.rook.io", namespace)
checkError(h.T(), err, fmt.Sprintf("cannot remove cluster %s", namespace))
crdCheckerFunc := func() error {
_, err := h.k8shelper.RookClientset.CockroachdbV1alpha1().Clusters(namespace).Get(namespace, metav1.GetOptions{})
return err
}
err = h.k8shelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc)
checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace))
_, err = h.k8shelper.DeleteResourceAndWait(false, "namespace", namespace)
checkError(h.T(), err, fmt.Sprintf("cannot delete namespace %s", namespace))
logger.Infof("removing the operator from namespace %s", systemNamespace)
_, err = h.k8shelper.DeleteResource("crd", "clusters.cockroachdb.rook.io")
checkError(h.T(), err, "cannot delete CRDs")
cockroachDBOperator := h.manifests.GetCockroachDBOperator(systemNamespace)
_, err = h.k8shelper.KubectlWithStdin(cockroachDBOperator, deleteFromStdinArgs...)
checkError(h.T(), err, "cannot uninstall rook-cockroachdb-operator")
err = UninstallHostPathProvisioner(h.k8shelper)
checkError(h.T(), err, "cannot uninstall hostpath provisioner")
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
logger.Infof("done removing the operator from namespace %s", systemNamespace)
}
func (h *CockroachDBInstaller) GatherAllCockroachDBLogs(systemNamespace, namespace, testName string) {
logger.Infof("Gathering all logs from cockroachdb cluster %s", namespace)
h.k8shelper.GetRookLogs("rook-cockroachdb-operator", Env.HostType, systemNamespace, testName)
h.k8shelper.GetRookLogs("rook-cockroachdb", Env.HostType, namespace, testName)
}
@@ -0,0 +1,153 @@
/*
Copyright 2018 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package installer
import (
"strconv"
)
type CockroachDBManifests struct {
}
func (i *CockroachDBManifests) GetCockroachDBCRDs() string {
return `apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: clusters.cockroachdb.rook.io
spec:
group: cockroachdb.rook.io
names:
kind: Cluster
listKind: ClusterList
plural: clusters
singular: cluster
scope: Namespaced
version: v1alpha1
`
}
func (i *CockroachDBManifests) GetCockroachDBOperator(namespace string) string {
return `kind: Namespace
apiVersion: v1
metadata:
name: ` + namespace + `
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-cockroachdb-operator
rules:
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- list
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- create
- update
- apiGroups:
- apps
resources:
- statefulsets
verbs:
- create
- apiGroups:
- policy
resources:
- poddisruptionbudgets
verbs:
- create
- apiGroups:
- cockroachdb.rook.io
resources:
- "*"
verbs:
- "*"
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-cockroachdb-operator
namespace: ` + namespace + `
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-cockroachdb-operator
namespace: ` + namespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-cockroachdb-operator
subjects:
- kind: ServiceAccount
name: rook-cockroachdb-operator
namespace: ` + namespace + `
---
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: rook-cockroachdb-operator
namespace: ` + namespace + `
spec:
replicas: 1
template:
metadata:
labels:
app: rook-cockroachdb-operator
spec:
serviceAccountName: rook-cockroachdb-operator
containers:
- name: rook-cockroachdb-operator
image: rook/cockroachdb:master
args: ["cockroachdb", "operator"]
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
`
}
func (i *CockroachDBManifests) GetCockroachDBCluster(namespace string, count int) string {
return `apiVersion: cockroachdb.rook.io/v1alpha1
kind: Cluster
metadata:
name: ` + namespace + `
namespace: ` + namespace + `
spec:
scope:
nodeCount: ` + strconv.Itoa(count) + `
secure: false
volumeSize: 1Gi
cachePercent: 25
maxSQLMemoryPercent: 25
`
}
+68
View File
@@ -0,0 +1,68 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package installer
import (
"strings"
"github.com/rook/rook/pkg/util/exec"
"github.com/rook/rook/pkg/util/sys"
)
func IsAdditionalDeviceAvailableOnCluster() bool {
executor := &exec.CommandExecutor{}
devices, err := sys.ListDevices(executor)
if err != nil {
return false
}
disks := 0
logger.Infof("devices : %v", devices)
for _, device := range devices {
if strings.Contains(device, "loop") {
continue
}
props, _ := sys.GetDeviceProperties(device, executor)
if props["TYPE"] != "disk" {
continue
}
ownPartitions, fs, err := sys.CheckIfDeviceAvailable(executor, device)
if err != nil {
logger.Warningf("failed to detect device %s availability. %+v", device, err)
continue
}
if !ownPartitions {
logger.Infof("skipping device %s since don't own partitions", device)
continue
}
if fs != "" {
logger.Infof("skipping device %s since it has file system %s", device, fs)
continue
}
if strings.HasPrefix(device, "rbd") {
logger.Infof("skipping unexpected rbd device: %s", device)
continue
}
logger.Infof("available device: %s", device)
disks++
}
if disks > 0 {
return true
}
logger.Info("No additional disks found on cluster")
return false
}
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package objects
package installer
import (
"flag"
@@ -37,7 +37,7 @@ var Env EnvironmentManifest
func init() {
Env = EnvironmentManifest{}
flag.StringVar(&Env.HostType, "host_type", "localhost", "Host were tests are run eg - localhost,GCE or AWS")
flag.StringVar(&Env.HostType, "host_type", "localhost", "Host where tests are run eg - localhost,GCE or AWS")
flag.StringVar(&Env.RookImageName, "rook_image", "rook/ceph", "Docker image name for the rook container to install, must be in docker hub or local environment")
flag.StringVar(&Env.ToolboxImageName, "toolbox_image", "rook/ceph-toolbox", "Docker image name of the toolbox container to install, must be in docker hub or local environment")
flag.BoolVar(&Env.SkipInstallRook, "skip_install_rook", false, "Indicate if Rook need to installed - false if tests are being running at Rook that is pre-installed")
+90
View File
@@ -0,0 +1,90 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package installer
import (
"fmt"
"os"
"runtime"
"testing"
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
"k8s.io/apimachinery/pkg/api/errors"
)
const (
// Version tag for the latest manifests
VersionMaster = "master"
)
var (
// ** Variables that might need to be changed depending on the dev environment. The init function below will modify some of them automatically. **
baseTestDir string
forceUseDevices = false
createBaseTestDir = true
// ** end of Variables to modify
logger = capnslog.NewPackageLogger("github.com/rook/rook", "installer")
createArgs = []string{"create", "-f"}
createFromStdinArgs = append(createArgs, "-")
deleteArgs = []string{"delete", "-f"}
deleteFromStdinArgs = append(deleteArgs, "-")
)
type TestSuite interface {
Setup()
Teardown()
}
func init() {
// this default will only work if running kubernetes on the local machine
baseTestDir, _ = os.Getwd()
// The following settings could apply to any environment when the kube context is running on the host and the tests are running inside a
// VM such as minikube. This is a cheap test for this condition, we need to find a better way to automate these settings.
if runtime.GOOS == "darwin" {
createBaseTestDir = false
baseTestDir = "/data"
}
}
func SystemNamespace(namespace string) string {
return fmt.Sprintf("%s-system", namespace)
}
func checkError(t *testing.T, err error, message string) {
// During cleanup the resource might not be found because the test might have failed before the test was done and never created the resource
if err == nil || errors.IsNotFound(err) {
return
}
assert.NoError(t, err, "%s. %+v", message, err)
}
func concatYaml(first, second string) string {
return first + `
---
` + second
}
// GatherCRDObjectDebuggingInfo gathers all the descriptions for pods, pvs and pvcs
func GatherCRDObjectDebuggingInfo(k8shelper *utils.K8sHelper, namespace string) {
k8shelper.PrintPodDescribeForNamespace(namespace)
k8shelper.PrintPVs(true /*detailed*/)
k8shelper.PrintPVCs(namespace, true /*detailed*/)
k8shelper.PrintStorageClasses(true /*detailed*/)
}
@@ -18,12 +18,28 @@ package installer
import (
"fmt"
"testing"
"github.com/rook/rook/tests/framework/utils"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
nfsServerCRD = "nfsservers.nfs.rook.io"
)
type NFSInstaller struct {
k8shelper *utils.K8sHelper
manifests *NFSManifests
T func() *testing.T
}
func NewNFSInstaller(k8shelper *utils.K8sHelper, t func() *testing.T) *NFSInstaller {
return &NFSInstaller{k8shelper, &NFSManifests{}, t}
}
// InstallNFSServer installs NFS operator, NFS CRD instance and NFS volume
func (h *InstallHelper) InstallNFSServer(systemNamespace, namespace string, count int) error {
func (h *NFSInstaller) InstallNFSServer(systemNamespace, namespace string, count int) error {
h.k8shelper.CreateAnonSystemClusterBinding()
// install hostpath provisioner if there isn't already a default storage class
@@ -32,7 +48,7 @@ func (h *InstallHelper) InstallNFSServer(systemNamespace, namespace string, coun
if err != nil {
return err
} else if !defaultExists {
if err := h.InstallHostPathProvisioner(); err != nil {
if err := InstallHostPathProvisioner(h.k8shelper); err != nil {
return err
}
storageClassName = "hostpath"
@@ -59,15 +75,15 @@ func (h *InstallHelper) InstallNFSServer(systemNamespace, namespace string, coun
}
// CreateNFSServerOperator creates nfs server in the provided namespace
func (h *InstallHelper) CreateNFSServerOperator(namespace string) error {
func (h *NFSInstaller) CreateNFSServerOperator(namespace string) error {
logger.Infof("starting nfsserver operator")
logger.Info("creating nfsserver CRDs")
if _, err := h.k8shelper.KubectlWithStdin(h.installData.GetNFSServerCRDs(), createFromStdinArgs...); err != nil {
if _, err := h.k8shelper.KubectlWithStdin(h.manifests.GetNFSServerCRDs(), createFromStdinArgs...); err != nil {
return err
}
nfsOperator := h.installData.GetNFSServerOperator(namespace)
nfsOperator := h.manifests.GetNFSServerOperator(namespace)
_, err := h.k8shelper.KubectlWithStdin(nfsOperator, createFromStdinArgs...)
if err != nil {
return fmt.Errorf("failed to create rook-nfs-operator pod: %+v ", err)
@@ -86,13 +102,13 @@ func (h *InstallHelper) CreateNFSServerOperator(namespace string) error {
}
// CreateNFSServer creates the NFS Server CRD instance
func (h *InstallHelper) CreateNFSServer(namespace string, count int, storageClassName string) error {
func (h *NFSInstaller) CreateNFSServer(namespace string, count int, storageClassName string) error {
if err := h.k8shelper.CreateNamespace(namespace); err != nil {
return err
}
logger.Infof("starting nfs server with kubectl and yaml")
nfsServer := h.installData.GetNFSServer(namespace, count, storageClassName)
nfsServer := h.manifests.GetNFSServer(namespace, count, storageClassName)
if _, err := h.k8shelper.KubectlWithStdin(nfsServer, createFromStdinArgs...); err != nil {
return fmt.Errorf("Failed to create nfs server: %+v ", err)
}
@@ -113,15 +129,15 @@ func (h *InstallHelper) CreateNFSServer(namespace string, count int, storageClas
}
// CreateNFSServerVolume creates NFS export PV and PVC
func (h *InstallHelper) CreateNFSServerVolume(namespace string) error {
func (h *NFSInstaller) CreateNFSServerVolume(namespace string) error {
logger.Info("creating volume from nfs server in namespace %s", namespace)
clusterIP, err := h.GetNFSServerClusterIP(namespace)
if err != nil {
return err
}
nfsServerPV := h.installData.GetNFSServerPV(namespace, clusterIP)
nfsServerPVC := h.installData.GetNFSServerPVC()
nfsServerPV := h.manifests.GetNFSServerPV(namespace, clusterIP)
nfsServerPVC := h.manifests.GetNFSServerPVC()
logger.Info("creating nfs server pv")
if _, err := h.k8shelper.KubectlWithStdin(nfsServerPV, createFromStdinArgs...); err != nil {
@@ -137,52 +153,52 @@ func (h *InstallHelper) CreateNFSServerVolume(namespace string) error {
}
// UninstallNFSServer uninstalls the NFS Server from the given namespace
func (h *InstallHelper) UninstallNFSServer(systemNamespace, namespace string) {
func (h *NFSInstaller) UninstallNFSServer(systemNamespace, namespace string) {
logger.Infof("uninstalling nfsserver from namespace %s", namespace)
_, err := h.k8shelper.DeleteResource("pvc", "nfs-pv-claim")
h.checkError(err, fmt.Sprintf("cannot remove nfs pvc"))
checkError(h.T(), err, fmt.Sprintf("cannot remove nfs pvc"))
_, err = h.k8shelper.DeleteResource("pv", "nfs-pv")
h.checkError(err, fmt.Sprintf("cannot remove nfs pv"))
checkError(h.T(), err, fmt.Sprintf("cannot remove nfs pv"))
_, err = h.k8shelper.DeleteResource("-n", namespace, "nfsservers.nfs.rook.io", namespace)
h.checkError(err, fmt.Sprintf("cannot remove nfsserver %s", namespace))
checkError(h.T(), err, fmt.Sprintf("cannot remove nfsserver %s", namespace))
crdCheckerFunc := func() error {
_, err := h.k8shelper.RookClientset.NfsV1alpha1().NFSServers(namespace).Get(namespace, metav1.GetOptions{})
return err
}
err = h.waitForCustomResourceDeletion(namespace, crdCheckerFunc)
h.checkError(err, fmt.Sprintf("failed to wait for crd %s deletion", namespace))
err = h.k8shelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc)
checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace))
_, err = h.k8shelper.DeleteResource("namespace", namespace)
h.checkError(err, fmt.Sprintf("cannot delete namespace %s", namespace))
checkError(h.T(), err, fmt.Sprintf("cannot delete namespace %s", namespace))
logger.Infof("removing the operator from namespace %s", systemNamespace)
_, err = h.k8shelper.DeleteResource("crd", "nfsservers.nfs.rook.io")
h.checkError(err, "cannot delete CRDs")
checkError(h.T(), err, "cannot delete CRDs")
nfsOperator := h.installData.GetNFSServerOperator(systemNamespace)
nfsOperator := h.manifests.GetNFSServerOperator(systemNamespace)
_, err = h.k8shelper.KubectlWithStdin(nfsOperator, deleteFromStdinArgs...)
h.checkError(err, "cannot uninstall rook-nfs-operator")
checkError(h.T(), err, "cannot uninstall rook-nfs-operator")
err = h.UninstallHostPathProvisioner()
h.checkError(err, "cannot uninstall hostpath provisioner")
err = UninstallHostPathProvisioner(h.k8shelper)
checkError(h.T(), err, "cannot uninstall hostpath provisioner")
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
logger.Infof("done removing the operator from namespace %s", systemNamespace)
}
// GatherAllNFSServerLogs gathers all NFS Server logs
func (h *InstallHelper) GatherAllNFSServerLogs(systemNamespace, namespace, testName string) {
func (h *NFSInstaller) GatherAllNFSServerLogs(systemNamespace, namespace, testName string) {
logger.Infof("Gathering all logs from NFSServer %s", namespace)
h.k8shelper.GetRookLogs("rook-nfs-operator", h.Env.HostType, systemNamespace, testName)
h.k8shelper.GetRookLogs("rook-nfs", h.Env.HostType, namespace, testName)
h.k8shelper.GetRookLogs("rook-nfs-operator", Env.HostType, systemNamespace, testName)
h.k8shelper.GetRookLogs("rook-nfs", Env.HostType, namespace, testName)
}
// GetNFSServerClusterIP gets the nfs server cluster ip on which it serves
func (h *InstallHelper) GetNFSServerClusterIP(namespace string) (string, error) {
func (h *NFSInstaller) GetNFSServerClusterIP(namespace string) (string, error) {
clusterIP := ""
service, err := h.k8shelper.GetService("rook-nfs", namespace)
if err != nil {
@@ -20,8 +20,11 @@ import (
"strconv"
)
type NFSManifests struct {
}
// GetNFSServerCRDs returns NFSServer CRD definition
func (i *InstallData) GetNFSServerCRDs() string {
func (i *NFSManifests) GetNFSServerCRDs() string {
return `apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
@@ -39,7 +42,7 @@ spec:
}
// GetNFSServerOperator returns the NFSServer operator definition
func (i *InstallData) GetNFSServerOperator(namespace string) string {
func (i *NFSManifests) GetNFSServerOperator(namespace string) string {
return `kind: Namespace
apiVersion: v1
metadata:
@@ -131,7 +134,7 @@ spec:
}
// GetNFSServerPV returns NFSServer PV definition
func (i *InstallData) GetNFSServerPV(namespace string, clusterIP string) string {
func (i *NFSManifests) GetNFSServerPV(namespace string, clusterIP string) string {
return `apiVersion: v1
kind: PersistentVolume
metadata:
@@ -152,7 +155,7 @@ spec:
}
// GetNFSServerPVC returns NFSServer PVC definition
func (i *InstallData) GetNFSServerPVC() string {
func (i *NFSManifests) GetNFSServerPVC() string {
return `apiVersion: v1
kind: PersistentVolumeClaim
metadata:
@@ -168,7 +171,7 @@ spec:
}
// GetNFSServer returns NFSServer CRD instance definition
func (i *InstallData) GetNFSServer(namespace string, count int, storageClassName string) string {
func (i *NFSManifests) GetNFSServer(namespace string, count int, storageClassName string) string {
return `apiVersion: v1
kind: PersistentVolumeClaim
metadata:
+13 -13
View File
@@ -34,68 +34,68 @@ const (
// ************************************************************************************************
// HostPath provisioner functions
// ************************************************************************************************
func (h *InstallHelper) InstallHostPathProvisioner() error {
func InstallHostPathProvisioner(k8shelper *utils.K8sHelper) error {
logger.Info("installing host path provisioner")
rbacResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerRBAC)
args := append(createArgs, rbacResourceURL)
out, err := h.k8shelper.Kubectl(args...)
out, err := k8shelper.Kubectl(args...)
if err != nil && !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create hostpath provisioner RBAC: %+v. %s", err, out)
}
deploymentResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerDeployment)
args = append(createArgs, deploymentResourceURL)
out, err = h.k8shelper.Kubectl(args...)
out, err = k8shelper.Kubectl(args...)
if err != nil && !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create hostpath provisioner deployment: %+v. %s", err, out)
}
storageClassResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerStorageClass)
args = append(createArgs, storageClassResourceURL)
out, err = h.k8shelper.Kubectl(args...)
out, err = k8shelper.Kubectl(args...)
if err != nil && !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create hostpath provisioner StorageClass: %+v. %s", err, out)
}
err = h.k8shelper.WaitForLabeledPodToRun("k8s-app=hostpath-provisioner", "kube-system")
err = k8shelper.WaitForLabeledPodToRun("k8s-app=hostpath-provisioner", "kube-system")
if err != nil {
logger.Errorf("hostpath provisioner pod is not running: %+v", err)
h.k8shelper.PrintPodDescribeForNamespace("kube-system")
h.k8shelper.PrintStorageClasses(true /*detailed*/)
k8shelper.PrintPodDescribeForNamespace("kube-system")
k8shelper.PrintStorageClasses(true /*detailed*/)
return err
}
err = h.k8shelper.IsStorageClassPresent(hostPathStorageClassName)
err = k8shelper.IsStorageClassPresent(hostPathStorageClassName)
if err != nil {
logger.Errorf("storageClass %s not found: %+v", hostPathStorageClassName, err)
h.k8shelper.PrintStorageClasses(true /*detailed*/)
k8shelper.PrintStorageClasses(true /*detailed*/)
return err
}
return nil
}
func (h *InstallHelper) UninstallHostPathProvisioner() error {
func UninstallHostPathProvisioner(k8shelper *utils.K8sHelper) error {
logger.Info("uninstalling host path provisioner")
storageClassResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerStorageClass)
args := append(deleteArgs, storageClassResourceURL)
out, err := h.k8shelper.Kubectl(args...)
out, err := k8shelper.Kubectl(args...)
if err != nil && !utils.IsKubectlErrorNotFound(out, err) {
return fmt.Errorf("failed to delete hostpath provisioner StorageClass: %+v. %s", err, out)
}
deploymentResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerDeployment)
args = append(deleteArgs, deploymentResourceURL)
out, err = h.k8shelper.Kubectl(args...)
out, err = k8shelper.Kubectl(args...)
if err != nil && !utils.IsKubectlErrorNotFound(out, err) {
return fmt.Errorf("failed to delete hostpath provisioner deployment: %+v. %s", err, out)
}
rbacResourceURL := fmt.Sprintf(hostPathProvisionerResourceBaseURL, hostPathProvisionerRBAC)
args = append(deleteArgs, rbacResourceURL)
out, err = h.k8shelper.Kubectl(args...)
out, err = k8shelper.Kubectl(args...)
if err != nil && !utils.IsKubectlErrorNotFound(out, err) {
return fmt.Errorf("failed to delete hostpath provisioner RBAC: %+v. %s", err, out)
}
-65
View File
@@ -1,65 +0,0 @@
package installer
import (
"github.com/rook/rook/tests/framework/utils"
)
func BlockResourceOperation(k8sh *utils.K8sHelper, yaml string, action string) (string, error) {
return k8sh.ResourceOperation(action, yaml)
}
func GetBlockPoolDef(poolName string, namespace string, replicaSize string) string {
return `apiVersion: ceph.rook.io/v1beta1
kind: Pool
metadata:
name: ` + poolName + `
namespace: ` + namespace + `
spec:
replicated:
size: ` + replicaSize
}
func GetBlockStorageClassDef(poolName string, storageClassName string, namespace string, varClusterName bool) string {
namespaceParameter := "clusterNamespace"
if varClusterName {
namespaceParameter = "clusterName"
}
return `apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: ` + storageClassName + `
provisioner: ceph.rook.io/block
parameters:
pool: ` + poolName + `
` + namespaceParameter + `: ` + namespace
}
func GetBlockPvcDef(claimName string, storageClassName string, accessModes string) string {
return `apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: ` + claimName + `
annotations:
volume.beta.kubernetes.io/storage-class: ` + storageClassName + `
spec:
accessModes:
- ` + accessModes + `
resources:
requests:
storage: 1M`
}
func concatYaml(first, second string) string {
return first + `
---
` + second
}
func GetBlockPoolStorageClassAndPvcDef(namespace string, poolName string, storageClassName string, blockName string, accessMode string) string {
return concatYaml(GetBlockPoolDef(poolName, namespace, "1"),
concatYaml(GetBlockStorageClassDef(poolName, storageClassName, namespace, false), GetBlockPvcDef(blockName, storageClassName, accessMode)))
}
func GetBlockPoolStorageClass(namespace string, poolName string, storageClassName string) string {
return concatYaml(GetBlockPoolDef(poolName, namespace, "1"), GetBlockStorageClassDef(poolName, storageClassName, namespace, false))
}
+43 -1
View File
@@ -290,11 +290,53 @@ func (k8sh *K8sHelper) ResourceOperation(action string, podDefinition string) (s
return "", fmt.Errorf("Could Not create resource in args : %v -- %v", args, err)
}
//DeleteResource performs a kubectl delete on give args
//DeletePod performs a kubectl delete pod on the given pod
func (k8sh *K8sHelper) DeletePod(namespace, name string) (string, error) {
args := append([]string{"--grace-period=0", "pod"}, name)
if namespace != "" {
args = append(args, []string{"-n", namespace}...)
}
return k8sh.DeleteResourceAndWait(true, args...)
}
//DeletePods performs a kubectl delete pod on the given pods
func (k8sh *K8sHelper) DeletePods(pods ...string) (msg string, err error) {
for _, pod := range pods {
msg, err = k8sh.DeletePod("", pod)
}
return
}
//DeleteResource performs a kubectl delete on the given args
func (k8sh *K8sHelper) DeleteResource(args ...string) (string, error) {
return k8sh.DeleteResourceAndWait(true, args...)
}
//WaitForCustomResourceDeletion waits for the CRD deletion
func (k8sh *K8sHelper) WaitForCustomResourceDeletion(namespace string, checkerFunc func() error) error {
if !k8sh.VersionAtLeast("v1.8.0") {
// v1.7 has an intermittent issue with long delay to delete resources so we will skip waiting
return nil
}
// wait for the operator to finalize and delete the CRD
for i := 0; i < 10; i++ {
err := checkerFunc()
if err == nil {
logger.Infof("custom resource %s still exists", namespace)
time.Sleep(2 * time.Second)
continue
}
if errors.IsNotFound(err) {
logger.Infof("custom resource %s deleted", namespace)
return nil
}
return err
}
logger.Errorf("gave up deleting custom resource %s", namespace)
return nil
}
// DeleteResource performs a kubectl delete on give args.
// If wait is false, a flag will be passed to indicate the delete should return immediately
func (k8sh *K8sHelper) DeleteResourceAndWait(wait bool, args ...string) (string, error) {
+51 -38
View File
@@ -21,6 +21,7 @@ import (
"strconv"
"time"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
@@ -33,18 +34,13 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
var (
blockMountPath = "/tmp/rook1"
blockPodName = "block-test"
)
// Smoke Test for Block Storage - Test check the following operations on Block Storage in order
//Create,Mount,Write,Read,Unmount and Delete.
func runBlockE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
podName := "block-test"
poolName := "replicapool"
storageClassName := "rook-ceph-block"
blockName := "block-pv-claim"
podName := "block-test"
defer blockTestDataCleanUp(helper, k8sh, namespace, poolName, storageClassName, blockName, podName)
logger.Infof("Block Storage End to End Integration Test - create, mount, write to, read from, and unmount")
@@ -54,28 +50,22 @@ func runBlockE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.
initBlockImages, _ := helper.BlockClient.List(namespace)
logger.Infof("step 1: Create block storage")
_, cbErr := installer.BlockResourceOperation(k8sh, installer.GetBlockPoolStorageClassAndPvcDef(namespace, poolName, storageClassName, blockName, "ReadWriteOnce"), "create")
_, cbErr := helper.PoolClient.CreateStorageClassAndPvc(namespace, poolName, storageClassName, blockName, "ReadWriteOnce")
require.Nil(s.T(), cbErr)
require.True(s.T(), retryBlockImageCountCheck(helper, len(initBlockImages), 1, namespace), "Make sure a new block is created")
logger.Infof("Block Storage created successfully")
require.True(s.T(), k8sh.WaitUntilPVCIsBound(defaultNamespace, blockName), "Make sure PVC is Bound")
logger.Infof("step 2: Mount block storage")
_, mtErr := helper.BlockClient.BlockMap(getBlockPodDefintion(podName, blockName, false), blockMountPath)
require.Nil(s.T(), mtErr)
crdName, err := k8sh.GetVolumeResourceName(defaultNamespace, blockName)
require.Nil(s.T(), err)
require.True(s.T(), k8sh.IsVolumeResourcePresent(installer.SystemNamespace(namespace), crdName), fmt.Sprintf("make sure Volume %s is created", crdName))
require.True(s.T(), k8sh.IsPodRunning(blockPodName, defaultNamespace), "make sure block-test pod is in running state")
logger.Infof("Block Storage Mounted successfully")
crdName := createPodWithBlock(helper, k8sh, s, namespace, blockName, podName)
logger.Infof("step 3: Write to block storage")
_, wtErr := helper.BlockClient.Write(blockPodName, blockMountPath, "Smoke Test Data form Block storage", "bsFile1", "")
_, wtErr := helper.BlockClient.Write(podName, "Smoke Test Data form Block storage", "bsFile1", "")
require.Nil(s.T(), wtErr)
logger.Infof("Write to Block storage successfully")
logger.Infof("step 4: Read from block storage")
read, rErr := helper.BlockClient.Read(blockPodName, blockMountPath, "bsFile1", "")
read, rErr := helper.BlockClient.Read(podName, "bsFile1", "")
require.Nil(s.T(), rErr)
assert.Contains(s.T(), read, "Smoke Test Data form Block storage", "make sure content of the files is unchanged")
logger.Infof("Read from Block storage successfully")
@@ -84,38 +74,49 @@ func runBlockE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.
restartOSDPods(k8sh, s, namespace)
logger.Infof("step 6: Read from block storage again")
read, rErr = helper.BlockClient.Read(blockPodName, blockMountPath, "bsFile1", "")
read, rErr = helper.BlockClient.Read(podName, "bsFile1", "")
require.Nil(s.T(), rErr)
assert.Contains(s.T(), read, "Smoke Test Data form Block storage", "make sure content of the files is unchanged")
logger.Infof("Read from Block storage successfully")
logger.Infof("step 7: Mount same block storage on a different pod. Should not be allowed")
otherPod := "block-test2"
_, mtErr = helper.BlockClient.BlockMap(getBlockPodDefintion(otherPod, blockName, false), blockMountPath)
_, mtErr := helper.BlockClient.BlockMap(getBlockPodDefintion(otherPod, blockName, false))
require.Nil(s.T(), mtErr)
require.True(s.T(), k8sh.IsPodInError(otherPod, defaultNamespace, "FailedMount", "Volume is already attached by pod"), "make sure block-test2 pod errors out while mounting the volume")
logger.Infof("Block Storage successfully fenced")
logger.Infof("step 8: Delete fenced pod")
_, unmtErr := helper.BlockClient.BlockUnmap(getBlockPodDefintion(otherPod, blockName, false), blockMountPath)
_, unmtErr := k8sh.DeletePod(k8sutil.DefaultNamespace, otherPod)
require.Nil(s.T(), unmtErr)
require.True(s.T(), k8sh.IsPodTerminated(otherPod, defaultNamespace), "make sure block-test2 pod is terminated")
logger.Infof("Fenced pod deleted successfully")
logger.Infof("step 9: Unmount block storage")
_, unmtErr = helper.BlockClient.BlockUnmap(getBlockPodDefintion(podName, blockName, false), blockMountPath)
_, unmtErr = k8sh.DeletePod(k8sutil.DefaultNamespace, podName)
require.Nil(s.T(), unmtErr)
require.True(s.T(), k8sh.IsVolumeResourceAbsent(installer.SystemNamespace(namespace), crdName), fmt.Sprintf("make sure Volume %s is deleted", crdName))
require.True(s.T(), k8sh.IsPodTerminated(blockPodName, defaultNamespace), "make sure block-test pod is terminated")
require.True(s.T(), k8sh.IsPodTerminated(podName, defaultNamespace), "make sure block-test pod is terminated")
logger.Infof("Block Storage unmounted successfully")
logger.Infof("step 10: Deleting block storage")
_, dbErr := installer.BlockResourceOperation(k8sh, installer.GetBlockPoolStorageClassAndPvcDef(namespace, poolName, storageClassName, blockName, "ReadWriteOnce"), "delete")
dbErr := helper.PoolClient.DeleteStorageClassAndPvc(namespace, poolName, storageClassName, blockName, "ReadWriteOnce")
require.Nil(s.T(), dbErr)
require.True(s.T(), retryBlockImageCountCheck(helper, len(initBlockImages), 0, namespace), "Make sure a block is deleted")
logger.Infof("Block Storage deleted successfully")
}
func createPodWithBlock(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, blockName, podName string) string {
_, mtErr := helper.BlockClient.BlockMap(getBlockPodDefintion(podName, blockName, false))
require.Nil(s.T(), mtErr)
crdName, err := k8sh.GetVolumeResourceName(defaultNamespace, blockName)
require.Nil(s.T(), err)
require.True(s.T(), k8sh.IsVolumeResourcePresent(installer.SystemNamespace(namespace), crdName), fmt.Sprintf("make sure Volume %s is created", crdName))
require.True(s.T(), k8sh.IsPodRunning(podName, defaultNamespace), "make sure block-test pod is in running state")
logger.Infof("Block Storage Mounted successfully")
return crdName
}
func restartOSDPods(k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
osdLabel := "app=rook-ceph-osd"
@@ -142,38 +143,51 @@ func runBlockE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s su
logger.Infof("Block Storage End to End Integration Test - create storageclass,pool and pvc")
logger.Infof("Running on Rook Cluster %s", clusterNamespace)
poolName := "rookpool"
storageClassName := "rook-ceph-block-lite"
blockName := "test-block-claim-lite"
podName := "test-pod-lite"
defer blockTestDataCleanUp(helper, k8sh, clusterNamespace, poolName, storageClassName, blockName, podName)
initBlockCount := setupBlockLite(helper, k8sh, s, clusterNamespace, poolName, storageClassName, blockName, podName)
deleteBlockLite(helper, k8sh, s, clusterNamespace, poolName, storageClassName, blockName, podName, initBlockCount)
}
func setupBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite,
clusterNamespace, poolName, storageClassName, blockName, podName string) int {
//Check initial number of blocks
defer blockTestDataCleanUp(helper, k8sh, clusterNamespace, poolName, "rook-ceph-block", "test-block", "block-test")
bc := helper.BlockClient
initialBlocks, err := bc.List(clusterNamespace)
initialBlocks, err := helper.BlockClient.List(clusterNamespace)
require.Nil(s.T(), err)
initBlockCount := len(initialBlocks)
logger.Infof("step : Create Pool,StorageClass and PVC")
volumeDef := installer.GetBlockPoolStorageClassAndPvcDef(clusterNamespace, poolName, "rook-ceph-block", "test-block-claim", "ReadWriteOnce")
res1, err := installer.BlockResourceOperation(k8sh, volumeDef, "create")
checkOrderedSubstrings(s.T(), res1, poolName, "created", "rook-ceph-block", "created", "test-block-claim", "created")
res1, err := helper.PoolClient.CreateStorageClassAndPvc(clusterNamespace, poolName, storageClassName, blockName, "ReadWriteOnce")
checkOrderedSubstrings(s.T(), res1, poolName, "created", storageClassName, "created", blockName, "created")
require.NoError(s.T(), err)
require.True(s.T(), k8sh.WaitUntilPVCIsBound(defaultNamespace, "test-block-claim"))
require.True(s.T(), k8sh.WaitUntilPVCIsBound(defaultNamespace, blockName))
//Make sure new block is created
b, _ := bc.List(clusterNamespace)
//Make sure new block is created
b, _ := helper.BlockClient.List(clusterNamespace)
assert.Equal(s.T(), initBlockCount+1, len(b), "Make sure new block image is created")
poolExists, err := helper.PoolClient.CephPoolExists(clusterNamespace, poolName)
assert.Nil(s.T(), err)
assert.True(s.T(), poolExists)
return initBlockCount
}
func deleteBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite,
clusterNamespace, poolName, storageClassName, blockName, podName string, initBlockCount int) {
logger.Infof("deleteBlockLite: cleaning up after test")
//Delete pvc and storageclass
_, err = installer.BlockResourceOperation(k8sh, volumeDef, "delete")
err := helper.PoolClient.DeleteStorageClassAndPvc(clusterNamespace, poolName, storageClassName, blockName, "ReadWriteOnce")
assert.NoError(s.T(), err)
assert.True(s.T(), k8sh.WaitUntilPVCIsDeleted(defaultNamespace, "test-block-claim"))
assert.True(s.T(), k8sh.WaitUntilPVCIsDeleted(defaultNamespace, blockName))
require.True(s.T(), retryBlockImageCountCheck(helper, initBlockCount, 0, clusterNamespace), "Make sure a new block is deleted")
b, _ = bc.List(clusterNamespace)
b, _ := helper.BlockClient.List(clusterNamespace)
assert.Equal(s.T(), initBlockCount, len(b), "Make sure new block image is deleted")
checkPoolDeleted(helper, s, clusterNamespace, poolName)
@@ -199,8 +213,8 @@ func checkPoolDeleted(helper *clients.TestClient, s suite.Suite, namespace, name
func blockTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, poolname, storageclassname, blockname, podname string) {
logger.Infof("Cleaning up block storage")
helper.BlockClient.BlockUnmap(getBlockPodDefintion(podname, blockname, false), blockMountPath)
installer.BlockResourceOperation(k8sh, installer.GetBlockPoolStorageClassAndPvcDef(namespace, poolname, storageclassname, blockname, "ReadWriteOnce"), "delete")
k8sh.DeletePod(k8sutil.DefaultNamespace, podname)
helper.PoolClient.DeleteStorageClassAndPvc(namespace, poolname, storageclassname, blockname, "ReadWriteOnce")
cleanupDynamicBlockStorage(helper, namespace)
}
@@ -227,7 +241,6 @@ func cleanupDynamicBlockStorage(helper *clients.TestClient, namespace string) {
for _, blockImage := range blockImagesList {
helper.BlockClient.DeleteBlockImage(blockImage, namespace)
}
}
func getBlockPodDefintion(podname, blockName string, readOnly bool) string {
@@ -245,7 +258,7 @@ spec:
imagePullPolicy: IfNotPresent
volumeMounts:
- name: block-persistent-storage
mountPath: /tmp/rook1
mountPath: ` + clients.TestMountPath + `
volumes:
- name: block-persistent-storage
persistentVolumeClaim:
+22 -28
View File
@@ -25,7 +25,6 @@ import (
"github.com/coreos/pkg/capnslog"
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/contracts"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
@@ -33,10 +32,12 @@ import (
"github.com/stretchr/testify/suite"
)
const (
defaultNamespace = "default"
)
var (
logger = capnslog.NewPackageLogger("github.com/rook/rook", "integrationTest")
defaultNamespace = "default"
)
//Test to make sure all rook components are installed and Running
@@ -76,30 +77,18 @@ func checkIfRookClusterIsHealthy(s suite.Suite, testClient *clients.TestClient,
require.Nil(s.T(), err)
}
func HandlePanics(r interface{}, op contracts.Setup, t func() *testing.T) {
func HandlePanics(r interface{}, op installer.TestSuite, t func() *testing.T) {
if r != nil {
logger.Infof("unexpected panic occurred during test %s, --> %v", t().Name(), r)
t().Fail()
op.TearDown()
op.Teardown()
t().FailNow()
}
}
//GetTestClient sets up SetTestClient for rook
func GetTestClient(kh *utils.K8sHelper, namespace string, op contracts.Setup, t func() *testing.T) *clients.TestClient {
helper, err := clients.CreateTestClient(kh, namespace)
if err != nil {
logger.Errorf("Cannot create rook test client, er -> %v", err)
t().Fail()
op.TearDown()
t().FailNow()
}
return helper
}
//BaseTestOperations struct for handling panic and test suite tear down
type BaseTestOperations struct {
installer *installer.InstallHelper
//TestCluster struct for handling panic and test suite tear down
type TestCluster struct {
installer *installer.CephInstaller
kh *utils.K8sHelper
helper *clients.TestClient
T func() *testing.T
@@ -110,20 +99,20 @@ type BaseTestOperations struct {
mons int
}
// StartBaseTestOperations creates new instance of BaseTestOperations struct
func StartBaseTestOperations(t func() *testing.T, namespace, storeType string, helmInstalled, useDevices bool, mons int) (BaseTestOperations, *utils.K8sHelper) {
// StartTestCluster creates new instance of TestCluster struct
func StartTestCluster(t func() *testing.T, namespace, storeType string, helmInstalled, useDevices bool, mons int, rookVersion string) (*TestCluster, *utils.K8sHelper) {
kh, err := utils.CreateK8sHelper(t)
require.NoError(t(), err)
i := installer.NewK8sRookhelper(kh.Clientset, t)
i := installer.NewCephInstaller(t, kh.Clientset, rookVersion)
op := BaseTestOperations{i, kh, nil, t, namespace, storeType, helmInstalled, useDevices, mons}
op.SetUp()
op := &TestCluster{i, kh, nil, t, namespace, storeType, helmInstalled, useDevices, mons}
op.Setup()
return op, kh
}
//SetUpRook is a wrapper for setting up rook
func (op BaseTestOperations) SetUp() {
func (op *TestCluster) Setup() {
isRookInstalled, err := op.installer.InstallRookOnK8sWithHostPathAndDevices(op.namespace, op.storeType,
op.helmInstalled, op.useDevices, cephv1beta1.MonSpec{Count: op.mons, AllowMultiplePerNode: true}, false /* startWithAllNodes */)
@@ -133,13 +122,18 @@ func (op BaseTestOperations) SetUp() {
op.installer.GatherAllRookLogs(op.namespace, installer.SystemNamespace(op.namespace), op.installer.T().Name())
}
op.T().Fail()
op.TearDown()
op.Teardown()
op.T().FailNow()
}
}
// SetInstallData updates the installer helper based on the version of Rook desired
func (op *TestCluster) SetInstallData(version string) {
}
//TearDownRook is a wrapper for tearDown after Suite
func (op BaseTestOperations) TearDown() {
func (op *TestCluster) Teardown() {
if op.installer.T().Failed() {
op.installer.GatherAllRookLogs(op.namespace, installer.SystemNamespace(op.namespace), op.installer.T().Name())
}
+42 -36
View File
@@ -23,13 +23,12 @@ import (
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"k8s.io/kubernetes/pkg/util/version"
)
var (
const (
fileMountPath = "/tmp/rookfs"
filePodName = "file-test"
)
@@ -37,21 +36,18 @@ var (
// Smoke Test for File System Storage - Test check the following operations on Filesystem Storage in order
//Create,Mount,Write,Read,Unmount and Delete.
func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) {
defer fileTestDataCleanUp(helper, k8sh, s, filePodName, namespace, filesystemName, fileMountPath)
defer fileTestDataCleanUp(helper, k8sh, s, filePodName, namespace, filesystemName)
logger.Infof("Running on Rook Cluster %s", namespace)
logger.Infof("File Storage End To End Integration Test - create, mount, write to, read from, and unmount")
logger.Infof("Step 1: Create file System")
fscErr := helper.FSClient.Create(filesystemName, namespace)
require.Nil(s.T(), fscErr)
filesystemList, _ := helper.FSClient.List(namespace)
require.Equal(s.T(), 1, len(filesystemList), "There should one shared file system present")
filesystemData := filesystemList[0]
require.Equal(s.T(), filesystemName, filesystemData.Name, "make sure filesystem name matches")
logger.Infof("File system created")
createFilesystem(helper, k8sh, s, namespace, filesystemName)
createFilesystemConsumerPod(helper, k8sh, s, namespace, filesystemName)
writeAndReadToFilesystem(helper, k8sh, s, namespace, "test_file")
cleanupFilesystemConsumer(helper, k8sh, s, namespace, filesystemName)
}
logger.Infof("Step 2: Mount file System")
mtfsErr := podWithFilesystem(k8sh, s, filePodName, namespace, filesystemName, fileMountPath, "create")
func createFilesystemConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) {
mtfsErr := podWithFilesystem(k8sh, s, filePodName, namespace, filesystemName, "create")
require.Nil(s.T(), mtfsErr)
filePodRunning := k8sh.IsPodRunning(filePodName, namespace)
if !filePodRunning {
@@ -61,25 +57,31 @@ func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S
}
require.True(s.T(), filePodRunning, "make sure file-test pod is in running state")
logger.Infof("File system mounted successfully")
}
logger.Infof("Step 3: Write to file system")
_, wfsErr := helper.FSClient.Write(filePodName, fileMountPath, "Smoke Test Data for file system storage", "fsFile1", namespace)
func writeAndReadToFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filename string) {
logger.Infof("Write to file system")
message := "Test Data for file system storage"
_, wfsErr := helper.FSClient.Write(filePodName, message, filename, namespace)
require.Nil(s.T(), wfsErr)
logger.Infof("Write to file system successful")
logger.Infof("Step 4: Read from file system")
read, rdErr := helper.FSClient.Read(filePodName, fileMountPath, "fsFile1", namespace)
logger.Infof("Read from file system")
read, rdErr := helper.FSClient.Read(filePodName, filename, namespace)
require.Nil(s.T(), rdErr)
assert.Contains(s.T(), read, "Smoke Test Data for file system storage", "make sure content of the files is unchanged")
require.Contains(s.T(), read, message)
logger.Infof("Read from file system successful")
}
logger.Infof("Step 5: UnMount file System")
umtfsErr := podWithFilesystem(k8sh, s, filePodName, namespace, filesystemName, fileMountPath, "delete")
require.Nil(s.T(), umtfsErr)
func cleanupFilesystemConsumer(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) {
logger.Infof("Unmount file System")
_, err := k8sh.DeleteResource("pod", filePodName, "-n", namespace)
require.Nil(s.T(), err)
require.True(s.T(), k8sh.IsPodTerminated(filePodName, namespace), "make sure file-test pod is terminated")
logger.Infof("File system mounted successfully")
logger.Infof("File system unmounted successfully")
logger.Infof("Step 6: Deleting file storage")
logger.Infof("Deleting file storage")
helper.FSClient.Delete(filesystemName, namespace)
//Delete is not deleting filesystem - known issue
//require.Nil(suite.T(), fsd_err)
@@ -90,22 +92,26 @@ func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S
func runFileE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) {
logger.Infof("File Storage End to End Integration Test - create Filesystem and make sure mds pod is running")
logger.Infof("Running on Rook Cluster %s", namespace)
logger.Infof("Step 1: Create file System")
fscErr := helper.FSClient.Create(filesystemName, namespace)
require.Nil(s.T(), fscErr)
filesystemList, _ := helper.FSClient.List(namespace)
logger.Infof("Retrieved file systems: %+v", filesystemList)
require.Equal(s.T(), 1, len(filesystemList), "There should one shared file system present")
createFilesystem(helper, k8sh, s, namespace, filesystemName)
}
func fileTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string, fileMountPath string) {
func createFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName string) {
logger.Infof("Create file System")
fscErr := helper.FSClient.Create(filesystemName, namespace)
require.Nil(s.T(), fscErr)
logger.Infof("File system %s created", filesystemName)
filesystemList, _ := helper.FSClient.List(namespace)
require.Equal(s.T(), 1, len(filesystemList), "There should be one shared file system present")
}
func fileTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string) {
logger.Infof("Cleaning up file system")
podWithFilesystem(k8sh, s, podname, namespace, filesystemName, fileMountPath, "delete")
podWithFilesystem(k8sh, s, podname, namespace, filesystemName, "delete")
helper.FSClient.Delete(filesystemName, namespace)
}
func podWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string, fileMountPath string, action string) error {
func podWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string, action string) error {
driverName := installer.SystemNamespace(namespace)
v := version.MustParseSemantic(k8sh.GetK8sServerVersion())
if v.LessThan(version.MustParseSemantic("1.10.0")) {
@@ -113,7 +119,7 @@ func podWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, nam
driverName = flexvolume.FlexDriverName
}
testPod := getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath, driverName)
testPod := getFilesystemTestPod(podname, namespace, filesystemName, driverName)
logger.Infof("creating test pod: %s", testPod)
_, err := k8sh.ResourceOperation(action, testPod)
if err != nil {
@@ -122,7 +128,7 @@ func podWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, nam
return nil
}
func getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath, driverName string) string {
func getFilesystemTestPod(podname, namespace, filesystemName, driverName string) string {
return `apiVersion: v1
kind: Pod
metadata:
@@ -138,7 +144,7 @@ spec:
imagePullPolicy: IfNotPresent
env:
volumeMounts:
- mountPath: "` + fileMountPath + `"
- mountPath: "` + clients.TestMountPath + `"
name: ` + filesystemName + `
volumes:
- name: ` + filesystemName + `
@@ -20,9 +20,7 @@ import (
"fmt"
"testing"
"github.com/rook/rook/pkg/daemon/ceph/model"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/contracts"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
@@ -48,16 +46,16 @@ type BlockCreateSuite struct {
kh *utils.K8sHelper
initBlockCount int
namespace string
installer *installer.InstallHelper
op contracts.Setup
installer *installer.CephInstaller
op *TestCluster
}
func (s *BlockCreateSuite) SetupSuite() {
var err error
s.namespace = "block-k8s-ns"
s.op, s.kh = StartBaseTestOperations(s.T, s.namespace, "bluestore", false, false, 1)
s.testClient = GetTestClient(s.kh, s.namespace, s.op, s.T)
s.op, s.kh = StartTestCluster(s.T, s.namespace, "bluestore", false, false, 1, installer.VersionMaster)
s.testClient = clients.CreateTestClient(s.kh, s.op.installer.Manifests)
initialBlocks, err := s.testClient.BlockClient.List(s.namespace)
assert.Nil(s.T(), err)
s.initBlockCount = len(initialBlocks)
@@ -73,7 +71,7 @@ func (s *BlockCreateSuite) TestCreatePVCWhenNoStorageClassExists() {
storageClassName := "rook-ceph-block"
defer s.tearDownTest(claimName, poolName, storageClassName, "ReadWriteOnce")
result, err := installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(claimName, storageClassName, "ReadWriteOnce"), "create")
result, err := s.testClient.BlockClient.CreatePvc(claimName, storageClassName, "ReadWriteOnce")
checkOrderedSubstrings(s.T(), result, "persistentvolumeclaim", claimName, "created")
require.NoError(s.T(), err)
@@ -105,12 +103,11 @@ func (s *BlockCreateSuite) TestCreateSamePVCTwice() {
s.testClient.BlockClient.List(s.namespace)
logger.Infof("create pool and storageclass")
pool := model.Pool{Name: poolName, ReplicatedConfig: model.ReplicatedPoolConfig{Size: 1}}
result0, err0 := s.testClient.PoolClient.Create(pool, s.namespace)
result0, err0 := s.testClient.PoolClient.Create(poolName, s.namespace, 1)
checkOrderedSubstrings(s.T(), result0, poolName, "created")
require.NoError(s.T(), err0)
result1, err1 := installer.BlockResourceOperation(s.kh, installer.GetBlockStorageClassDef(poolName, storageClassName, s.namespace, true), "create")
result1, err1 := s.testClient.BlockClient.CreateStorageClass(poolName, storageClassName, s.namespace, true)
checkOrderedSubstrings(s.T(), result1, storageClassName, "created")
require.NoError(s.T(), err1)
@@ -119,7 +116,7 @@ func (s *BlockCreateSuite) TestCreateSamePVCTwice() {
require.Nil(s.T(), err)
logger.Infof("create pvc")
result2, err2 := installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(claimName, storageClassName, "ReadWriteOnce"), "create")
result2, err2 := s.testClient.BlockClient.CreatePvc(claimName, storageClassName, "ReadWriteOnce")
checkOrderedSubstrings(s.T(), result2, claimName, "created")
require.NoError(s.T(), err2)
@@ -131,7 +128,7 @@ func (s *BlockCreateSuite) TestCreateSamePVCTwice() {
assert.Equal(s.T(), s.initBlockCount+1, len(b1), "Make sure new block image is created")
logger.Infof("Create same pvc again")
_, err3 := installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(claimName, storageClassName, "ReadWriteOnce"), "create")
_, err3 := s.testClient.BlockClient.CreatePvc(claimName, storageClassName, "ReadWriteOnce")
checkOrderedSubstrings(s.T(), err3.Error(), claimName, "already exists")
logger.Infof("check status of PVC")
@@ -152,7 +149,8 @@ func (s *BlockCreateSuite) TestBlockStorageMountUnMountForStatefulSets() {
defer s.statefulSetDataCleanup(defaultNamespace, poolName, storageClassName, statefulSetName, statefulPodsName)
logger.Infof("Test case when block persistent volumes are scaled up and down along with StatefulSet")
logger.Info("Step 1: Create pool and storageClass")
_, cbErr := installer.BlockResourceOperation(s.kh, installer.GetBlockPoolStorageClass(s.namespace, poolName, storageClassName), "create")
_, cbErr := s.testClient.PoolClient.CreateStorageClass(s.namespace, poolName, storageClassName)
assert.Nil(s.T(), cbErr)
logger.Info("Step 2 : Deploy statefulSet with 1X replication")
service, statefulset := getBlockStatefulSetAndServiceDefinition(defaultNamespace, statefulSetName, statefulPodsName, storageClassName)
@@ -198,19 +196,17 @@ func (s *BlockCreateSuite) statefulSetDataCleanup(namespace, poolName, storageCl
//Delete all PVCs
s.kh.DeletePvcWithLabel(defaultNamespace, statefulSetName)
//Delete storageclass and pool
installer.BlockResourceOperation(s.kh, installer.GetBlockPoolStorageClass(s.namespace, poolName, storageClassName), "delete")
s.testClient.PoolClient.DeleteStorageClass(s.namespace, poolName, storageClassName)
}
func (s *BlockCreateSuite) tearDownTest(claimName string, poolName string, storageClassName string, accessMode string) {
installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(claimName, storageClassName, accessMode), "delete")
installer.BlockResourceOperation(s.kh, installer.GetBlockPoolDef(poolName, s.namespace, "1"), "delete")
installer.BlockResourceOperation(s.kh, installer.GetBlockStorageClassDef(poolName, storageClassName, s.namespace, true), "delete")
s.testClient.BlockClient.DeletePvc(claimName, storageClassName, accessMode)
s.testClient.PoolClient.Delete(poolName, s.namespace)
s.testClient.BlockClient.DeleteStorageClass(poolName, storageClassName, s.namespace)
}
func (s *BlockCreateSuite) TearDownSuite() {
s.op.TearDown()
s.op.Teardown()
}
func (s *BlockCreateSuite) CheckCreatingPVC(pvcName, pvcAccessMode string) {
@@ -221,10 +217,10 @@ func (s *BlockCreateSuite) CheckCreatingPVC(pvcName, pvcAccessMode string) {
defer s.tearDownTest(claimName, poolName, storageClassName, pvcAccessMode)
//create pool and storageclass
result0, err0 := installer.BlockResourceOperation(s.kh, installer.GetBlockPoolDef(poolName, s.namespace, "1"), "create")
result0, err0 := s.testClient.PoolClient.Create(poolName, s.namespace, 1)
checkOrderedSubstrings(s.T(), result0, poolName, "created")
require.NoError(s.T(), err0)
result1, err1 := installer.BlockResourceOperation(s.kh, installer.GetBlockStorageClassDef(poolName, storageClassName, s.namespace, true), "create")
result1, err1 := s.testClient.BlockClient.CreateStorageClass(poolName, storageClassName, s.namespace, true)
checkOrderedSubstrings(s.T(), result1, storageClassName, "created")
require.NoError(s.T(), err1)
@@ -233,7 +229,7 @@ func (s *BlockCreateSuite) CheckCreatingPVC(pvcName, pvcAccessMode string) {
require.Nil(s.T(), err)
//create pvc
result2, err2 := installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(claimName, storageClassName, pvcAccessMode), "create")
result2, err2 := s.testClient.BlockClient.CreatePvc(claimName, storageClassName, pvcAccessMode)
checkOrderedSubstrings(s.T(), result2, claimName, "created")
require.NoError(s.T(), err2)
+20 -20
View File
@@ -49,66 +49,66 @@ func TestCockroachDBSuite(t *testing.T) {
type CockroachDBSuite struct {
suite.Suite
k8sHelper *utils.K8sHelper
installHelper *installer.InstallHelper
k8shelper *utils.K8sHelper
installer *installer.CockroachDBInstaller
namespace string
systemNamespace string
instanceCount int
}
func (suite *CockroachDBSuite) SetupSuite() {
suite.SetUp()
suite.Setup()
}
func (suite *CockroachDBSuite) TearDownSuite() {
suite.TearDown()
suite.Teardown()
}
func (suite *CockroachDBSuite) SetUp() {
func (suite *CockroachDBSuite) Setup() {
suite.namespace = "cockroachdb-ns"
suite.systemNamespace = installer.SystemNamespace(suite.namespace)
suite.instanceCount = 1
k8sHelper, err := utils.CreateK8sHelper(suite.T)
k8shelper, err := utils.CreateK8sHelper(suite.T)
require.NoError(suite.T(), err)
suite.k8sHelper = k8sHelper
suite.k8shelper = k8shelper
k8sversion := suite.k8sHelper.GetK8sServerVersion()
k8sversion := suite.k8shelper.GetK8sServerVersion()
logger.Infof("Installing cockroachdb on k8s %s", k8sversion)
suite.installHelper = installer.NewK8sRookhelper(suite.k8sHelper.Clientset, suite.T)
suite.installer = installer.NewCockroachDBInstaller(suite.k8shelper, suite.T)
err = suite.installHelper.InstallCockroachDB(suite.systemNamespace, suite.namespace, suite.instanceCount)
err = suite.installer.InstallCockroachDB(suite.systemNamespace, suite.namespace, suite.instanceCount)
if err != nil {
logger.Errorf("cockroachdb was not installed successfully: %+v", err)
suite.T().Fail()
suite.TearDown()
suite.Teardown()
suite.T().FailNow()
}
}
func (suite *CockroachDBSuite) TearDown() {
func (suite *CockroachDBSuite) Teardown() {
if suite.T().Failed() {
suite.installHelper.GatherCRDObjectDebuggingInfo(suite.systemNamespace)
suite.installHelper.GatherCRDObjectDebuggingInfo(suite.namespace)
suite.installHelper.GatherAllCockroachDBLogs(suite.systemNamespace, suite.namespace, suite.T().Name())
installer.GatherCRDObjectDebuggingInfo(suite.k8shelper, suite.systemNamespace)
installer.GatherCRDObjectDebuggingInfo(suite.k8shelper, suite.namespace)
suite.installer.GatherAllCockroachDBLogs(suite.systemNamespace, suite.namespace, suite.T().Name())
}
suite.installHelper.UninstallCockroachDB(suite.systemNamespace, suite.namespace)
suite.installer.UninstallCockroachDB(suite.systemNamespace, suite.namespace)
}
func (suite *CockroachDBSuite) TestCockroachDBClusterInstallation() {
logger.Infof("Verifying that all expected pods in cockroachdb cluster %s are running", suite.namespace)
// verify cockroachdb operator is running OK
assert.True(suite.T(), suite.k8sHelper.CheckPodCountAndState("rook-cockroachdb-operator", suite.systemNamespace, 1, "Running"),
assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-cockroachdb-operator", suite.systemNamespace, 1, "Running"),
"1 rook-cockroachdb-operator must be in Running state")
// verify cockroachdb cluster instances are running OK
assert.True(suite.T(), suite.k8sHelper.CheckPodCountAndState("rook-cockroachdb", suite.namespace, suite.instanceCount, "Running"),
assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-cockroachdb", suite.namespace, suite.instanceCount, "Running"),
fmt.Sprintf("%d rook-cockroachdb pods must be in Running state", suite.instanceCount))
// determine the cockroachdb operator pod name
podNames, err := suite.k8sHelper.GetPodNamesForApp("rook-cockroachdb-operator", suite.systemNamespace)
podNames, err := suite.k8shelper.GetPodNamesForApp("rook-cockroachdb-operator", suite.systemNamespace)
require.NoError(suite.T(), err)
require.Equal(suite.T(), 1, len(podNames))
operatorPodName := podNames[0]
@@ -123,7 +123,7 @@ func (suite *CockroachDBSuite) TestCockroachDBClusterInstallation() {
inc := 0
var result string
for inc < utils.RetryLoop {
result, err = suite.k8sHelper.Exec(suite.systemNamespace, operatorPodName, command, commandArgs)
result, err = suite.k8shelper.Exec(suite.systemNamespace, operatorPodName, command, commandArgs)
logger.Infof("cockroachdb sql command exited, err: %+v. result: %s", err, result)
if err == nil {
break
+5 -5
View File
@@ -20,7 +20,7 @@ import (
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/contracts"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/suite"
)
@@ -57,18 +57,18 @@ type HelmSuite struct {
suite.Suite
helper *clients.TestClient
kh *utils.K8sHelper
op contracts.Setup
op *TestCluster
namespace string
}
func (hs *HelmSuite) SetupSuite() {
hs.namespace = "helm-ns"
hs.op, hs.kh = StartBaseTestOperations(hs.T, hs.namespace, "bluestore", true, false, 1)
hs.helper = GetTestClient(hs.kh, hs.namespace, hs.op, hs.T)
hs.op, hs.kh = StartTestCluster(hs.T, hs.namespace, "bluestore", true, false, 1, installer.VersionMaster)
hs.helper = clients.CreateTestClient(hs.kh, hs.op.installer.Manifests)
}
func (hs *HelmSuite) TearDownSuite() {
hs.op.TearDown()
hs.op.Teardown()
}
//Test to make sure all rook components are installed and Running
@@ -22,7 +22,6 @@ import (
"time"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/contracts"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/require"
@@ -55,12 +54,11 @@ func TestMultiClusterDeploySuite(t *testing.T) {
type MultiClusterDeploySuite struct {
suite.Suite
helper1 *clients.TestClient
helper2 *clients.TestClient
testClient *clients.TestClient
k8sh *utils.K8sHelper
namespace1 string
namespace2 string
op contracts.Setup
op *MCTestOperations
}
//Deploy Multiple Rook clusters
@@ -70,89 +68,84 @@ func (mrc *MultiClusterDeploySuite) SetupSuite() {
mrc.namespace2 = "mrc-n2"
mrc.op, mrc.k8sh = NewMCTestOperations(mrc.T, mrc.namespace1, mrc.namespace2)
mrc.helper1 = GetTestClient(mrc.k8sh, mrc.namespace1, mrc.op, mrc.T)
mrc.helper2 = GetTestClient(mrc.k8sh, mrc.namespace2, mrc.op, mrc.T)
mrc.testClient = clients.CreateTestClient(mrc.k8sh, mrc.op.installer.Manifests)
mrc.createPools()
}
func (mrc *MultiClusterDeploySuite) createPools() {
// create a test pool in each cluster so that we get some PGs
poolName := "multi-cluster-pool1"
logger.Infof("Creating pool %s", poolName)
result, err := installer.BlockResourceOperation(mrc.k8sh, installer.GetBlockPoolDef(poolName, mrc.namespace1, "1"), "create")
result, err := mrc.testClient.PoolClient.Create(poolName, mrc.namespace1, 1)
checkOrderedSubstrings(mrc.T(), result, poolName, "created")
require.Nil(mrc.T(), err)
poolName = "multi-cluster-pool2"
logger.Infof("Creating pool %s", poolName)
result, err = installer.BlockResourceOperation(mrc.k8sh, installer.GetBlockPoolDef(poolName, mrc.namespace2, "1"), "create")
result, err = mrc.testClient.PoolClient.Create(poolName, mrc.namespace2, 1)
checkOrderedSubstrings(mrc.T(), result, poolName, "created")
require.Nil(mrc.T(), err)
}
func (mrc *MultiClusterDeploySuite) TearDownSuite() {
mrc.op.TearDown()
mrc.op.Teardown()
}
//Test to make sure all rook components are installed and Running
func (mrc *MultiClusterDeploySuite) TestInstallingMultipleRookClusters() {
//Check if Rook cluster 1 is deployed successfully
checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace1, 1)
checkIfRookClusterIsHealthy(mrc.Suite, mrc.helper1, mrc.namespace1)
checkIfRookClusterIsHealthy(mrc.Suite, mrc.testClient, mrc.namespace1)
//Check if Rook cluster 2 is deployed successfully
checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace2, 1)
checkIfRookClusterIsHealthy(mrc.Suite, mrc.helper2, mrc.namespace2)
checkIfRookClusterIsHealthy(mrc.Suite, mrc.testClient, mrc.namespace2)
}
//Test Block Store Creation on multiple rook clusters
func (mrc *MultiClusterDeploySuite) TestBlockStoreOnMultipleRookCluster() {
runBlockE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, mrc.namespace1)
runBlockE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, mrc.namespace2)
runBlockE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace1)
runBlockE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace2)
}
//Test Filesystem Creation on multiple rook clusters
func (mrc *MultiClusterDeploySuite) TestFileStoreOnMultiRookCluster() {
runFileE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, mrc.namespace1, "test-fs-1")
runFileE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, mrc.namespace2, "test-fs-2")
runFileE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace1, "test-fs-1")
runFileE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace2, "test-fs-2")
}
//Test Object Store Creation on multiple rook clusters
func (mrc *MultiClusterDeploySuite) TestObjectStoreOnMultiRookCluster() {
runObjectE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, mrc.namespace1, "default-c1", 2)
runObjectE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, mrc.namespace2, "default-c2", 1)
runObjectE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace1, "default-c1", 2)
runObjectE2ETestLite(mrc.testClient, mrc.k8sh, mrc.Suite, mrc.namespace2, "default-c2", 1)
}
//MCTestOperations struct for handling panic and test suite tear down
type MCTestOperations struct {
installer *installer.InstallHelper
installData *installer.InstallData
installer *installer.CephInstaller
kh *utils.K8sHelper
T func() *testing.T
namespace1 string
namespace2 string
systemNamespace string
helper1 *clients.TestClient
helper2 *clients.TestClient
}
//NewMCTestOperations creates new instance of BaseTestOperations struct
func NewMCTestOperations(t func() *testing.T, namespace1 string, namespace2 string) (MCTestOperations, *utils.K8sHelper) {
//NewMCTestOperations creates new instance of TestCluster struct
func NewMCTestOperations(t func() *testing.T, namespace1 string, namespace2 string) (*MCTestOperations, *utils.K8sHelper) {
kh, err := utils.CreateK8sHelper(t)
require.NoError(t(), err)
i := installer.NewK8sRookhelper(kh.Clientset, t)
d := installer.NewK8sInstallData()
i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster)
op := MCTestOperations{i, d, kh, t, namespace1, namespace2, installer.SystemNamespace(namespace1), nil, nil}
op.SetUp()
op := &MCTestOperations{i, kh, t, namespace1, namespace2, installer.SystemNamespace(namespace1)}
op.Setup()
return op, kh
}
//SetUpRook is wrapper for setting up multiple rook clusters.
func (o MCTestOperations) SetUp() {
func (o MCTestOperations) Setup() {
var err error
err = o.installer.CreateK8sRookOperator(installer.SystemNamespace(o.namespace1))
err = o.installer.CreateCephOperator(installer.SystemNamespace(o.namespace1))
require.NoError(o.T(), err)
require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-operator", o.systemNamespace, "Running"),
@@ -179,7 +172,7 @@ func (o MCTestOperations) SetUp() {
}
//TearDownRook is a wrapper for tearDown after suite
func (o MCTestOperations) TearDown() {
func (o MCTestOperations) Teardown() {
defer func() {
if r := recover(); r != nil {
logger.Infof("Unexpected Errors while cleaning up MultiCluster test --> %v", r)
+20 -20
View File
@@ -47,8 +47,8 @@ func TestNfsSuite(t *testing.T) {
type NfsSuite struct {
suite.Suite
k8sHelper *utils.K8sHelper
installHelper *installer.InstallHelper
k8shelper *utils.K8sHelper
installer *installer.NFSInstaller
rwClient *clients.ReadWriteOperation
namespace string
systemNamespace string
@@ -56,60 +56,60 @@ type NfsSuite struct {
}
func (suite *NfsSuite) SetupSuite() {
suite.SetUp()
suite.Setup()
}
func (suite *NfsSuite) TearDownSuite() {
suite.TearDown()
suite.Teardown()
}
func (suite *NfsSuite) SetUp() {
func (suite *NfsSuite) Setup() {
suite.namespace = "nfs-ns"
suite.systemNamespace = installer.SystemNamespace(suite.namespace)
suite.instanceCount = 1
k8sHelper, err := utils.CreateK8sHelper(suite.T)
k8shelper, err := utils.CreateK8sHelper(suite.T)
require.NoError(suite.T(), err)
suite.k8sHelper = k8sHelper
suite.k8shelper = k8shelper
k8sversion := suite.k8sHelper.GetK8sServerVersion()
k8sversion := suite.k8shelper.GetK8sServerVersion()
logger.Infof("Installing nfs server on k8s %s", k8sversion)
suite.installHelper = installer.NewK8sRookhelper(suite.k8sHelper.Clientset, suite.T)
suite.installer = installer.NewNFSInstaller(suite.k8shelper, suite.T)
suite.rwClient = clients.CreateReadWriteOperation(suite.k8sHelper)
suite.rwClient = clients.CreateReadWriteOperation(suite.k8shelper)
err = suite.installHelper.InstallNFSServer(suite.systemNamespace, suite.namespace, suite.instanceCount)
err = suite.installer.InstallNFSServer(suite.systemNamespace, suite.namespace, suite.instanceCount)
if err != nil {
logger.Errorf("nfs server installation failed: %+v", err)
suite.T().Fail()
suite.TearDown()
suite.Teardown()
suite.T().FailNow()
}
}
func (suite *NfsSuite) TearDown() {
func (suite *NfsSuite) Teardown() {
if suite.T().Failed() {
suite.installHelper.GatherCRDObjectDebuggingInfo(suite.systemNamespace)
suite.installHelper.GatherCRDObjectDebuggingInfo(suite.namespace)
suite.installHelper.GatherAllNFSServerLogs(suite.systemNamespace, suite.namespace, suite.T().Name())
installer.GatherCRDObjectDebuggingInfo(suite.k8shelper, suite.systemNamespace)
installer.GatherCRDObjectDebuggingInfo(suite.k8shelper, suite.namespace)
suite.installer.GatherAllNFSServerLogs(suite.systemNamespace, suite.namespace, suite.T().Name())
}
suite.installHelper.UninstallNFSServer(suite.systemNamespace, suite.namespace)
suite.installer.UninstallNFSServer(suite.systemNamespace, suite.namespace)
}
func (suite *NfsSuite) TestNfsServerInstallation() {
logger.Infof("Verifying that nfs server pod %s is running", suite.namespace)
// verify nfs server operator is running OK
assert.True(suite.T(), suite.k8sHelper.CheckPodCountAndState("rook-nfs-operator", suite.systemNamespace, 1, "Running"),
assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-nfs-operator", suite.systemNamespace, 1, "Running"),
"1 rook-nfs-operator must be in Running state")
// verify nfs server instances are running OK
assert.True(suite.T(), suite.k8sHelper.CheckPodCountAndState("rook-nfs", suite.namespace, suite.instanceCount, "Running"),
assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-nfs", suite.namespace, suite.instanceCount, "Running"),
fmt.Sprintf("%d rook-nfs pods must be in Running state", suite.instanceCount))
// verify nfs server storage
assert.True(suite.T(), true, suite.k8sHelper.WaitUntilPVCIsBound("default", "nfs-pv-claim"))
assert.True(suite.T(), true, suite.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim"))
defer suite.rwClient.Delete()
podList, err := suite.rwClient.CreateWriteClient("nfs-pv-claim")
+2 -5
View File
@@ -20,7 +20,6 @@ import (
"fmt"
"time"
"github.com/rook/rook/pkg/daemon/ceph/model"
"github.com/stretchr/testify/require"
)
@@ -29,8 +28,7 @@ func (suite *SmokeSuite) TestPoolResize() {
logger.Infof("Pool Resize Smoke Test")
poolName := "testpool"
pool := model.Pool{Name: "testpool", ReplicatedConfig: model.ReplicatedPoolConfig{Size: 1}}
out, err := suite.helper.PoolClient.Create(pool, suite.namespace)
out, err := suite.helper.PoolClient.Create(poolName, suite.namespace, 1)
logger.Infof("poolCreate: %+v", out)
require.Nil(suite.T(), err)
@@ -56,8 +54,7 @@ func (suite *SmokeSuite) TestPoolResize() {
require.Equal(suite.T(), true, poolFound, "pool not found")
pool.ReplicatedConfig.Size = 3
_, err = suite.helper.PoolClient.Update(pool, suite.namespace)
_, err = suite.helper.PoolClient.Update(poolName, suite.namespace, 3)
logger.Infof("poolCreate (modify): %+v", out)
require.Nil(suite.T(), err)
+4 -5
View File
@@ -23,7 +23,6 @@ import (
"testing"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/contracts"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
@@ -69,7 +68,7 @@ func TestSmokeSuite(t *testing.T) {
type SmokeSuite struct {
suite.Suite
helper *clients.TestClient
op contracts.Setup
op *TestCluster
k8sh *utils.K8sHelper
namespace string
}
@@ -77,12 +76,12 @@ type SmokeSuite struct {
func (suite *SmokeSuite) SetupSuite() {
suite.namespace = "smoke-ns"
useDevices := true
suite.op, suite.k8sh = StartBaseTestOperations(suite.T, suite.namespace, "bluestore", false, useDevices, 3)
suite.helper = GetTestClient(suite.k8sh, suite.namespace, suite.op, suite.T)
suite.op, suite.k8sh = StartTestCluster(suite.T, suite.namespace, "bluestore", false, useDevices, 3, installer.VersionMaster)
suite.helper = clients.CreateTestClient(suite.k8sh, suite.op.installer.Manifests)
}
func (suite *SmokeSuite) TearDownSuite() {
suite.op.TearDown()
suite.op.Teardown()
}
func (suite *SmokeSuite) TestBlockStorage_SmokeTest() {
+70 -95
View File
@@ -22,7 +22,6 @@ import (
"fmt"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/contracts"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
@@ -76,8 +75,8 @@ type BlockMountUnMountSuite struct {
namespace string
pvcNameRWO string
pvcNameRWX string
installer *installer.InstallHelper
op contracts.Setup
installer *installer.CephInstaller
op *TestCluster
}
func (s *BlockMountUnMountSuite) SetupSuite() {
@@ -87,8 +86,8 @@ func (s *BlockMountUnMountSuite) SetupSuite() {
s.pvcNameRWX = "block-persistent-rwx"
useHelm := false
useDevices := true
s.op, s.kh = StartBaseTestOperations(s.T, s.namespace, "filestore", useHelm, useDevices, 1)
s.testClient = GetTestClient(s.kh, s.namespace, s.op, s.T)
s.op, s.kh = StartTestCluster(s.T, s.namespace, "filestore", useHelm, useDevices, 1, installer.VersionMaster)
s.testClient = clients.CreateTestClient(s.kh, s.op.installer.Manifests)
s.bc = s.testClient.BlockClient
}
@@ -98,16 +97,16 @@ func (s *BlockMountUnMountSuite) setupPVCs() {
storageClassNameRWO := "rook-ceph-block-rwo"
// Create PVCs
_, cbErr := installer.BlockResourceOperation(s.kh, installer.GetBlockPoolStorageClassAndPvcDef(s.namespace, poolNameRWO, storageClassNameRWO, s.pvcNameRWO, "ReadWriteOnce"), "create")
_, cbErr := s.testClient.PoolClient.CreateStorageClassAndPvc(s.namespace, poolNameRWO, storageClassNameRWO, s.pvcNameRWO, "ReadWriteOnce")
require.Nil(s.T(), cbErr)
require.True(s.T(), s.kh.WaitUntilPVCIsBound(defaultNamespace, s.pvcNameRWO), "Make sure PVC is Bound")
_, cbErr2 := installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef(s.pvcNameRWX, storageClassNameRWO, "ReadWriteMany"), "create")
_, cbErr2 := s.testClient.BlockClient.CreatePvc(s.pvcNameRWX, storageClassNameRWO, "ReadWriteMany")
require.Nil(s.T(), cbErr2)
require.True(s.T(), s.kh.WaitUntilPVCIsBound(defaultNamespace, s.pvcNameRWX), "Make sure PVC is Bound")
// Mount PVC on a pod and write some data.
_, mtErr := s.bc.BlockMap(getBlockPodDefintion("setup-block-rwo", s.pvcNameRWO, false), blockMountPath)
_, mtErr := s.bc.BlockMap(getBlockPodDefintion("setup-block-rwo", s.pvcNameRWO, false))
require.Nil(s.T(), mtErr)
crdName, err := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWO)
require.Nil(s.T(), err)
@@ -119,7 +118,7 @@ func (s *BlockMountUnMountSuite) setupPVCs() {
}
require.True(s.T(), rwoVolumePresent, fmt.Sprintf("make sure rwo Volume %s is created", crdName))
_, mtErr1 := s.bc.BlockMap(getBlockPodDefintion("setup-block-rwx", s.pvcNameRWX, false), blockMountPath)
_, mtErr1 := s.bc.BlockMap(getBlockPodDefintion("setup-block-rwx", s.pvcNameRWX, false))
require.Nil(s.T(), mtErr1)
crdName1, err1 := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWX)
require.Nil(s.T(), err1)
@@ -128,37 +127,29 @@ func (s *BlockMountUnMountSuite) setupPVCs() {
require.True(s.T(), s.kh.IsPodRunning("setup-block-rwx", defaultNamespace), "make sure setup-block-rwx pod is in running state")
// Write Data to Pod
_, wtErr1 := s.bc.Write("setup-block-rwo", blockMountPath, "Persisted message one", "bsFile1", "")
_, wtErr1 := s.bc.Write("setup-block-rwo", "Persisted message one", "bsFile1", "")
require.Nil(s.T(), wtErr1)
_, wtErr2 := s.bc.Write("setup-block-rwx", blockMountPath, "Persisted message one", "bsFile1", "")
_, wtErr2 := s.bc.Write("setup-block-rwx", "Persisted message one", "bsFile1", "")
require.Nil(s.T(), wtErr2)
// Unmount pod
_, unmtErr1 := s.bc.BlockUnmap(getBlockPodDefintion("setup-block-rwo", s.pvcNameRWO, false), blockMountPath)
_, unmtErr2 := s.bc.BlockUnmap(getBlockPodDefintion("setup-block-rwx", s.pvcNameRWO, false), blockMountPath)
_, unmtErr1 := s.kh.DeletePods("setup-block-rwo", "setup-block-rwx")
require.Nil(s.T(), unmtErr1)
require.Nil(s.T(), unmtErr2)
require.True(s.T(), s.kh.IsPodTerminated("setup-block-rwo", defaultNamespace), "make sure setup-block-rwo pod is terminated")
require.True(s.T(), s.kh.IsPodTerminated("setup-block-rwx", defaultNamespace), "make sure setup-block-rwx pod is terminated")
}
func (s *BlockMountUnMountSuite) TearDownSuite() {
logger.Infof("Cleaning up block storage")
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("setup-block-rwo", s.pvcNameRWO, false), blockMountPath)
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("setup-block-rwx", s.pvcNameRWX, false), blockMountPath)
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwo-block-rw-one", s.pvcNameRWO, false), blockMountPath)
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwo-block-rw-two", s.pvcNameRWO, false), blockMountPath)
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true), blockMountPath)
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwo-block-ro-two", s.pvcNameRWO, true), blockMountPath)
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwx-block-rw-one", s.pvcNameRWX, false), blockMountPath)
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwx-block-rw-two", s.pvcNameRWX, false), blockMountPath)
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true), blockMountPath)
s.testClient.BlockClient.BlockUnmap(getBlockPodDefintion("rwx-block-ro-two", s.pvcNameRWX, true), blockMountPath)
installer.BlockResourceOperation(s.kh, installer.GetBlockPoolStorageClassAndPvcDef(s.namespace, "block-pool-rwo", "rook-ceph-block-rwo", s.pvcNameRWO, "ReadWriteOnce"), "delete")
installer.BlockResourceOperation(s.kh, installer.GetBlockPoolStorageClassAndPvcDef(s.namespace, "block-pool-rwx", "rook-ceph-block-rwx", s.pvcNameRWX, "ReadWriteMany"), "delete")
s.kh.DeletePods(
"setup-block-rwo", "setup-block-rwx", "rwo-block-rw-one", "rwo-block-rw-two", "rwo-block-ro-one",
"rwo-block-ro-two", "rwx-block-rw-one", "rwx-block-rw-two", "rwx-block-ro-one", "rwx-block-ro-two")
s.testClient.PoolClient.DeleteStorageClassAndPvc(s.namespace, "block-pool-rwo", "rook-ceph-block-rwo", s.pvcNameRWO, "ReadWriteOnce")
s.testClient.PoolClient.DeleteStorageClassAndPvc(s.namespace, "block-pool-rwx", "rook-ceph-block-rwx", s.pvcNameRWX, "ReadWriteMany")
cleanupDynamicBlockStorage(s.testClient, s.namespace)
s.op.TearDown()
s.op.Teardown()
}
func (s *BlockMountUnMountSuite) TestBlockStorageMountUnMountForDifferentAccessModes() {
@@ -167,11 +158,11 @@ func (s *BlockMountUnMountSuite) TestBlockStorageMountUnMountForDifferentAccessM
logger.Infof("Test case when existing RWO PVC is mounted and unmounted on pods with various accessModes")
logger.Infof("Step 1.1: Mount existing ReadWriteOnce and ReadWriteMany PVC on a Pod with RW access")
//mount PVC with RWO access on a pod with readonly set to false
_, mtErr1 := s.bc.BlockMap(getBlockPodDefintion("rwo-block-rw-one", s.pvcNameRWO, false), blockMountPath)
require.Nil(s.T(), mtErr1)
_, err := s.bc.BlockMap(getBlockPodDefintion("rwo-block-rw-one", s.pvcNameRWO, false))
require.Nil(s.T(), err)
//mount PVC with RWX access on a pod with readonly set to false
_, mtErr2 := s.bc.BlockMap(getBlockPodDefintion("rwx-block-rw-one", s.pvcNameRWX, false), blockMountPath)
require.Nil(s.T(), mtErr2)
_, err = s.bc.BlockMap(getBlockPodDefintion("rwx-block-rw-one", s.pvcNameRWX, false))
require.Nil(s.T(), err)
crdName1, err1 := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWO)
crdName2, err2 := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWX)
assert.Nil(s.T(), err1)
@@ -184,85 +175,76 @@ func (s *BlockMountUnMountSuite) TestBlockStorageMountUnMountForDifferentAccessM
logger.Infof("Step 2: Check if previously persisted data is readable from ReadWriteOnce and ReadWriteMany PVC")
//Read data on RWO PVC Mounted on pod with RW Access
read1, rErr1 := s.bc.Read("rwo-block-rw-one", blockMountPath, "bsFile1", "")
assert.Nil(s.T(), rErr1)
read1, err := s.bc.Read("rwo-block-rw-one", "bsFile1", "")
assert.Nil(s.T(), err)
assert.Contains(s.T(), read1, "Persisted message one", "make sure previously persisted data is readable")
//Read data on RWX PVC Mounted on pod with RW Access
read2, rErr2 := s.bc.Read("rwx-block-rw-one", blockMountPath, "bsFile1", "")
assert.Nil(s.T(), rErr2)
read2, err := s.bc.Read("rwx-block-rw-one", "bsFile1", "")
assert.Nil(s.T(), err)
assert.Contains(s.T(), read2, "Persisted message one", "make sure previously persisted data is readable")
logger.Infof("Step 3: Check if read/write works on ReadWriteOnce and ReadWriteMany PVC")
//Write data on RWO PVC Mounted on pod with RW Access
_, wtErr1 := s.bc.Write("rwo-block-rw-one", blockMountPath, "Persisted message two", "bsFile2", "")
assert.Nil(s.T(), wtErr1)
_, err = s.bc.Write("rwo-block-rw-one", "Persisted message two", "bsFile2", "")
assert.Nil(s.T(), err)
//Read data on RWO PVC Mounted on pod with RW Access
read1, rErr1 = s.bc.Read("rwo-block-rw-one", blockMountPath, "bsFile2", "")
assert.Nil(s.T(), rErr1)
read1, err = s.bc.Read("rwo-block-rw-one", "bsFile2", "")
assert.Nil(s.T(), err)
assert.Contains(s.T(), read1, "Persisted message two", "make sure new persisted data is readable")
//Write data on RWX PVC Mounted on pod with RW Access
_, wtErr2 := s.bc.Write("rwx-block-rw-one", blockMountPath, "Persisted message two", "bsFile2", "")
assert.Nil(s.T(), wtErr2)
_, err = s.bc.Write("rwx-block-rw-one", "Persisted message two", "bsFile2", "")
assert.Nil(s.T(), err)
//Read data on RWX PVC Mounted on pod with RW Access
read2, rErr2 = s.bc.Read("rwx-block-rw-one", blockMountPath, "bsFile2", "")
assert.Nil(s.T(), rErr2)
read2, err = s.bc.Read("rwx-block-rw-one", "bsFile2", "")
assert.Nil(s.T(), err)
assert.Contains(s.T(), read2, "Persisted message two", "make sure new persisted data is readable")
//Mount another Pod with RW access on same PVC
logger.Infof("Step 4: Mount existing ReadWriteOnce and ReadWriteMany PVC on a new Pod with RW access")
//Mount RWO PVC on a new pod with ReadOnly set to false
_, mtErr1 = s.bc.BlockMap(getBlockPodDefintion("rwo-block-rw-two", s.pvcNameRWO, false), blockMountPath)
assert.Nil(s.T(), mtErr1)
_, err = s.bc.BlockMap(getBlockPodDefintion("rwo-block-rw-two", s.pvcNameRWO, false))
assert.Nil(s.T(), err)
//Mount RWX PVC on a new pod with ReadOnly set to false
_, mtErr2 = s.bc.BlockMap(getBlockPodDefintion("rwx-block-rw-two", s.pvcNameRWX, false), blockMountPath)
assert.Nil(s.T(), mtErr2)
_, err = s.bc.BlockMap(getBlockPodDefintion("rwx-block-rw-two", s.pvcNameRWX, false))
assert.Nil(s.T(), err)
assert.True(s.T(), s.kh.IsPodInError("rwo-block-rw-two", defaultNamespace, "FailedMount", "Volume is already attached by pod"), "make sure rwo-block-rw-two pod errors out while mounting the volume")
assert.True(s.T(), s.kh.IsPodInError("rwx-block-rw-two", defaultNamespace, "FailedMount", "Volume is already attached by pod"), "make sure rwx-block-rw-two pod errors out while mounting the volume")
_, unmtErr1 := s.bc.BlockUnmap(getBlockPodDefintion("rwo-block-rw-two", s.pvcNameRWO, false), blockMountPath)
assert.Nil(s.T(), unmtErr1)
_, unmtErr2 := s.bc.BlockUnmap(getBlockPodDefintion("rwx-block-rw-two", s.pvcNameRWX, false), blockMountPath)
assert.Nil(s.T(), unmtErr2)
_, err = s.kh.DeletePods("rwo-block-rw-two", "rwx-block-rw-two")
assert.Nil(s.T(), err)
assert.True(s.T(), s.kh.IsPodTerminated("rwo-block-rw-two", defaultNamespace), "make sure rwo-block-rw-two pod is terminated")
assert.True(s.T(), s.kh.IsPodTerminated("rwx-block-rw-two", defaultNamespace), "make sure rwx-block-rw-two pod is terminated")
logger.Infof("Step 5: Mount existing ReadWriteOnce and ReadWriteMany PVC on a new Pod with RO access")
//Mount RWO PVC on a new pod with ReadOnly set to true
_, mtErr1 = s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true), blockMountPath)
assert.Nil(s.T(), mtErr1)
_, err = s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true))
assert.Nil(s.T(), err)
//Mount RWX PVC on a new pod with ReadOnly set to true
_, mtErr2 = s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true), blockMountPath)
assert.Nil(s.T(), mtErr2)
_, err = s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true))
assert.Nil(s.T(), err)
assert.True(s.T(), s.kh.IsPodInError("rwo-block-ro-one", defaultNamespace, "FailedMount", "Volume is already attached by pod"), "make sure rwo-block-ro-one pod errors out while mounting the volume")
assert.True(s.T(), s.kh.IsPodInError("rwx-block-ro-one", defaultNamespace, "FailedMount", "Volume is already attached by pod"), "make sure rwx-block-ro-one pod errors out while mounting the volume")
_, unmtErr1 = s.bc.BlockUnmap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true), blockMountPath)
assert.Nil(s.T(), unmtErr1)
_, unmtErr2 = s.bc.BlockUnmap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true), blockMountPath)
assert.Nil(s.T(), unmtErr2)
_, err = s.kh.DeletePods("rwo-block-ro-one", "rwx-block-ro-one")
assert.Nil(s.T(), err)
assert.True(s.T(), s.kh.IsPodTerminated("rwo-block-ro-one", defaultNamespace), "make sure rwo-block-ro-one pod is terminated")
assert.True(s.T(), s.kh.IsPodTerminated("rwx-block-ro-one", defaultNamespace), "make sure rwx-block-ro-one pod is terminated")
logger.Infof("Step 6: UnMount Pod with RWX and RWO access")
//UnMount RWO PVC
_, unmtErr1 = s.bc.BlockUnmap(getBlockPodDefintion("rwo-block-rw-one", s.pvcNameRWO, false), blockMountPath)
assert.Nil(s.T(), unmtErr1)
//UnMount RWX PVC
_, unmtErr2 = s.bc.BlockUnmap(getBlockPodDefintion("rwx-block-rw-one", s.pvcNameRWX, false), blockMountPath)
assert.Nil(s.T(), unmtErr2)
_, err = s.kh.DeletePods("rwo-block-rw-one", "rwx-block-rw-one")
assert.Nil(s.T(), err)
assert.True(s.T(), s.kh.IsPodTerminated("rwo-block-rw-one", defaultNamespace), "make sure rwo-block-rw-one pod is terminated")
assert.True(s.T(), s.kh.IsPodTerminated("rwx-block-rw-one", defaultNamespace), "make sure rwx-lock-rw-one pod is terminated")
logger.Infof("Step 7: Mount ReadWriteOnce and ReadWriteMany PVC on two different pods with ReadOnlyMany with Readonly Access")
//Mount RWO PVC on 2 pods with ReadOnly set to True
_, mtErr1_1 := s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true), blockMountPath)
_, mtErr2_1 := s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-two", s.pvcNameRWO, true), blockMountPath)
assert.Nil(s.T(), mtErr1_1)
assert.Nil(s.T(), mtErr2_1)
_, err1 = s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true))
_, err2 = s.bc.BlockMap(getBlockPodDefintion("rwo-block-ro-two", s.pvcNameRWO, true))
assert.Nil(s.T(), err1)
assert.Nil(s.T(), err2)
//Mount RWX PVC on 2 pods with ReadOnly set to True
_, mtErr1_2 := s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true), blockMountPath)
_, mtErr2_2 := s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-two", s.pvcNameRWX, true), blockMountPath)
assert.Nil(s.T(), mtErr1_2)
assert.Nil(s.T(), mtErr2_2)
_, err1 = s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true))
_, err2 = s.bc.BlockMap(getBlockPodDefintion("rwx-block-ro-two", s.pvcNameRWX, true))
assert.Nil(s.T(), err1)
assert.Nil(s.T(), err2)
assert.True(s.T(), s.kh.IsPodRunning("rwo-block-ro-one", defaultNamespace), "make sure rwo-block-ro-one pod is in running state")
assert.True(s.T(), s.kh.IsPodRunning("rwo-block-ro-two", defaultNamespace), "make sure rwo-block-ro-two pod is in running state")
assert.True(s.T(), s.kh.IsPodRunning("rwx-block-ro-one", defaultNamespace), "make sure rwx-block-ro-one pod is in running state")
@@ -270,39 +252,32 @@ func (s *BlockMountUnMountSuite) TestBlockStorageMountUnMountForDifferentAccessM
logger.Infof("Step 8: Read Data from both ReadyOnlyMany and ReadWriteOnce pods with ReadOnly Access")
//Read data from RWO PVC via both ReadOnly pods
read1_1, rErr1_1 := s.bc.Read("rwo-block-ro-one", blockMountPath, "bsFile1", "")
read2_1, rErr2_1 := s.bc.Read("rwo-block-ro-two", blockMountPath, "bsFile1", "")
assert.Nil(s.T(), rErr1_1)
assert.Nil(s.T(), rErr2_1)
read1_1, err1 := s.bc.Read("rwo-block-ro-one", "bsFile1", "")
read2_1, err2 := s.bc.Read("rwo-block-ro-two", "bsFile1", "")
assert.Nil(s.T(), err1)
assert.Nil(s.T(), err2)
assert.Contains(s.T(), read1_1, "Persisted message one", "make sure previously persisted data is readable from ReadOnlyMany access Pod")
assert.Contains(s.T(), read2_1, "Persisted message one", "make sure previously persisted data is readable from ReadOnlyMany access Pod")
//Read data from RWX PVC via both ReadOnly pods
read1_2, rErr1_2 := s.bc.Read("rwx-block-ro-one", blockMountPath, "bsFile1", "")
read2_2, rErr2_2 := s.bc.Read("rwx-block-ro-two", blockMountPath, "bsFile1", "")
assert.Nil(s.T(), rErr1_2)
assert.Nil(s.T(), rErr2_2)
read1_2, err1 := s.bc.Read("rwx-block-ro-one", "bsFile1", "")
read2_2, err2 := s.bc.Read("rwx-block-ro-two", "bsFile1", "")
assert.Nil(s.T(), err1)
assert.Nil(s.T(), err2)
assert.Contains(s.T(), read1_2, "Persisted message one", "make sure previously persisted data is readable from ReadOnlyMany access Pod")
assert.Contains(s.T(), read2_2, "Persisted message one", "make sure previously persisted data is readable from ReadOnlyMany access Pod")
logger.Infof("Step 9: Write Data to Pod with ReadOnlyMany and ReadWriteOnce PVC mounted with ReadOnly access")
//Write data to RWO PVC via pod with ReadOnly Set to true
_, wtErr1 = s.bc.Write("rwo-block-ro-one", blockMountPath, "Persisted message three", "bsFile3", "")
assert.Contains(s.T(), wtErr1.Error(), "Unable to write data to pod")
_, err = s.bc.Write("rwo-block-ro-one", "Persisted message three", "bsFile3", "")
assert.Contains(s.T(), err.Error(), "Unable to write data to pod")
//Write data to RWx PVC via pod with ReadOnly Set to true
_, wtErr2 = s.bc.Write("rwx-block-ro-one", blockMountPath, "Persisted message three", "bsFile3", "")
assert.Contains(s.T(), wtErr2.Error(), "Unable to write data to pod")
_, err = s.bc.Write("rwx-block-ro-one", "Persisted message three", "bsFile3", "")
assert.Contains(s.T(), err.Error(), "Unable to write data to pod")
logger.Infof("Step 10: UnMount Pod with ReadOnlyMany and ReadWriteOnce PVCs")
//UnMount RWO PVC from both ReadOnly Pods
_, unmtErr1_1 := s.bc.BlockUnmap(getBlockPodDefintion("rwo-block-ro-one", s.pvcNameRWO, true), blockMountPath)
_, unmtErr2_1 := s.bc.BlockUnmap(getBlockPodDefintion("rwo-block-ro-two", s.pvcNameRWO, true), blockMountPath)
assert.Nil(s.T(), unmtErr1_1)
assert.Nil(s.T(), unmtErr2_1)
//UnMount RWX PVC from both ReadOnly Pods
_, unmtErr1_2 := s.bc.BlockUnmap(getBlockPodDefintion("rwx-block-ro-one", s.pvcNameRWX, true), blockMountPath)
_, unmtErr2_2 := s.bc.BlockUnmap(getBlockPodDefintion("rwx-block-ro-two", s.pvcNameRWX, true), blockMountPath)
assert.Nil(s.T(), unmtErr1_2)
assert.Nil(s.T(), unmtErr2_2)
_, err = s.kh.DeletePods("rwo-block-ro-one", "rwo-block-ro-two", "rwx-block-ro-one", "rwx-block-ro-two")
assert.Nil(s.T(), err)
assert.True(s.T(), s.kh.IsPodTerminated("rwo-block-ro-one", defaultNamespace), "make sure rwo-block-ro-one pod is terminated")
assert.True(s.T(), s.kh.IsPodTerminated("rwo-block-ro-two", defaultNamespace), "make sure rwo-block-ro-two pod is terminated")
assert.True(s.T(), s.kh.IsPodTerminated("rwx-block-ro-one", defaultNamespace), "make sure rwx-lock-ro-one pod is terminated")
+17 -17
View File
@@ -18,13 +18,13 @@ var (
)
// Create StorageClass and poll if needed
func createStorageClassAndPool(t func() *testing.T, kh *utils.K8sHelper, namespace string, storageClassName string, poolName string) {
func createStorageClassAndPool(t func() *testing.T, testClient *clients.TestClient, kh *utils.K8sHelper, namespace string, storageClassName string, poolName string) {
//create storage class
if err := kh.IsStorageClassPresent(storageClassName); err != nil {
logger.Infof("Install pool and storage class for rook block")
_, err := installer.BlockResourceOperation(kh, installer.GetBlockPoolDef(poolName, namespace, "3"), "create")
_, err := testClient.PoolClient.Create(poolName, namespace, 3)
require.NoError(t(), err)
_, err = installer.BlockResourceOperation(kh, installer.GetBlockStorageClassDef(poolName, storageClassName, namespace, false), "create")
_, err = testClient.BlockClient.CreateStorageClass(poolName, storageClassName, namespace, false)
require.NoError(t(), err)
//make sure storageclass is created
@@ -90,7 +90,7 @@ func mountUnmountPVCOnPod(k8sh *utils.K8sHelper, podName string, pvcName string,
return result, err
}
func performBlockOperations(installer *installer.InstallHelper, db *utils.MySQLHelper) {
func performBlockOperations(db *utils.MySQLHelper) {
var wg sync.WaitGroup
for i := 1; i <= installer.Env.LoadConcurrentRuns; i++ {
wg.Add(1)
@@ -133,29 +133,29 @@ func dbOperation(db *utils.MySQLHelper, wg *sync.WaitGroup, runtime int, loadSiz
}
//BaseTestOperations struct for handling panic and test suite tear down
type BaseLoadTestOperations struct {
installer *installer.InstallHelper
// LoadTestCluster struct for handling panic and test suite tear down
type LoadTestCluster struct {
installer *installer.CephInstaller
kh *utils.K8sHelper
helper *clients.TestClient
T func() *testing.T
namespace string
}
// StartBaseTestOperations creates new instance of BaseTestOperations struct
func StartBaseLoadTestOperations(t func() *testing.T, namespace string) (BaseLoadTestOperations, *utils.K8sHelper, *installer.InstallHelper) {
// StartLoadTestCluster creates new instance of TestCluster struct
func StartLoadTestCluster(t func() *testing.T, namespace string) (LoadTestCluster, *utils.K8sHelper, *installer.CephInstaller) {
kh, err := utils.CreateK8sHelper(t)
require.NoError(t(), err)
i := installer.NewK8sRookhelper(kh.Clientset, t)
i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster)
op := BaseLoadTestOperations{i, kh, nil, t, namespace}
op.SetUp()
op := LoadTestCluster{i, kh, nil, t, namespace}
op.Setup()
return op, kh, i
}
//SetUpRook is a wrapper for setting up rook
func (o BaseLoadTestOperations) SetUp() {
// Setup is a wrapper for setting up rook
func (o LoadTestCluster) Setup() {
if !o.kh.IsRookInstalled(o.namespace) {
isRookInstalled, err := o.installer.InstallRookOnK8sWithHostPathAndDevices(o.namespace, "bluestore",
@@ -166,14 +166,14 @@ func (o BaseLoadTestOperations) SetUp() {
}
// Enable chaos monkey if enable_chaos flag is present
if o.installer.Env.EnableChaos {
if installer.Env.EnableChaos {
c := NewChaosHelper(o.namespace, o.kh)
go c.Monkey()
}
}
//TearDownRook is a wrapper for tearDown after suite
func (o BaseLoadTestOperations) TearDown() {
func (o LoadTestCluster) Teardown() {
// No Clean up for load test
}
@@ -262,7 +262,7 @@ spec:
imagePullPolicy: IfNotPresent
volumeMounts:
- name: block-persistent-storage
mountPath: /tmp/rook1
mountPath: ` + clients.TestMountPath + `
volumes:
- name: block-persistent-storage
persistentVolumeClaim:
+1 -1
View File
@@ -69,7 +69,7 @@ func isObjectStorePresent(kh *utils.K8sHelper, namespace string, storeName strin
return false
}
func performObjectStoreOperations(installer *installer.InstallHelper, s3 *utils.S3Helper, bucketName string) {
func performObjectStoreOperations(s3 *utils.S3Helper, bucketName string) {
var wg sync.WaitGroup
for i := 1; i <= installer.Env.LoadConcurrentRuns; i++ {
wg.Add(1)
+14 -11
View File
@@ -21,9 +21,10 @@ import (
"sync"
"testing"
"github.com/rook/rook/tests/framework/clients"
"time"
"github.com/rook/rook/tests/framework/contracts"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/suite"
@@ -44,18 +45,20 @@ func TestBlockLongHaul(t *testing.T) {
type BlockLongHaulSuite struct {
suite.Suite
kh *utils.K8sHelper
installer *installer.InstallHelper
namespace string
op contracts.Setup
kh *utils.K8sHelper
installer *installer.CephInstaller
namespace string
op installer.TestSuite
testClient *clients.TestClient
}
//Test set up - does the following in order
//create pool and storage class, create a PVC, Create a MySQL app/service that uses pvc
func (s *BlockLongHaulSuite) SetupSuite() {
s.namespace = "longhaul-ns"
s.op, s.kh, s.installer = StartBaseLoadTestOperations(s.T, s.namespace)
createStorageClassAndPool(s.T, s.kh, s.namespace, "rook-ceph-block", "rook-pool")
s.op, s.kh, s.installer = StartLoadTestCluster(s.T, s.namespace)
s.testClient = clients.CreateTestClient(s.kh, s.installer.Manifests)
createStorageClassAndPool(s.T, s.testClient, s.kh, s.namespace, "rook-ceph-block", "rook-pool")
}
//create a n number ofPVC, Create a MySQL app/service that uses pvc
@@ -63,8 +66,8 @@ func (s *BlockLongHaulSuite) SetupSuite() {
//All other PVC and mounts are created and deleted/unmounted per test
func (s *BlockLongHaulSuite) TestBlockLonghaulRun() {
var wg sync.WaitGroup
wg.Add(s.installer.Env.LoadVolumeNumber)
for i := 1; i <= s.installer.Env.LoadVolumeNumber; i++ {
wg.Add(installer.Env.LoadVolumeNumber)
for i := 1; i <= installer.Env.LoadVolumeNumber; i++ {
if i == 1 {
go BlockVolumeOperations(s, &wg, "rook-ceph-block", "mysqlapp-persist", "mysqldb", "mysql-persist-claim", false)
} else {
@@ -79,7 +82,7 @@ func (s *BlockLongHaulSuite) TestBlockLonghaulRun() {
func BlockVolumeOperations(s *BlockLongHaulSuite, wg *sync.WaitGroup, storageClassName string, appName string, appLabel string, pvcName string, cleanup bool) {
defer wg.Done()
db := createPVCAndMountMysqlPod(s.T, s.kh, storageClassName, appName, appLabel, pvcName)
performBlockOperations(s.installer, db)
performBlockOperations(db)
if cleanup {
mySqlPodOperation(s.kh, storageClassName, appName, appLabel, pvcName, "delete")
s.kh.WaitUntilPodWithLabelDeleted(appLabel, "default")
@@ -91,7 +94,7 @@ func BlockVolumeOperations(s *BlockLongHaulSuite, wg *sync.WaitGroup, storageCla
func (s *BlockLongHaulSuite) TearDownSuite() {
//clean up all ephemeral block storage, index 1 is persistent block storage which is going to be used among different test runs.
for i := 2; i <= s.installer.Env.LoadVolumeNumber; i++ {
for i := 2; i <= installer.Env.LoadVolumeNumber; i++ {
mySqlPodOperation(s.kh, "rook-ceph-block", "mysqlapp-ephemeral-"+strconv.Itoa(i), "mysqldbeph"+strconv.Itoa(i), "mysql-ephemeral-claim"+strconv.Itoa(i), "delete")
}
s.kh = nil
+11 -14
View File
@@ -24,7 +24,6 @@ import (
"time"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/contracts"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
@@ -46,42 +45,40 @@ func TestBlockWithFencingLongHaul(t *testing.T) {
type BlockLongHaulSuiteWithFencing struct {
suite.Suite
kh *utils.K8sHelper
installer *installer.InstallHelper
installer *installer.CephInstaller
testClient *clients.TestClient
namespace string
op contracts.Setup
op installer.TestSuite
}
//Test set up - does the following in order
//create pool and storage class, Create a PVC and mount a pod with ReadOnly=false.
//Write some data to the pvc and unmount the pod
func (s *BlockLongHaulSuiteWithFencing) SetupSuite() {
var err error
s.namespace = "longhaul-ns"
s.op, s.kh, s.installer = StartBaseLoadTestOperations(s.T, s.namespace)
createStorageClassAndPool(s.T, s.kh, s.namespace, "rook-ceph-block", "rook-pool")
s.testClient, err = clients.CreateTestClient(s.kh, s.namespace)
require.Nil(s.T(), err)
s.op, s.kh, s.installer = StartLoadTestCluster(s.T, s.namespace)
s.testClient = clients.CreateTestClient(s.kh, s.installer.Manifests)
createStorageClassAndPool(s.T, s.testClient, s.kh, s.namespace, "rook-ceph-block", "rook-pool")
if _, err := s.kh.GetPVCStatus(defaultNamespace, "block-pv-one"); err != nil {
logger.Infof("Creating PVC and mounting it to pod with readOnly set to false")
installer.BlockResourceOperation(s.kh, installer.GetBlockPvcDef("block-pv-one", "rook-ceph-block", "ReadWriteOnce"), "create")
_, err = s.testClient.BlockClient.CreatePvc("block-pv-one", "rook-ceph-block", "ReadWriteOnce")
require.Nil(s.T(), err)
mountUnmountPVCOnPod(s.kh, "block-rw", "block-pv-one", "false", "create")
require.True(s.T(), s.kh.IsPodRunning("block-rw", defaultNamespace))
s.testClient.BlockClient.Write("block-rw", "/tmp/rook1", "this is long running test", "longhaul", defaultNamespace)
s.testClient.BlockClient.Write("block-rw", "this is long running test", "longhaul", defaultNamespace)
mountUnmountPVCOnPod(s.kh, "block-rw", "block-pv-one", "false", "delete")
require.True(s.T(), s.kh.IsPodTerminated("block-rw", defaultNamespace))
time.Sleep(5 * time.Second)
}
}
//Mount n number of pods on the same PVC created earlier with readOnly=True.
func (s *BlockLongHaulSuiteWithFencing) TestBlockWithFencingLonghaulRun() {
var wg sync.WaitGroup
wg.Add(s.installer.Env.LoadVolumeNumber)
for i := 1; i <= s.installer.Env.LoadVolumeNumber; i++ {
wg.Add(installer.Env.LoadVolumeNumber)
for i := 1; i <= installer.Env.LoadVolumeNumber; i++ {
go blockVolumeFencingOperations(s, &wg, "block-read"+strconv.Itoa(i), "block-pv-one")
}
wg.Wait()
@@ -91,7 +88,7 @@ func blockVolumeFencingOperations(s *BlockLongHaulSuiteWithFencing, wg *sync.Wai
defer wg.Done()
mountUnmountPVCOnPod(s.kh, podName, pvcName, "true", "create")
require.True(s.T(), s.kh.IsPodRunning(podName, defaultNamespace))
read, rErr := s.testClient.BlockClient.Read(podName, "/tmp/rook1", "longhaul", "default")
read, rErr := s.testClient.BlockClient.Read(podName, "longhaul", "default")
require.Nil(s.T(), rErr)
assert.Contains(s.T(), read, "this is long running test")
mountUnmountPVCOnPod(s.kh, podName, pvcName, "true", "delete")
+7 -11
View File
@@ -22,10 +22,8 @@ import (
"testing"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/contracts"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -47,25 +45,23 @@ func TestObjectLongHaul(t *testing.T) {
type ObjectLongHaulSuite struct {
suite.Suite
kh *utils.K8sHelper
installer *installer.InstallHelper
installer *installer.CephInstaller
tc *clients.TestClient
namespace string
op contracts.Setup
op installer.TestSuite
}
func (s *ObjectLongHaulSuite) SetupSuite() {
var err error
s.namespace = "longhaul-ns"
s.op, s.kh, s.installer = StartBaseLoadTestOperations(s.T, s.namespace)
s.tc, err = clients.CreateTestClient(s.kh, s.namespace)
require.Nil(s.T(), err)
s.op, s.kh, s.installer = StartLoadTestCluster(s.T, s.namespace)
s.tc = clients.CreateTestClient(s.kh, s.installer.Manifests)
}
func (s *ObjectLongHaulSuite) TestObjectLonghaulRun() {
var wg sync.WaitGroup
storeName := "longhaulstore"
wg.Add(s.installer.Env.LoadVolumeNumber)
for i := 1; i <= s.installer.Env.LoadVolumeNumber; i++ {
wg.Add(installer.Env.LoadVolumeNumber)
for i := 1; i <= installer.Env.LoadVolumeNumber; i++ {
if i == 1 {
go ObjectStoreOperations(s, &wg, s.namespace, storeName+strconv.Itoa(i), false)
} else {
@@ -90,7 +86,7 @@ func ObjectStoreOperations(s *ObjectLongHaulSuite, wg *sync.WaitGroup, namespace
s.T().Fail()
}
performObjectStoreOperations(s.installer, s3, bucketName)
performObjectStoreOperations(s3, bucketName)
if deleteStore {
delOpts := metav1.DeleteOptions{}
s.tc.ObjectClient.Delete(namespace, storeName)