From 058ffe8adf046d66107cb4fe5c46ec50c0d6be8f Mon Sep 17 00:00:00 2001 From: travisn Date: Fri, 8 Feb 2019 08:33:36 -0700 Subject: [PATCH] integration test for ceph nfs daemons Signed-off-by: travisn --- tests/framework/clients/nfs.go | 69 +++++++++++++++++++ tests/framework/clients/test_client.go | 2 + tests/framework/installer/ceph_manifests.go | 32 +++++++++ .../installer/ceph_manifests_v0.9.go | 4 ++ tests/integration/base_file_test.go | 12 ++++ 5 files changed, 119 insertions(+) create mode 100644 tests/framework/clients/nfs.go diff --git a/tests/framework/clients/nfs.go b/tests/framework/clients/nfs.go new file mode 100644 index 000000000..4c2361005 --- /dev/null +++ b/tests/framework/clients/nfs.go @@ -0,0 +1,69 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clients + +import ( + "fmt" + + "github.com/rook/rook/tests/framework/installer" + "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NFSOperation is a wrapper for k8s rook file operations +type NFSOperation struct { + k8sh *utils.K8sHelper + manifests installer.CephManifests +} + +// CreateNFSOperation Constructor to create NFSOperation - client to perform ceph nfs operations on k8s +func CreateNFSOperation(k8sh *utils.K8sHelper, manifests installer.CephManifests) *NFSOperation { + return &NFSOperation{k8sh, manifests} +} + +// Create creates a filesystem in Rook +func (n *NFSOperation) Create(namespace, name, pool string, daemonCount int) error { + + logger.Infof("creating the NFS daemons via CRD") + if err := n.k8sh.ResourceOperation("create", n.manifests.GetNFS(namespace, name, pool, daemonCount)); err != nil { + return err + } + + logger.Infof("Make sure rook-ceph-nfs pod is running") + err := n.k8sh.WaitForLabeledPodsToRun(fmt.Sprintf("ceph_nfs=%s", name), namespace) + assert.Nil(n.k8sh.T(), err) + + assert.True(n.k8sh.T(), n.k8sh.CheckPodCountAndState("rook-ceph-nfs", namespace, daemonCount, "Running"), + "Make sure all nfs daemon pods are in Running state") + + return nil +} + +// Delete deletes a filesystem in Rook +func (n *NFSOperation) Delete(namespace, name string) error { + options := &metav1.DeleteOptions{} + logger.Infof("Deleting nfs %s in namespace %s", name, namespace) + err := n.k8sh.RookClientset.CephV1().CephNFSs(namespace).Delete(name, options) + if err != nil && !errors.IsNotFound(err) { + return err + } + + logger.Infof("Deleted nfs %s in namespace %s", name, namespace) + return nil +} diff --git a/tests/framework/clients/test_client.go b/tests/framework/clients/test_client.go index 03c989039..cd7f53355 100644 --- a/tests/framework/clients/test_client.go +++ b/tests/framework/clients/test_client.go @@ -32,6 +32,7 @@ var ( type TestClient struct { BlockClient *BlockOperation FSClient *FilesystemOperation + NFSClient *NFSOperation ObjectClient *ObjectOperation ObjectUserClient *ObjectUserOperation PoolClient *PoolOperation @@ -47,6 +48,7 @@ func CreateTestClient(k8sHelper *utils.K8sHelper, manifests installer.CephManife return &TestClient{ CreateBlockOperation(k8sHelper, manifests), CreateFilesystemOperation(k8sHelper, manifests), + CreateNFSOperation(k8sHelper, manifests), CreateObjectOperation(k8sHelper, manifests), CreateObjectUserOperation(k8sHelper, manifests), CreatePoolOperation(k8sHelper, manifests), diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 9f20da4bf..95080732e 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -37,6 +37,7 @@ type CephManifests interface { GetBlockPoolStorageClassAndPvcDef(namespace string, poolName string, storageClassName string, reclaimPolicy string, blockName string, accessMode string) string GetBlockPoolStorageClass(namespace string, poolName string, storageClassName string, reclaimPolicy string) string GetFilesystem(namepace, name string, activeCount int) string + GetNFS(namepace, name, pool string, daemonCount int) string GetObjectStore(namespace, name string, replicaCount, port int) string GetObjectStoreUser(namespace, name string, displayName string, store string) string } @@ -170,6 +171,22 @@ spec: --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition +metadata: + name: cephnfses.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephNFS + listKind: CephNFSList + plural: cephnfses + singular: cephnfs + shortNames: + - nfs + scope: Namespaced + version: v1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition metadata: name: cephobjectstores.ceph.rook.io spec: @@ -1109,6 +1126,21 @@ spec: activeStandby: true` } +// GetFilesystem returns the manifest to create a Rook Ceph NFS resource with the given config. +func (m *CephManifestsMaster) GetNFS(namespace, name, pool string, count int) string { + return `apiVersion: ceph.rook.io/v1 +kind: CephNFS +metadata: + name: ` + name + ` + namespace: ` + namespace + ` +spec: + rados: + pool: ` + pool + ` + namespace: nfs-ns + server: + active: ` + strconv.Itoa(count) +} + func (m *CephManifestsMaster) GetObjectStore(namespace, name string, replicaCount, port int) string { return `apiVersion: ceph.rook.io/v1 kind: CephObjectStore diff --git a/tests/framework/installer/ceph_manifests_v0.9.go b/tests/framework/installer/ceph_manifests_v0.9.go index 5991eee2b..102b6c060 100644 --- a/tests/framework/installer/ceph_manifests_v0.9.go +++ b/tests/framework/installer/ceph_manifests_v0.9.go @@ -780,6 +780,10 @@ spec: activeStandby: true` } +func (m *CephManifestsV0_9) GetNFS(namespace, name, pool string, count int) string { + panic("GetNFS not implemented") +} + func (m *CephManifestsV0_9) GetObjectStore(namespace, name string, replicaCount, port int) string { return `apiVersion: ceph.rook.io/v1 kind: CephObjectStore diff --git a/tests/integration/base_file_test.go b/tests/integration/base_file_test.go index 73e0773f3..5bd9ba983 100644 --- a/tests/integration/base_file_test.go +++ b/tests/integration/base_file_test.go @@ -43,11 +43,23 @@ func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S createFilesystemConsumerPod(helper, k8sh, s, namespace, filesystemName) err := writeAndReadToFilesystem(helper, k8sh, s, namespace, filePodName, "test_file") require.Nil(s.T(), err) + + testNFSDaemons(helper, k8sh, s, namespace, filesystemName) + downscaleMetadataServers(helper, k8sh, s, namespace, filesystemName) cleanupFilesystemConsumer(helper, k8sh, s, namespace, filesystemName, filePodName) cleanupFilesystem(helper, k8sh, s, namespace, filesystemName) } +func testNFSDaemons(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) { + name := "my-nfs" + err := helper.NFSClient.Create(namespace, name, filesystemName+"-data0", 2) + require.Nil(s.T(), err) + + err = helper.NFSClient.Delete(namespace, name) + assert.Nil(s.T(), err) +} + func createFilesystemConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) { mtfsErr := podWithFilesystem(k8sh, s, filePodName, namespace, filesystemName, "create", getFilesystemTestPod) require.Nil(s.T(), mtfsErr)