From 5b017cfd67e5fde4179c1d25e9f4bbc18fa9c484 Mon Sep 17 00:00:00 2001 From: Rakshith R Date: Tue, 31 May 2022 15:45:00 +0530 Subject: [PATCH] ci: add tests for nfs csi pvc This commit adds nfs csi pvc test into ceph smoke suite. Signed-off-by: Rakshith R --- tests/framework/clients/nfs.go | 7 ++ tests/framework/installer/ceph_installer.go | 16 ++++ tests/framework/installer/ceph_manifests.go | 32 ++++++++ .../installer/ceph_manifests_previous.go | 8 ++ tests/framework/installer/ceph_settings.go | 2 + tests/integration/ceph_base_file_test.go | 12 --- tests/integration/ceph_base_nfs_test.go | 77 +++++++++++++++++++ tests/integration/ceph_smoke_test.go | 5 ++ 8 files changed, 147 insertions(+), 12 deletions(-) create mode 100644 tests/integration/ceph_base_nfs_test.go diff --git a/tests/framework/clients/nfs.go b/tests/framework/clients/nfs.go index b58ebe54f..29b0fc415 100644 --- a/tests/framework/clients/nfs.go +++ b/tests/framework/clients/nfs.go @@ -80,3 +80,10 @@ func (n *NFSOperation) Delete(namespace, name string) error { logger.Infof("Deleted nfs %s in namespace %s", name, namespace) return nil } + +// CreateStorageClass creates a storage class for NFS clients +func (f *NFSOperation) CreateStorageClass(fsName, nfsClusterName, systemNamespace, namespace, storageClassName string) error { + server := fmt.Sprintf("rook-ceph-nfs-%s-a.%s.svc.cluster.local", nfsClusterName, namespace) + + return f.k8sh.ResourceOperation("apply", f.manifests.GetNFSStorageClass(fsName, nfsClusterName, server, storageClassName)) +} diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index f2dd2eb5c..9facffcd5 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -129,6 +129,13 @@ func (h *CephInstaller) CreateCephOperator() (err error) { return errors.Errorf("Failed to create rook-operator pod: %v ", err) } + if h.settings.EnableCSINFS { + csiNFSRBAC := h.Manifests.GetCSINFSRBAC() + if _, err = h.k8shelper.KubectlWithStdin(csiNFSRBAC, createFromStdinArgs...); err != nil { + return err + } + } + err = h.startAdmissionController() if err != nil { return errors.Errorf("Failed to start admission controllers: %v", err) @@ -761,6 +768,15 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(manifests ...CephManifests) } else { logger.Infof("done deleting all the resources in the common manifest") } + if h.settings.EnableCSINFS { + _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetCSINFSRBAC(), deleteFromStdinArgs...) + if err != nil { + logger.Errorf("failed to remove csi nfs rbac manifest. %v", err) + } else { + logger.Info("done deleting all the resources in the csi nfs rbac manifest") + } + } + } } diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index bbeb3a2b8..ff59020da 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -27,6 +27,7 @@ import ( type CephManifests interface { Settings() *TestCephSettings GetCRDs(k8shelper *utils.K8sHelper) string + GetCSINFSRBAC() string GetOperator() string GetCommon() string GetCommonExternal() string @@ -36,6 +37,7 @@ type CephManifests interface { GetBlockPool(poolName, replicaSize string) string GetBlockStorageClass(poolName, storageClassName, reclaimPolicy string) string GetFileStorageClass(fsName, storageClassName string) string + GetNFSStorageClass(fsName, nfsClusterName, server, storageClassName string) string GetBlockSnapshotClass(snapshotClassName, reclaimPolicy string) string GetFileStorageSnapshotClass(snapshotClassName, reclaimPolicy string) string GetFilesystem(name string, activeCount int) string @@ -77,6 +79,10 @@ func (m *CephManifestsMaster) GetCRDs(k8shelper *utils.K8sHelper) string { return m.settings.readManifest("crds.yaml") } +func (m *CephManifestsMaster) GetCSINFSRBAC() string { + return m.settings.readManifest("/csi/nfs/rbac.yaml") +} + func (m *CephManifestsMaster) GetOperator() string { var manifest string if utils.IsPlatformOpenShift() { @@ -353,6 +359,32 @@ parameters: return sc } +func (m *CephManifestsMaster) GetNFSStorageClass(fsName, nfsClusterName, server, storageClassName string) string { + // Create a CSI driver storage class + csiCephFSNodeSecret := "rook-csi-cephfs-node" //nolint:gosec // We safely suppress gosec in tests file + csiCephFSProvisionerSecret := "rook-csi-cephfs-provisioner" //nolint:gosec // We safely suppress gosec in tests file + sc := ` +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ` + storageClassName + ` +provisioner: ` + m.settings.OperatorNamespace + `.nfs.csi.ceph.com +parameters: + clusterID: ` + m.settings.Namespace + ` + fsName: ` + fsName + ` + nfsCluster: ` + nfsClusterName + ` + server: ` + server + ` + pool: ` + fsName + `-data0 + csi.storage.k8s.io/provisioner-secret-name: ` + csiCephFSProvisionerSecret + ` + csi.storage.k8s.io/provisioner-secret-namespace: ` + m.settings.Namespace + ` + csi.storage.k8s.io/node-stage-secret-name: ` + csiCephFSNodeSecret + ` + csi.storage.k8s.io/node-stage-secret-namespace: ` + m.settings.Namespace + ` +mountOptions: + - "nolock" +` + return sc +} + // GetFilesystem returns the manifest to create a Rook filesystem resource with the given config. func (m *CephManifestsMaster) GetFilesystem(name string, activeCount int) string { return `apiVersion: ceph.rook.io/v1 diff --git a/tests/framework/installer/ceph_manifests_previous.go b/tests/framework/installer/ceph_manifests_previous.go index 6b8225559..eb0b29e82 100644 --- a/tests/framework/installer/ceph_manifests_previous.go +++ b/tests/framework/installer/ceph_manifests_previous.go @@ -41,6 +41,10 @@ func (m *CephManifestsPreviousVersion) GetCRDs(k8shelper *utils.K8sHelper) strin return m.settings.readManifestFromGitHub("crds.yaml") } +func (m *CephManifestsPreviousVersion) GetCSINFSRBAC() string { + return m.settings.readManifestFromGitHub("/csi/nfs/rbac.yaml") +} + // GetRookOperator returns rook Operator manifest func (m *CephManifestsPreviousVersion) GetOperator() string { var manifest string @@ -105,6 +109,10 @@ func (m *CephManifestsPreviousVersion) GetFileStorageClass(fsName, storageClassN return m.latest.GetFileStorageClass(fsName, storageClassName) } +func (m *CephManifestsPreviousVersion) GetNFSStorageClass(fsName, nfsClusterName, server, storageClassName string) string { + return m.latest.GetNFSStorageClass(fsName, nfsClusterName, server, storageClassName) +} + // GetFilesystem returns the manifest to create a Rook filesystem resource with the given config. func (m *CephManifestsPreviousVersion) GetFilesystem(name string, activeCount int) string { return m.latest.GetFilesystem(name, activeCount) diff --git a/tests/framework/installer/ceph_settings.go b/tests/framework/installer/ceph_settings.go index bea8a5bb0..cbf7a499f 100644 --- a/tests/framework/installer/ceph_settings.go +++ b/tests/framework/installer/ceph_settings.go @@ -47,6 +47,7 @@ type TestCephSettings struct { ConnectionsEncrypted bool ConnectionsCompressed bool EnableVolumeReplication bool + EnableCSINFS bool ChangeHostName bool RookVersion string CephVersion cephv1.CephVersionSpec @@ -87,6 +88,7 @@ func (s *TestCephSettings) replaceOperatorSettings(manifest string) string { manifest = strings.ReplaceAll(manifest, `# CSI_LOG_LEVEL: "0"`, `CSI_LOG_LEVEL: "5"`) manifest = strings.ReplaceAll(manifest, `ROOK_ENABLE_DISCOVERY_DAEMON: "false"`, fmt.Sprintf(`ROOK_ENABLE_DISCOVERY_DAEMON: "%t"`, s.EnableDiscovery)) manifest = strings.ReplaceAll(manifest, `CSI_ENABLE_VOLUME_REPLICATION: "false"`, fmt.Sprintf(`CSI_ENABLE_VOLUME_REPLICATION: "%t"`, s.EnableVolumeReplication)) + manifest = strings.ReplaceAll(manifest, `ROOK_CSI_ENABLE_NFS: "false"`, fmt.Sprintf(`ROOK_CSI_ENABLE_NFS: "%t"`, s.EnableCSINFS)) return manifest } diff --git a/tests/integration/ceph_base_file_test.go b/tests/integration/ceph_base_file_test.go index 29ad0ea93..213e934cb 100644 --- a/tests/integration/ceph_base_file_test.go +++ b/tests/integration/ceph_base_file_test.go @@ -263,9 +263,6 @@ func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, filePodName, "test_file") assert.NoError(s.T(), err) - // Start the NFS daemons - testNFSDaemons(helper, k8sh, s, settings, filesystemName) - t := s.T() ctx := context.TODO() @@ -404,15 +401,6 @@ func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S } } -func testNFSDaemons(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) { - name := "my-nfs" - err := helper.NFSClient.Create(settings.Namespace, name, 2) - require.Nil(s.T(), err) - - err = helper.NFSClient.Delete(settings.Namespace, name) - assert.Nil(s.T(), err) -} - func createFilesystemConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName, storageClassName string) { err := createPodWithFilesystem(k8sh, s, settings, filePodName, filesystemName, storageClassName, false) require.NoError(s.T(), err) diff --git a/tests/integration/ceph_base_nfs_test.go b/tests/integration/ceph_base_nfs_test.go new file mode 100644 index 000000000..a452312cf --- /dev/null +++ b/tests/integration/ceph_base_nfs_test.go @@ -0,0 +1,77 @@ +/* +Copyright 2022 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/tests/framework/clients" + "github.com/rook/rook/tests/framework/installer" + "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// Smoke Test for File System Storage for CephNFS - Test check the following operations on Filesystem Storage in order +// Create,Mount,Write,Read,Unmount and Delete. +func runNFSFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) { + defer fileTestDataCleanUp(helper, k8sh, s, filePodName, settings.Namespace, filesystemName) + logger.Infof("Running on Rook Cluster %s", settings.Namespace) + logger.Infof("File Storage End To End Integration Test for CephNFS- create, mount, write to, read from, and unmount") + activeCount := 2 + createFilesystem(helper, k8sh, s, settings, filesystemName, activeCount) + + nfsClusterName := "my-nfs" + err := helper.NFSClient.Create(settings.Namespace, nfsClusterName, 2) + require.Nil(s.T(), err) + + if settings.EnableCSINFS { + // Following two commands are needed to be able to create NFS exports in ceph v17.2 + // refer: https://github.com/rook/rook/blob/master/Documentation/CRDs/ceph-nfs-crd.md#ceph-v1720 + parameters := []string{"orch", "set", "backend"} + clusterInfo := client.AdminTestClusterInfo(settings.Namespace) + cmd, args := client.FinalizeCephCommandArgs("ceph", clusterInfo, parameters, k8sh.MakeContext().ConfigDir) + res, err := k8sh.MakeContext().Executor.ExecuteCommandWithOutput(cmd, args...) + if err != nil { + logger.Errorf("Error executing command %q: <%v>, %q", parameters, err, res) + assert.NoError(s.T(), err) + } + parameters = []string{"mgr", "module", "disable", "rook"} + cmd, args = client.FinalizeCephCommandArgs("ceph", clusterInfo, parameters, k8sh.MakeContext().ConfigDir) + res, err = k8sh.MakeContext().Executor.ExecuteCommandWithOutput(cmd, args...) + if err != nil { + logger.Errorf("Error executing command %q: <%v>, %q", parameters, err, res) + assert.NoError(s.T(), err) + } + + storageClassName := "nfs-storageclass" + err = helper.NFSClient.CreateStorageClass(filesystemName, nfsClusterName, settings.OperatorNamespace, settings.Namespace, storageClassName) + assert.NoError(s.T(), err) + createFilesystemConsumerPod(helper, k8sh, s, settings, filesystemName, storageClassName) + + // Test reading and writing to the first pod + err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, filePodName, "test_file") + assert.NoError(s.T(), err) + + cleanupFilesystemConsumer(helper, k8sh, s, settings.Namespace, filePodName) + assert.NoError(s.T(), err) + } + + err = helper.NFSClient.Delete(settings.Namespace, nfsClusterName) + assert.Nil(s.T(), err) + cleanupFilesystem(helper, k8sh, s, settings.Namespace, filesystemName) +} diff --git a/tests/integration/ceph_smoke_test.go b/tests/integration/ceph_smoke_test.go index 679442b8d..7a3fb970d 100644 --- a/tests/integration/ceph_smoke_test.go +++ b/tests/integration/ceph_smoke_test.go @@ -94,6 +94,7 @@ func (s *SmokeSuite) SetupSuite() { ConnectionsCompressed: true, UseCrashPruner: true, EnableVolumeReplication: true, + EnableCSINFS: true, ChangeHostName: true, RookVersion: installer.LocalBuildTag, CephVersion: installer.ReturnCephVersion(), @@ -120,6 +121,10 @@ func (s *SmokeSuite) TestFileStorage_SmokeTest() { runFileE2ETest(s.helper, s.k8sh, s.Suite, s.settings, "smoke-test-fs", preserveFilesystemOnDelete) } +func (s *SmokeSuite) TestNetworkFileStorage_SmokeTest() { + runNFSFileE2ETest(s.helper, s.k8sh, s.Suite, s.settings, "smoke-test-nfs") +} + func (s *SmokeSuite) TestObjectStorage_SmokeTest() { if utils.IsPlatformOpenShift() { s.T().Skip("object store tests skipped on openshift")