ci: add tests for nfs csi pvc

This commit adds nfs csi pvc test into
ceph smoke suite.

Signed-off-by: Rakshith R <rar@redhat.com>
This commit is contained in:
Rakshith R committed 2022-06-20 15:01:22 +05:30
1 parent 800d3e5050
commit 5b017cfd67
8 files changed
+147 -12

No files matched your search

+7
View File
@@ -80,3 +80,10 @@ func (n *NFSOperation) Delete(namespace, name string) error {
logger.Infof("Deleted nfs %s in namespace %s", name, namespace)
return nil
}
// CreateStorageClass creates a storage class for NFS clients
func (f *NFSOperation) CreateStorageClass(fsName, nfsClusterName, systemNamespace, namespace, storageClassName string) error {
server := fmt.Sprintf("rook-ceph-nfs-%s-a.%s.svc.cluster.local", nfsClusterName, namespace)
return f.k8sh.ResourceOperation("apply", f.manifests.GetNFSStorageClass(fsName, nfsClusterName, server, storageClassName))
}
@@ -129,6 +129,13 @@ func (h *CephInstaller) CreateCephOperator() (err error) {
return errors.Errorf("Failed to create rook-operator pod: %v ", err)
}
if h.settings.EnableCSINFS {
csiNFSRBAC := h.Manifests.GetCSINFSRBAC()
if _, err = h.k8shelper.KubectlWithStdin(csiNFSRBAC, createFromStdinArgs...); err != nil {
return err
}
}
err = h.startAdmissionController()
if err != nil {
return errors.Errorf("Failed to start admission controllers: %v", err)
@@ -761,6 +768,15 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(manifests ...CephManifests)
} else {
logger.Infof("done deleting all the resources in the common manifest")
}
if h.settings.EnableCSINFS {
_, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetCSINFSRBAC(), deleteFromStdinArgs...)
if err != nil {
logger.Errorf("failed to remove csi nfs rbac manifest. %v", err)
} else {
logger.Info("done deleting all the resources in the csi nfs rbac manifest")
}
}
}
}
@@ -27,6 +27,7 @@ import (
type CephManifests interface {
Settings() *TestCephSettings
GetCRDs(k8shelper *utils.K8sHelper) string
GetCSINFSRBAC() string
GetOperator() string
GetCommon() string
GetCommonExternal() string
@@ -36,6 +37,7 @@ type CephManifests interface {
GetBlockPool(poolName, replicaSize string) string
GetBlockStorageClass(poolName, storageClassName, reclaimPolicy string) string
GetFileStorageClass(fsName, storageClassName string) string
GetNFSStorageClass(fsName, nfsClusterName, server, storageClassName string) string
GetBlockSnapshotClass(snapshotClassName, reclaimPolicy string) string
GetFileStorageSnapshotClass(snapshotClassName, reclaimPolicy string) string
GetFilesystem(name string, activeCount int) string
@@ -77,6 +79,10 @@ func (m *CephManifestsMaster) GetCRDs(k8shelper *utils.K8sHelper) string {
return m.settings.readManifest("crds.yaml")
}
func (m *CephManifestsMaster) GetCSINFSRBAC() string {
return m.settings.readManifest("/csi/nfs/rbac.yaml")
}
func (m *CephManifestsMaster) GetOperator() string {
var manifest string
if utils.IsPlatformOpenShift() {
@@ -353,6 +359,32 @@ parameters:
return sc
}
func (m *CephManifestsMaster) GetNFSStorageClass(fsName, nfsClusterName, server, storageClassName string) string {
// Create a CSI driver storage class
csiCephFSNodeSecret := "rook-csi-cephfs-node" //nolint:gosec // We safely suppress gosec in tests file
csiCephFSProvisionerSecret := "rook-csi-cephfs-provisioner" //nolint:gosec // We safely suppress gosec in tests file
sc := `
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: ` + storageClassName + `
provisioner: ` + m.settings.OperatorNamespace + `.nfs.csi.ceph.com
parameters:
clusterID: ` + m.settings.Namespace + `
fsName: ` + fsName + `
nfsCluster: ` + nfsClusterName + `
server: ` + server + `
pool: ` + fsName + `-data0
csi.storage.k8s.io/provisioner-secret-name: ` + csiCephFSProvisionerSecret + `
csi.storage.k8s.io/provisioner-secret-namespace: ` + m.settings.Namespace + `
csi.storage.k8s.io/node-stage-secret-name: ` + csiCephFSNodeSecret + `
csi.storage.k8s.io/node-stage-secret-namespace: ` + m.settings.Namespace + `
mountOptions:
- "nolock"
`
return sc
}
// GetFilesystem returns the manifest to create a Rook filesystem resource with the given config.
func (m *CephManifestsMaster) GetFilesystem(name string, activeCount int) string {
return `apiVersion: ceph.rook.io/v1
@@ -41,6 +41,10 @@ func (m *CephManifestsPreviousVersion) GetCRDs(k8shelper *utils.K8sHelper) strin
return m.settings.readManifestFromGitHub("crds.yaml")
}
func (m *CephManifestsPreviousVersion) GetCSINFSRBAC() string {
return m.settings.readManifestFromGitHub("/csi/nfs/rbac.yaml")
}
// GetRookOperator returns rook Operator manifest
func (m *CephManifestsPreviousVersion) GetOperator() string {
var manifest string
@@ -105,6 +109,10 @@ func (m *CephManifestsPreviousVersion) GetFileStorageClass(fsName, storageClassN
return m.latest.GetFileStorageClass(fsName, storageClassName)
}
func (m *CephManifestsPreviousVersion) GetNFSStorageClass(fsName, nfsClusterName, server, storageClassName string) string {
return m.latest.GetNFSStorageClass(fsName, nfsClusterName, server, storageClassName)
}
// GetFilesystem returns the manifest to create a Rook filesystem resource with the given config.
func (m *CephManifestsPreviousVersion) GetFilesystem(name string, activeCount int) string {
return m.latest.GetFilesystem(name, activeCount)
@@ -47,6 +47,7 @@ type TestCephSettings struct {
ConnectionsEncrypted bool
ConnectionsCompressed bool
EnableVolumeReplication bool
EnableCSINFS bool
ChangeHostName bool
RookVersion string
CephVersion cephv1.CephVersionSpec
@@ -87,6 +88,7 @@ func (s *TestCephSettings) replaceOperatorSettings(manifest string) string {
manifest = strings.ReplaceAll(manifest, `# CSI_LOG_LEVEL: "0"`, `CSI_LOG_LEVEL: "5"`)
manifest = strings.ReplaceAll(manifest, `ROOK_ENABLE_DISCOVERY_DAEMON: "false"`, fmt.Sprintf(`ROOK_ENABLE_DISCOVERY_DAEMON: "%t"`, s.EnableDiscovery))
manifest = strings.ReplaceAll(manifest, `CSI_ENABLE_VOLUME_REPLICATION: "false"`, fmt.Sprintf(`CSI_ENABLE_VOLUME_REPLICATION: "%t"`, s.EnableVolumeReplication))
manifest = strings.ReplaceAll(manifest, `ROOK_CSI_ENABLE_NFS: "false"`, fmt.Sprintf(`ROOK_CSI_ENABLE_NFS: "%t"`, s.EnableCSINFS))
return manifest
}
-12
View File
@@ -263,9 +263,6 @@ func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S
err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, filePodName, "test_file")
assert.NoError(s.T(), err)
// Start the NFS daemons
testNFSDaemons(helper, k8sh, s, settings, filesystemName)
t := s.T()
ctx := context.TODO()
@@ -404,15 +401,6 @@ func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S
}
}
func testNFSDaemons(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) {
name := "my-nfs"
err := helper.NFSClient.Create(settings.Namespace, name, 2)
require.Nil(s.T(), err)
err = helper.NFSClient.Delete(settings.Namespace, name)
assert.Nil(s.T(), err)
}
func createFilesystemConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName, storageClassName string) {
err := createPodWithFilesystem(k8sh, s, settings, filePodName, filesystemName, storageClassName, false)
require.NoError(s.T(), err)
+77
View File
@@ -0,0 +1,77 @@
/*
Copyright 2022 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
// Smoke Test for File System Storage for CephNFS - Test check the following operations on Filesystem Storage in order
// Create,Mount,Write,Read,Unmount and Delete.
func runNFSFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) {
defer fileTestDataCleanUp(helper, k8sh, s, filePodName, settings.Namespace, filesystemName)
logger.Infof("Running on Rook Cluster %s", settings.Namespace)
logger.Infof("File Storage End To End Integration Test for CephNFS- create, mount, write to, read from, and unmount")
activeCount := 2
createFilesystem(helper, k8sh, s, settings, filesystemName, activeCount)
nfsClusterName := "my-nfs"
err := helper.NFSClient.Create(settings.Namespace, nfsClusterName, 2)
require.Nil(s.T(), err)
if settings.EnableCSINFS {
// Following two commands are needed to be able to create NFS exports in ceph v17.2
// refer: https://github.com/rook/rook/blob/master/Documentation/CRDs/ceph-nfs-crd.md#ceph-v1720
parameters := []string{"orch", "set", "backend"}
clusterInfo := client.AdminTestClusterInfo(settings.Namespace)
cmd, args := client.FinalizeCephCommandArgs("ceph", clusterInfo, parameters, k8sh.MakeContext().ConfigDir)
res, err := k8sh.MakeContext().Executor.ExecuteCommandWithOutput(cmd, args...)
if err != nil {
logger.Errorf("Error executing command %q: <%v>, %q", parameters, err, res)
assert.NoError(s.T(), err)
}
parameters = []string{"mgr", "module", "disable", "rook"}
cmd, args = client.FinalizeCephCommandArgs("ceph", clusterInfo, parameters, k8sh.MakeContext().ConfigDir)
res, err = k8sh.MakeContext().Executor.ExecuteCommandWithOutput(cmd, args...)
if err != nil {
logger.Errorf("Error executing command %q: <%v>, %q", parameters, err, res)
assert.NoError(s.T(), err)
}
storageClassName := "nfs-storageclass"
err = helper.NFSClient.CreateStorageClass(filesystemName, nfsClusterName, settings.OperatorNamespace, settings.Namespace, storageClassName)
assert.NoError(s.T(), err)
createFilesystemConsumerPod(helper, k8sh, s, settings, filesystemName, storageClassName)
// Test reading and writing to the first pod
err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, filePodName, "test_file")
assert.NoError(s.T(), err)
cleanupFilesystemConsumer(helper, k8sh, s, settings.Namespace, filePodName)
assert.NoError(s.T(), err)
}
err = helper.NFSClient.Delete(settings.Namespace, nfsClusterName)
assert.Nil(s.T(), err)
cleanupFilesystem(helper, k8sh, s, settings.Namespace, filesystemName)
}
+5
View File
@@ -94,6 +94,7 @@ func (s *SmokeSuite) SetupSuite() {
ConnectionsCompressed: true,
UseCrashPruner: true,
EnableVolumeReplication: true,
EnableCSINFS: true,
ChangeHostName: true,
RookVersion: installer.LocalBuildTag,
CephVersion: installer.ReturnCephVersion(),
@@ -120,6 +121,10 @@ func (s *SmokeSuite) TestFileStorage_SmokeTest() {
runFileE2ETest(s.helper, s.k8sh, s.Suite, s.settings, "smoke-test-fs", preserveFilesystemOnDelete)
}
func (s *SmokeSuite) TestNetworkFileStorage_SmokeTest() {
runNFSFileE2ETest(s.helper, s.k8sh, s.Suite, s.settings, "smoke-test-nfs")
}
func (s *SmokeSuite) TestObjectStorage_SmokeTest() {
if utils.IsPlatformOpenShift() {
s.T().Skip("object store tests skipped on openshift")