ci: add tests for nfs csi pvc
This commit adds nfs csi pvc test into ceph smoke suite. Signed-off-by: Rakshith R <rar@redhat.com>
This commit is contained in:
1 parent
800d3e5050
commit
5b017cfd67
8 files changed
+147
-12
No files matched your search
@@ -80,3 +80,10 @@ func (n *NFSOperation) Delete(namespace, name string) error {
|
||||
logger.Infof("Deleted nfs %s in namespace %s", name, namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateStorageClass creates a storage class for NFS clients
|
||||
func (f *NFSOperation) CreateStorageClass(fsName, nfsClusterName, systemNamespace, namespace, storageClassName string) error {
|
||||
server := fmt.Sprintf("rook-ceph-nfs-%s-a.%s.svc.cluster.local", nfsClusterName, namespace)
|
||||
|
||||
return f.k8sh.ResourceOperation("apply", f.manifests.GetNFSStorageClass(fsName, nfsClusterName, server, storageClassName))
|
||||
}
|
||||
@@ -129,6 +129,13 @@ func (h *CephInstaller) CreateCephOperator() (err error) {
|
||||
return errors.Errorf("Failed to create rook-operator pod: %v ", err)
|
||||
}
|
||||
|
||||
if h.settings.EnableCSINFS {
|
||||
csiNFSRBAC := h.Manifests.GetCSINFSRBAC()
|
||||
if _, err = h.k8shelper.KubectlWithStdin(csiNFSRBAC, createFromStdinArgs...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = h.startAdmissionController()
|
||||
if err != nil {
|
||||
return errors.Errorf("Failed to start admission controllers: %v", err)
|
||||
@@ -761,6 +768,15 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(manifests ...CephManifests)
|
||||
} else {
|
||||
logger.Infof("done deleting all the resources in the common manifest")
|
||||
}
|
||||
if h.settings.EnableCSINFS {
|
||||
_, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetCSINFSRBAC(), deleteFromStdinArgs...)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to remove csi nfs rbac manifest. %v", err)
|
||||
} else {
|
||||
logger.Info("done deleting all the resources in the csi nfs rbac manifest")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
type CephManifests interface {
|
||||
Settings() *TestCephSettings
|
||||
GetCRDs(k8shelper *utils.K8sHelper) string
|
||||
GetCSINFSRBAC() string
|
||||
GetOperator() string
|
||||
GetCommon() string
|
||||
GetCommonExternal() string
|
||||
@@ -36,6 +37,7 @@ type CephManifests interface {
|
||||
GetBlockPool(poolName, replicaSize string) string
|
||||
GetBlockStorageClass(poolName, storageClassName, reclaimPolicy string) string
|
||||
GetFileStorageClass(fsName, storageClassName string) string
|
||||
GetNFSStorageClass(fsName, nfsClusterName, server, storageClassName string) string
|
||||
GetBlockSnapshotClass(snapshotClassName, reclaimPolicy string) string
|
||||
GetFileStorageSnapshotClass(snapshotClassName, reclaimPolicy string) string
|
||||
GetFilesystem(name string, activeCount int) string
|
||||
@@ -77,6 +79,10 @@ func (m *CephManifestsMaster) GetCRDs(k8shelper *utils.K8sHelper) string {
|
||||
return m.settings.readManifest("crds.yaml")
|
||||
}
|
||||
|
||||
func (m *CephManifestsMaster) GetCSINFSRBAC() string {
|
||||
return m.settings.readManifest("/csi/nfs/rbac.yaml")
|
||||
}
|
||||
|
||||
func (m *CephManifestsMaster) GetOperator() string {
|
||||
var manifest string
|
||||
if utils.IsPlatformOpenShift() {
|
||||
@@ -353,6 +359,32 @@ parameters:
|
||||
return sc
|
||||
}
|
||||
|
||||
func (m *CephManifestsMaster) GetNFSStorageClass(fsName, nfsClusterName, server, storageClassName string) string {
|
||||
// Create a CSI driver storage class
|
||||
csiCephFSNodeSecret := "rook-csi-cephfs-node" //nolint:gosec // We safely suppress gosec in tests file
|
||||
csiCephFSProvisionerSecret := "rook-csi-cephfs-provisioner" //nolint:gosec // We safely suppress gosec in tests file
|
||||
sc := `
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: ` + storageClassName + `
|
||||
provisioner: ` + m.settings.OperatorNamespace + `.nfs.csi.ceph.com
|
||||
parameters:
|
||||
clusterID: ` + m.settings.Namespace + `
|
||||
fsName: ` + fsName + `
|
||||
nfsCluster: ` + nfsClusterName + `
|
||||
server: ` + server + `
|
||||
pool: ` + fsName + `-data0
|
||||
csi.storage.k8s.io/provisioner-secret-name: ` + csiCephFSProvisionerSecret + `
|
||||
csi.storage.k8s.io/provisioner-secret-namespace: ` + m.settings.Namespace + `
|
||||
csi.storage.k8s.io/node-stage-secret-name: ` + csiCephFSNodeSecret + `
|
||||
csi.storage.k8s.io/node-stage-secret-namespace: ` + m.settings.Namespace + `
|
||||
mountOptions:
|
||||
- "nolock"
|
||||
`
|
||||
return sc
|
||||
}
|
||||
|
||||
// GetFilesystem returns the manifest to create a Rook filesystem resource with the given config.
|
||||
func (m *CephManifestsMaster) GetFilesystem(name string, activeCount int) string {
|
||||
return `apiVersion: ceph.rook.io/v1
|
||||
|
||||
@@ -41,6 +41,10 @@ func (m *CephManifestsPreviousVersion) GetCRDs(k8shelper *utils.K8sHelper) strin
|
||||
return m.settings.readManifestFromGitHub("crds.yaml")
|
||||
}
|
||||
|
||||
func (m *CephManifestsPreviousVersion) GetCSINFSRBAC() string {
|
||||
return m.settings.readManifestFromGitHub("/csi/nfs/rbac.yaml")
|
||||
}
|
||||
|
||||
// GetRookOperator returns rook Operator manifest
|
||||
func (m *CephManifestsPreviousVersion) GetOperator() string {
|
||||
var manifest string
|
||||
@@ -105,6 +109,10 @@ func (m *CephManifestsPreviousVersion) GetFileStorageClass(fsName, storageClassN
|
||||
return m.latest.GetFileStorageClass(fsName, storageClassName)
|
||||
}
|
||||
|
||||
func (m *CephManifestsPreviousVersion) GetNFSStorageClass(fsName, nfsClusterName, server, storageClassName string) string {
|
||||
return m.latest.GetNFSStorageClass(fsName, nfsClusterName, server, storageClassName)
|
||||
}
|
||||
|
||||
// GetFilesystem returns the manifest to create a Rook filesystem resource with the given config.
|
||||
func (m *CephManifestsPreviousVersion) GetFilesystem(name string, activeCount int) string {
|
||||
return m.latest.GetFilesystem(name, activeCount)
|
||||
|
||||
@@ -47,6 +47,7 @@ type TestCephSettings struct {
|
||||
ConnectionsEncrypted bool
|
||||
ConnectionsCompressed bool
|
||||
EnableVolumeReplication bool
|
||||
EnableCSINFS bool
|
||||
ChangeHostName bool
|
||||
RookVersion string
|
||||
CephVersion cephv1.CephVersionSpec
|
||||
@@ -87,6 +88,7 @@ func (s *TestCephSettings) replaceOperatorSettings(manifest string) string {
|
||||
manifest = strings.ReplaceAll(manifest, `# CSI_LOG_LEVEL: "0"`, `CSI_LOG_LEVEL: "5"`)
|
||||
manifest = strings.ReplaceAll(manifest, `ROOK_ENABLE_DISCOVERY_DAEMON: "false"`, fmt.Sprintf(`ROOK_ENABLE_DISCOVERY_DAEMON: "%t"`, s.EnableDiscovery))
|
||||
manifest = strings.ReplaceAll(manifest, `CSI_ENABLE_VOLUME_REPLICATION: "false"`, fmt.Sprintf(`CSI_ENABLE_VOLUME_REPLICATION: "%t"`, s.EnableVolumeReplication))
|
||||
manifest = strings.ReplaceAll(manifest, `ROOK_CSI_ENABLE_NFS: "false"`, fmt.Sprintf(`ROOK_CSI_ENABLE_NFS: "%t"`, s.EnableCSINFS))
|
||||
return manifest
|
||||
}
|
||||
|
||||
|
||||
@@ -263,9 +263,6 @@ func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S
|
||||
err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, filePodName, "test_file")
|
||||
assert.NoError(s.T(), err)
|
||||
|
||||
// Start the NFS daemons
|
||||
testNFSDaemons(helper, k8sh, s, settings, filesystemName)
|
||||
|
||||
t := s.T()
|
||||
ctx := context.TODO()
|
||||
|
||||
@@ -404,15 +401,6 @@ func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S
|
||||
}
|
||||
}
|
||||
|
||||
func testNFSDaemons(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) {
|
||||
name := "my-nfs"
|
||||
err := helper.NFSClient.Create(settings.Namespace, name, 2)
|
||||
require.Nil(s.T(), err)
|
||||
|
||||
err = helper.NFSClient.Delete(settings.Namespace, name)
|
||||
assert.Nil(s.T(), err)
|
||||
}
|
||||
|
||||
func createFilesystemConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName, storageClassName string) {
|
||||
err := createPodWithFilesystem(k8sh, s, settings, filePodName, filesystemName, storageClassName, false)
|
||||
require.NoError(s.T(), err)
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
/*
|
||||
Copyright 2022 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package integration
|
||||
|
||||
import (
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
// Smoke Test for File System Storage for CephNFS - Test check the following operations on Filesystem Storage in order
|
||||
// Create,Mount,Write,Read,Unmount and Delete.
|
||||
func runNFSFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) {
|
||||
defer fileTestDataCleanUp(helper, k8sh, s, filePodName, settings.Namespace, filesystemName)
|
||||
logger.Infof("Running on Rook Cluster %s", settings.Namespace)
|
||||
logger.Infof("File Storage End To End Integration Test for CephNFS- create, mount, write to, read from, and unmount")
|
||||
activeCount := 2
|
||||
createFilesystem(helper, k8sh, s, settings, filesystemName, activeCount)
|
||||
|
||||
nfsClusterName := "my-nfs"
|
||||
err := helper.NFSClient.Create(settings.Namespace, nfsClusterName, 2)
|
||||
require.Nil(s.T(), err)
|
||||
|
||||
if settings.EnableCSINFS {
|
||||
// Following two commands are needed to be able to create NFS exports in ceph v17.2
|
||||
// refer: https://github.com/rook/rook/blob/master/Documentation/CRDs/ceph-nfs-crd.md#ceph-v1720
|
||||
parameters := []string{"orch", "set", "backend"}
|
||||
clusterInfo := client.AdminTestClusterInfo(settings.Namespace)
|
||||
cmd, args := client.FinalizeCephCommandArgs("ceph", clusterInfo, parameters, k8sh.MakeContext().ConfigDir)
|
||||
res, err := k8sh.MakeContext().Executor.ExecuteCommandWithOutput(cmd, args...)
|
||||
if err != nil {
|
||||
logger.Errorf("Error executing command %q: <%v>, %q", parameters, err, res)
|
||||
assert.NoError(s.T(), err)
|
||||
}
|
||||
parameters = []string{"mgr", "module", "disable", "rook"}
|
||||
cmd, args = client.FinalizeCephCommandArgs("ceph", clusterInfo, parameters, k8sh.MakeContext().ConfigDir)
|
||||
res, err = k8sh.MakeContext().Executor.ExecuteCommandWithOutput(cmd, args...)
|
||||
if err != nil {
|
||||
logger.Errorf("Error executing command %q: <%v>, %q", parameters, err, res)
|
||||
assert.NoError(s.T(), err)
|
||||
}
|
||||
|
||||
storageClassName := "nfs-storageclass"
|
||||
err = helper.NFSClient.CreateStorageClass(filesystemName, nfsClusterName, settings.OperatorNamespace, settings.Namespace, storageClassName)
|
||||
assert.NoError(s.T(), err)
|
||||
createFilesystemConsumerPod(helper, k8sh, s, settings, filesystemName, storageClassName)
|
||||
|
||||
// Test reading and writing to the first pod
|
||||
err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, filePodName, "test_file")
|
||||
assert.NoError(s.T(), err)
|
||||
|
||||
cleanupFilesystemConsumer(helper, k8sh, s, settings.Namespace, filePodName)
|
||||
assert.NoError(s.T(), err)
|
||||
}
|
||||
|
||||
err = helper.NFSClient.Delete(settings.Namespace, nfsClusterName)
|
||||
assert.Nil(s.T(), err)
|
||||
cleanupFilesystem(helper, k8sh, s, settings.Namespace, filesystemName)
|
||||
}
|
||||
@@ -94,6 +94,7 @@ func (s *SmokeSuite) SetupSuite() {
|
||||
ConnectionsCompressed: true,
|
||||
UseCrashPruner: true,
|
||||
EnableVolumeReplication: true,
|
||||
EnableCSINFS: true,
|
||||
ChangeHostName: true,
|
||||
RookVersion: installer.LocalBuildTag,
|
||||
CephVersion: installer.ReturnCephVersion(),
|
||||
@@ -120,6 +121,10 @@ func (s *SmokeSuite) TestFileStorage_SmokeTest() {
|
||||
runFileE2ETest(s.helper, s.k8sh, s.Suite, s.settings, "smoke-test-fs", preserveFilesystemOnDelete)
|
||||
}
|
||||
|
||||
func (s *SmokeSuite) TestNetworkFileStorage_SmokeTest() {
|
||||
runNFSFileE2ETest(s.helper, s.k8sh, s.Suite, s.settings, "smoke-test-nfs")
|
||||
}
|
||||
|
||||
func (s *SmokeSuite) TestObjectStorage_SmokeTest() {
|
||||
if utils.IsPlatformOpenShift() {
|
||||
s.T().Skip("object store tests skipped on openshift")
|
||||
|
||||
Reference in new issue
Block a user