From 16732915891ff2e98b0448d2e7ff533ef7e0c071 Mon Sep 17 00:00:00 2001 From: Santosh Pillai Date: Fri, 3 Apr 2020 13:54:52 +0530 Subject: [PATCH] ceph: Integration Tests: moving GetCleanupPod() and GetCleanupVerificationPod() methods out of CephManifests interface The CephManifests interface is intended for manifests that install the product (RBAC, CRDs, operator, etc). These cleanup methods are more about test code. The test code needs to behave the same whether it is a normal install or an upgrade. We don't need to version it differently for upgrades between the CephManifestsMaster and CephManifestsV1_1 implementations Signed-off-by: Santosh Pillai --- tests/framework/installer/ceph_installer.go | 70 ++++++++++++++++++- tests/framework/installer/ceph_manifests.go | 68 ------------------ .../installer/ceph_manifests_v1.1.go | 37 ---------- 3 files changed, 68 insertions(+), 107 deletions(-) diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 2ad7d2101..786c03f9a 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -26,6 +26,7 @@ import ( "testing" "time" + "github.com/google/uuid" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/cluster" @@ -651,13 +652,13 @@ func (h *CephInstaller) checkCephHealthStatus(namespace string) { } func (h *CephInstaller) cleanupDir(node, dir string) error { - resources := h.Manifests.GetCleanupPod(node, dir) + resources := h.GetCleanupPod(node, dir) _, err := h.k8shelper.KubectlWithStdin(resources, createFromStdinArgs...) return err } func (h *CephInstaller) verifyDirCleanup(node, dir string) error { - resources := h.Manifests.GetCleanupVerificationPod(node, dir) + resources := h.GetCleanupVerificationPod(node, dir) _, err := h.k8shelper.KubectlWithStdin(resources, createFromStdinArgs...) return err } @@ -842,6 +843,71 @@ spec: ` } +// GetCleanupPod gets a cleanup Pod that cleans up the dataDirHostPath +func (h *CephInstaller) GetCleanupPod(node, removalDir string) string { + return `apiVersion: batch/v1 +kind: Job +metadata: + name: rook-cleanup-` + uuid.Must(uuid.NewRandom()).String() + ` +spec: + template: + spec: + restartPolicy: Never + containers: + - name: rook-cleaner + image: rook/ceph:` + VersionMaster + ` + securityContext: + privileged: true + volumeMounts: + - name: cleaner + mountPath: /scrub + command: + - "sh" + - "-c" + - "rm -rf /scrub/*" + nodeSelector: + kubernetes.io/hostname: ` + node + ` + volumes: + - name: cleaner + hostPath: + path: ` + removalDir +} + +// GetCleanupVerificationPod verifies that the dataDirHostPath is empty +func (h *CephInstaller) GetCleanupVerificationPod(node, hostPathDir string) string { + return `apiVersion: batch/v1 +kind: Job +metadata: + name: rook-verify-cleanup-` + uuid.Must(uuid.NewRandom()).String() + ` +spec: + template: + spec: + restartPolicy: Never + containers: + - name: rook-cleaner + image: rook/ceph:` + VersionMaster + ` + securityContext: + privileged: true + volumeMounts: + - name: cleaner + mountPath: /scrub + command: + - "sh" + - "-c" + - | + set -xEeuo pipefail + #Assert dataDirHostPath is empty + if [ "$(ls -A /scrub/)" ]; then + exit 1 + fi + nodeSelector: + kubernetes.io/hostname: ` + node + ` + volumes: + - name: cleaner + hostPath: + path: ` + hostPathDir +} + func (h *CephInstaller) addCleanupPolicy(namespace string) error { cluster, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(namespace, metav1.GetOptions{}) if err != nil { diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 3b5608951..c5f77d938 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -21,7 +21,6 @@ import ( "strconv" "strings" - "github.com/google/uuid" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" ) @@ -38,8 +37,6 @@ type CephManifests interface { GetRookCluster(settings *ClusterSettings) string GetRookExternalCluster(settings *ClusterExternalSettings) string GetRookToolBox(namespace string) string - GetCleanupPod(node, removalDir string) string - GetCleanupVerificationPod(node, hostDirPath string) string GetBlockPoolDef(poolName, namespace, replicaSize string) string GetBlockStorageClassDef(csi bool, poolName, storageClassName, reclaimPolicy, namespace, systemNamespace string) string GetFileStorageClassDef(fsName, storageClassName, namespace string) string @@ -2025,71 +2022,6 @@ spec: path: mon-endpoints` } -// GetCleanupPod gets a cleanup Pod manifest -func (m *CephManifestsMaster) GetCleanupPod(node, removalDir string) string { - return `apiVersion: batch/v1 -kind: Job -metadata: - name: rook-cleanup-` + uuid.Must(uuid.NewRandom()).String() + ` -spec: - template: - spec: - restartPolicy: Never - containers: - - name: rook-cleaner - image: rook/ceph:` + m.imageTag + ` - securityContext: - privileged: true - volumeMounts: - - name: cleaner - mountPath: /scrub - command: - - "sh" - - "-c" - - "rm -rf /scrub/*" - nodeSelector: - kubernetes.io/hostname: ` + node + ` - volumes: - - name: cleaner - hostPath: - path: ` + removalDir -} - -// GetCleanupVerificationPod asserts that the dataDirHostPath is empty -func (m *CephManifestsMaster) GetCleanupVerificationPod(node, hostPathDir string) string { - return `apiVersion: batch/v1 -kind: Job -metadata: - name: rook-verify-cleanup-` + uuid.Must(uuid.NewRandom()).String() + ` -spec: - template: - spec: - restartPolicy: Never - containers: - - name: rook-cleaner - image: rook/ceph:` + m.imageTag + ` - securityContext: - privileged: true - volumeMounts: - - name: cleaner - mountPath: /scrub - command: - - "sh" - - "-c" - - | - set -xEeuo pipefail - #Assert dataDirHostPath is empty - if [ "$(ls -A /scrub/)" ]; then - exit 1 - fi - nodeSelector: - kubernetes.io/hostname: ` + node + ` - volumes: - - name: cleaner - hostPath: - path: ` + hostPathDir -} - func (m *CephManifestsMaster) GetBlockPoolDef(poolName string, namespace string, replicaSize string) string { return `apiVersion: ceph.rook.io/v1 kind: CephBlockPool diff --git a/tests/framework/installer/ceph_manifests_v1.1.go b/tests/framework/installer/ceph_manifests_v1.1.go index 9006b4738..f847e55bf 100644 --- a/tests/framework/installer/ceph_manifests_v1.1.go +++ b/tests/framework/installer/ceph_manifests_v1.1.go @@ -18,8 +18,6 @@ package installer import ( "strconv" - - "github.com/google/uuid" ) // CephManifestsV1_1 wraps rook yaml definitions @@ -1668,41 +1666,6 @@ spec: path: mon-endpoints` } -// GetCleanupPod gets a cleanup Pod manifest -func (m *CephManifestsV1_1) GetCleanupPod(node, removalDir string) string { - return `apiVersion: batch/v1 -kind: Job -metadata: - name: rook-cleanup-` + uuid.Must(uuid.NewRandom()).String() + ` -spec: - template: - spec: - restartPolicy: Never - containers: - - name: rook-cleaner - image: rook/ceph:` + m.imageTag + ` - securityContext: - privileged: true - volumeMounts: - - name: cleaner - mountPath: /scrub - command: - - "sh" - - "-c" - - "rm -rf /scrub/*" - nodeSelector: - kubernetes.io/hostname: ` + node + ` - volumes: - - name: cleaner - hostPath: - path: ` + removalDir -} - -// GetCleanupVerificationPod asserts that the dataDirHostPath is empty -func (m *CephManifestsV1_1) GetCleanupVerificationPod(node, hostPathDir string) string { - return "" -} - func (m *CephManifestsV1_1) GetBlockPoolDef(poolName string, namespace string, replicaSize string) string { return `apiVersion: ceph.rook.io/v1 kind: CephBlockPool