diff --git a/pkg/daemon/ceph/client/command.go b/pkg/daemon/ceph/client/command.go index 32f68e256..9712a2658 100644 --- a/pkg/daemon/ceph/client/command.go +++ b/pkg/daemon/ceph/client/command.go @@ -25,9 +25,10 @@ import ( "github.com/rook/rook/pkg/clusterd" ) -// RunAllCephCommandsInToolbox - when running the e2e tests, all ceph commands need to be run in the toolbox. +// RunAllCephCommandsInToolboxPod - when running the e2e tests, all ceph commands need to be run in the toolbox. // Everywhere else, the ceph tools are assumed to be in the container where we can shell out. -var RunAllCephCommandsInToolbox = false +// This is the name of the pod. +var RunAllCephCommandsInToolboxPod string const ( // AdminUsername is the name of the admin user @@ -66,8 +67,8 @@ func FinalizeCephCommandArgs(command string, clusterInfo *ClusterInfo, args []st } // If the command should be run inside the toolbox pod, include the kubectl args to call the toolbox - if RunAllCephCommandsInToolbox { - toolArgs := []string{"exec", "-i", "rook-ceph-tools", "-n", clusterInfo.Namespace, "--", command} + if RunAllCephCommandsInToolboxPod != "" { + toolArgs := []string{"exec", "-i", RunAllCephCommandsInToolboxPod, "-n", clusterInfo.Namespace, "--", command} return Kubectl, append(toolArgs, args...) } diff --git a/pkg/daemon/ceph/client/command_test.go b/pkg/daemon/ceph/client/command_test.go index f6af57537..3f1ba01bc 100644 --- a/pkg/daemon/ceph/client/command_test.go +++ b/pkg/daemon/ceph/client/command_test.go @@ -23,7 +23,7 @@ import ( ) func TestFinalizeCephCommandArgs(t *testing.T) { - RunAllCephCommandsInToolbox = false + RunAllCephCommandsInToolboxPod = "" configDir := "/var/lib/rook/rook-ceph" expectedCommand := "ceph" args := []string{"quorum_status"} @@ -43,7 +43,7 @@ func TestFinalizeCephCommandArgs(t *testing.T) { } func TestFinalizeRadosGWAdminCommandArgs(t *testing.T) { - RunAllCephCommandsInToolbox = false + RunAllCephCommandsInToolboxPod = "" configDir := "/var/lib/rook/rook-ceph" expectedCommand := "radosgw-admin" args := []string{ @@ -73,7 +73,7 @@ func TestFinalizeRadosGWAdminCommandArgs(t *testing.T) { } func TestFinalizeCephCommandArgsToolBox(t *testing.T) { - RunAllCephCommandsInToolbox = true + RunAllCephCommandsInToolboxPod = "rook-ceph-tools" configDir := "/var/lib/rook/rook-ceph" expectedCommand := "ceph" args := []string{"health"} @@ -93,5 +93,5 @@ func TestFinalizeCephCommandArgsToolBox(t *testing.T) { cmd, args := FinalizeCephCommandArgs(expectedCommand, clusterInfo, args, configDir) assert.Exactly(t, "kubectl", cmd) assert.Exactly(t, expectedArgs, args) - RunAllCephCommandsInToolbox = false + RunAllCephCommandsInToolboxPod = "" } diff --git a/pkg/daemon/ceph/client/mon_test.go b/pkg/daemon/ceph/client/mon_test.go index 414cdc6d0..c283de320 100644 --- a/pkg/daemon/ceph/client/mon_test.go +++ b/pkg/daemon/ceph/client/mon_test.go @@ -38,7 +38,7 @@ func TestCephArgs(t *testing.T) { assert.Equal(t, "--name=client.admin", args[3]) assert.Equal(t, "--keyring=/etc/a/client.admin.keyring", args[4]) - RunAllCephCommandsInToolbox = true + RunAllCephCommandsInToolboxPod = "rook-ceph-tools" args = []string{} command, args = FinalizeCephCommandArgs(CephTool, clusterInfo, args, "/etc") assert.Equal(t, Kubectl, command) @@ -51,7 +51,7 @@ func TestCephArgs(t *testing.T) { assert.Equal(t, "--", args[5]) assert.Equal(t, CephTool, args[6]) assert.Equal(t, "--connect-timeout=15", args[7]) - RunAllCephCommandsInToolbox = false + RunAllCephCommandsInToolboxPod = "" // cluster under /var/lib/rook args = []string{"myarg"} diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 64c936899..3284b5e99 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -293,7 +293,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R // Remove finalizer err = removeFinalizer(r.client, request.NamespacedName) if err != nil { - return reconcile.Result{}, errors.Wrap(err, "failed to remove finalize") + return reconcile.Result{}, errors.Wrap(err, "failed to remove finalizer") } // Return and do not requeue. Successful deletion. @@ -421,7 +421,7 @@ func (c *ClusterController) checkIfVolumesExist(cluster *cephv1.CephCluster) err return err } } - if controllerutil.FlexDriverEnabled(c.context) { + if !controllerutil.FlexDriverEnabled(c.context) { logger.Debugf("Flex driver disabled, skipping check for volume attachments for cluster %q", cluster.Namespace) return nil } diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index e03538354..f3d6feb96 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -142,6 +142,7 @@ func IsReadyToReconcile(c client.Client, clustercontext *clusterd.Context, names } } + logger.Debugf("%q: CephCluster %q initial reconcile is not complete yet...", controllerName, namespacedName.Namespace) return cephCluster, false, cephClusterExists, WaitForRequeueIfCephClusterNotReady } diff --git a/tests/framework/clients/block.go b/tests/framework/clients/block.go index 43075df2f..c7f8d5cad 100644 --- a/tests/framework/clients/block.go +++ b/tests/framework/clients/block.go @@ -62,26 +62,26 @@ func (b *BlockOperation) Create(manifest string, size int) (string, error) { } -func (b *BlockOperation) CreateStorageClassAndPVC(csi bool, pvcNamespace, clusterNamespace, systemNamespace, poolName, storageClassName, reclaimPolicy, blockName, mode string) error { - if err := b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockPoolDef(poolName, clusterNamespace, "1")); err != nil { +func (b *BlockOperation) CreateStorageClassAndPVC(pvcNamespace, poolName, storageClassName, reclaimPolicy, blockName, mode string) error { + if err := b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockPool(poolName, "1")); err != nil { return err } - if err := b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockStorageClassDef(csi, poolName, storageClassName, reclaimPolicy, clusterNamespace, systemNamespace)); err != nil { + if err := b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockStorageClass(poolName, storageClassName, reclaimPolicy)); err != nil { return err } - return b.k8sClient.ResourceOperation("apply", b.manifests.GetPVC(blockName, pvcNamespace, storageClassName, mode, "1M")) + return b.k8sClient.ResourceOperation("apply", installer.GetPVC(blockName, pvcNamespace, storageClassName, mode, "1M")) } func (b *BlockOperation) CreatePVC(namespace, claimName, storageClassName, mode, size string) error { - return b.k8sClient.ResourceOperation("apply", b.manifests.GetPVC(claimName, namespace, storageClassName, mode, size)) + return b.k8sClient.ResourceOperation("apply", installer.GetPVC(claimName, namespace, storageClassName, mode, size)) } func (b *BlockOperation) CreatePod(podName, claimName, namespace, mountPoint string, readOnly bool) error { - return b.k8sClient.ResourceOperation("apply", b.manifests.GetPod(podName, claimName, namespace, mountPoint, readOnly)) + return b.k8sClient.ResourceOperation("apply", installer.GetPodWithVolume(podName, claimName, namespace, mountPoint, readOnly)) } func (b *BlockOperation) CreateStorageClass(csi bool, poolName, storageClassName, reclaimPolicy, namespace string) error { - return b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockStorageClassDef(csi, poolName, storageClassName, reclaimPolicy, namespace, installer.SystemNamespace(namespace))) + return b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockStorageClass(poolName, storageClassName, reclaimPolicy)) } func (b *BlockOperation) DeletePVC(namespace, claimName string) error { @@ -91,27 +91,27 @@ func (b *BlockOperation) DeletePVC(namespace, claimName string) error { } func (b *BlockOperation) CreatePVCRestore(namespace, claimName, snapshotName, storageClassName, mode, size string) error { - return b.k8sClient.ResourceOperation("apply", b.manifests.GetPVCRestore(claimName, snapshotName, namespace, storageClassName, mode, size)) + return b.k8sClient.ResourceOperation("apply", installer.GetPVCRestore(claimName, snapshotName, namespace, storageClassName, mode, size)) } func (b *BlockOperation) CreatePVCClone(namespace, cloneClaimName, parentClaimName, storageClassName, mode, size string) error { - return b.k8sClient.ResourceOperation("apply", b.manifests.GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, mode, size)) + return b.k8sClient.ResourceOperation("apply", installer.GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, mode, size)) } func (b *BlockOperation) CreateSnapshotClass(snapshotClassName, deletePolicy, namespace string) error { - return b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockSnapshotClass(snapshotClassName, namespace, namespace, deletePolicy)) + return b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockSnapshotClass(snapshotClassName, deletePolicy)) } func (b *BlockOperation) DeleteSnapshotClass(snapshotClassName, deletePolicy, namespace string) error { - return b.k8sClient.ResourceOperation("delete", b.manifests.GetBlockSnapshotClass(snapshotClassName, namespace, namespace, deletePolicy)) + return b.k8sClient.ResourceOperation("delete", b.manifests.GetBlockSnapshotClass(snapshotClassName, deletePolicy)) } func (b *BlockOperation) CreateSnapshot(snapshotName, claimName, snapshotClassName, namespace string) error { - return b.k8sClient.ResourceOperation("apply", b.manifests.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace)) + return b.k8sClient.ResourceOperation("apply", installer.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace)) } func (b *BlockOperation) DeleteSnapshot(snapshotName, claimName, snapshotClassName, namespace string) error { - return b.k8sClient.ResourceOperation("delete", b.manifests.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace)) + return b.k8sClient.ResourceOperation("delete", installer.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace)) } func (b *BlockOperation) DeleteStorageClass(storageClassName string) error { diff --git a/tests/framework/clients/bucket.go b/tests/framework/clients/bucket.go index 4d62ac01f..70e73f8b5 100644 --- a/tests/framework/clients/bucket.go +++ b/tests/framework/clients/bucket.go @@ -37,20 +37,20 @@ func CreateBucketOperation(k8sh *utils.K8sHelper, manifests installer.CephManife } func (b *BucketOperation) CreateBucketStorageClass(namespace string, storeName string, storageClassName string, reclaimPolicy string, region string) error { - return b.k8sh.ResourceOperation("create", b.manifests.GetBucketStorageClass(namespace, storeName, storageClassName, reclaimPolicy, region)) + return b.k8sh.ResourceOperation("create", b.manifests.GetBucketStorageClass(storeName, storageClassName, reclaimPolicy, region)) } func (b *BucketOperation) DeleteBucketStorageClass(namespace string, storeName string, storageClassName string, reclaimPolicy string, region string) error { - err := b.k8sh.ResourceOperation("delete", b.manifests.GetBucketStorageClass(namespace, storeName, storageClassName, reclaimPolicy, region)) + err := b.k8sh.ResourceOperation("delete", b.manifests.GetBucketStorageClass(storeName, storageClassName, reclaimPolicy, region)) return err } func (b *BucketOperation) CreateObc(obcName string, storageClassName string, bucketName string, maxObject string, createBucket bool) error { - return b.k8sh.ResourceOperation("create", b.manifests.GetObc(obcName, storageClassName, bucketName, maxObject, createBucket)) + return b.k8sh.ResourceOperation("create", b.manifests.GetOBC(obcName, storageClassName, bucketName, maxObject, createBucket)) } func (b *BucketOperation) DeleteObc(obcName string, storageClassName string, bucketName string, maxObject string, createBucket bool) error { - return b.k8sh.ResourceOperation("delete", b.manifests.GetObc(obcName, storageClassName, bucketName, maxObject, createBucket)) + return b.k8sh.ResourceOperation("delete", b.manifests.GetOBC(obcName, storageClassName, bucketName, maxObject, createBucket)) } // CheckOBC, returns true if the obc, secret and configmap are all in the "check" state, diff --git a/tests/framework/clients/client.go b/tests/framework/clients/client.go index 19236b890..2b3834b23 100644 --- a/tests/framework/clients/client.go +++ b/tests/framework/clients/client.go @@ -42,7 +42,7 @@ func CreateClientOperation(k8sh *utils.K8sHelper, manifests installer.CephManife // Create creates a client in Rook func (c *ClientOperation) Create(name, namespace string, caps map[string]string) error { logger.Infof("creating the client via CRD") - if err := c.k8sh.ResourceOperation("apply", c.manifests.GetClient(namespace, name, caps)); err != nil { + if err := c.k8sh.ResourceOperation("apply", c.manifests.GetClient(name, caps)); err != nil { return err } return nil @@ -76,7 +76,7 @@ func (c *ClientOperation) Get(clusterInfo *client.ClusterInfo, clientName string func (c *ClientOperation) Update(clusterInfo *client.ClusterInfo, clientName string, caps map[string]string) (updatedcaps map[string]string, error error) { context := c.k8sh.MakeContext() logger.Infof("updating the client via CRD") - if err := c.k8sh.ResourceOperation("apply", c.manifests.GetClient(clusterInfo.Namespace, clientName, caps)); err != nil { + if err := c.k8sh.ResourceOperation("apply", c.manifests.GetClient(clientName, caps)); err != nil { return nil, err } diff --git a/tests/framework/clients/filesystem.go b/tests/framework/clients/filesystem.go index 3a442ec6a..049e06fdc 100644 --- a/tests/framework/clients/filesystem.go +++ b/tests/framework/clients/filesystem.go @@ -42,7 +42,7 @@ func CreateFilesystemOperation(k8sh *utils.K8sHelper, manifests installer.CephMa // Create creates a filesystem in Rook func (f *FilesystemOperation) Create(name, namespace string, activeCount int) error { logger.Infof("creating the filesystem via CRD") - if err := f.k8sh.ResourceOperation("apply", f.manifests.GetFilesystem(namespace, name, activeCount)); err != nil { + if err := f.k8sh.ResourceOperation("apply", f.manifests.GetFilesystem(name, activeCount)); err != nil { return err } @@ -58,32 +58,32 @@ func (f *FilesystemOperation) Create(name, namespace string, activeCount int) er // CreateStorageClass creates a storage class for CephFS clients func (f *FilesystemOperation) CreateStorageClass(fsName, systemNamespace, namespace, storageClassName string) error { - return f.k8sh.ResourceOperation("apply", f.manifests.GetFileStorageClassDef(fsName, storageClassName, systemNamespace, namespace)) + return f.k8sh.ResourceOperation("apply", f.manifests.GetFileStorageClass(fsName, storageClassName)) } // CreateSnapshotClass creates a snapshot class for CephFS clients func (f *FilesystemOperation) CreateSnapshotClass(snapshotClassName, reclaimPolicy, namespace string) error { - return f.k8sh.ResourceOperation("apply", f.manifests.GetFileStorageSnapshotClass(snapshotClassName, namespace, namespace, reclaimPolicy)) + return f.k8sh.ResourceOperation("apply", f.manifests.GetFileStorageSnapshotClass(snapshotClassName, reclaimPolicy)) } // CreatePVCRestore creates a pvc from snapshot func (f *FilesystemOperation) CreatePVCRestore(namespace, claimName, snapshotName, storageClassName, mode, size string) error { - return f.k8sh.ResourceOperation("apply", f.manifests.GetPVCRestore(claimName, snapshotName, namespace, storageClassName, mode, size)) + return f.k8sh.ResourceOperation("apply", installer.GetPVCRestore(claimName, snapshotName, namespace, storageClassName, mode, size)) } // CreatePVCClone creates a pvc from pvc func (f *FilesystemOperation) CreatePVCClone(namespace, cloneClaimName, parentClaimName, storageClassName, mode, size string) error { - return f.k8sh.ResourceOperation("apply", f.manifests.GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, mode, size)) + return f.k8sh.ResourceOperation("apply", installer.GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, mode, size)) } // CreateSnapshot creates a snapshot from pvc func (f *FilesystemOperation) CreateSnapshot(snapshotName, claimName, snapshotClassName, namespace string) error { - return f.k8sh.ResourceOperation("apply", f.manifests.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace)) + return f.k8sh.ResourceOperation("apply", installer.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace)) } // DeleteSnapshot deletes the snapshot func (f *FilesystemOperation) DeleteSnapshot(snapshotName, claimName, snapshotClassName, namespace string) error { - return f.k8sh.ResourceOperation("delete", f.manifests.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace)) + return f.k8sh.ResourceOperation("delete", installer.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace)) } func (f *FilesystemOperation) DeletePVC(namespace, claimName string) error { @@ -104,21 +104,21 @@ func (f *FilesystemOperation) DeleteStorageClass(storageClassName string) error } func (f *FilesystemOperation) CreatePVC(namespace, claimName, storageClassName, mode, size string) error { - return f.k8sh.ResourceOperation("apply", f.manifests.GetPVC(claimName, namespace, storageClassName, mode, size)) + return f.k8sh.ResourceOperation("apply", installer.GetPVC(claimName, namespace, storageClassName, mode, size)) } func (f *FilesystemOperation) CreatePod(podName, claimName, namespace, mountPoint string, readOnly bool) error { - return f.k8sh.ResourceOperation("apply", f.manifests.GetPod(podName, claimName, namespace, mountPoint, readOnly)) + return f.k8sh.ResourceOperation("apply", installer.GetPodWithVolume(podName, claimName, namespace, mountPoint, readOnly)) } func (f *FilesystemOperation) DeleteSnapshotClass(snapshotClassName, deletePolicy, namespace string) error { - return f.k8sh.ResourceOperation("delete", f.manifests.GetFileStorageSnapshotClass(snapshotClassName, namespace, namespace, deletePolicy)) + return f.k8sh.ResourceOperation("delete", f.manifests.GetFileStorageSnapshotClass(snapshotClassName, deletePolicy)) } // ScaleDown scales down the number of active metadata servers of a filesystem in Rook func (f *FilesystemOperation) ScaleDown(name, namespace string) error { logger.Infof("scaling down the number of filesystem active metadata servers via CRD") - if err := f.k8sh.ResourceOperation("apply", f.manifests.GetFilesystem(namespace, name, 1)); err != nil { + if err := f.k8sh.ResourceOperation("apply", f.manifests.GetFilesystem(name, 1)); err != nil { return err } @@ -144,7 +144,7 @@ func (f *FilesystemOperation) Delete(name, namespace string) error { } logger.Infof("Deleted filesystem %s in namespace %s", name, namespace) - return f.k8sh.WaitForCustomResourceDeletion(namespace, crdCheckerFunc) + return f.k8sh.WaitForCustomResourceDeletion(namespace, name, crdCheckerFunc) } // List lists filesystems in Rook diff --git a/tests/framework/clients/nfs.go b/tests/framework/clients/nfs.go index eb3b60db8..c3b7ab2d8 100644 --- a/tests/framework/clients/nfs.go +++ b/tests/framework/clients/nfs.go @@ -42,7 +42,7 @@ func CreateNFSOperation(k8sh *utils.K8sHelper, manifests installer.CephManifests func (n *NFSOperation) Create(namespace, name, pool string, daemonCount int) error { logger.Infof("creating the NFS daemons via CRD") - if err := n.k8sh.ResourceOperation("apply", n.manifests.GetNFS(namespace, name, pool, daemonCount)); err != nil { + if err := n.k8sh.ResourceOperation("apply", n.manifests.GetNFS(name, pool, daemonCount)); err != nil { return err } diff --git a/tests/framework/clients/object.go b/tests/framework/clients/object.go index 27e506d6f..6fdbc7417 100644 --- a/tests/framework/clients/object.go +++ b/tests/framework/clients/object.go @@ -43,7 +43,7 @@ func CreateObjectOperation(k8sh *utils.K8sHelper, manifests installer.CephManife func (o *ObjectOperation) Create(namespace, storeName string, replicaCount int32) error { logger.Infof("creating the object store via CRD") - if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStore(namespace, storeName, int(replicaCount), rgwPort)); err != nil { + if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStore(storeName, int(replicaCount), rgwPort)); err != nil { return err } diff --git a/tests/framework/clients/object_user.go b/tests/framework/clients/object_user.go index c89a7beda..aee822228 100644 --- a/tests/framework/clients/object_user.go +++ b/tests/framework/clients/object_user.go @@ -68,7 +68,7 @@ func (o *ObjectUserOperation) UserSecretExists(namespace string, store string, u func (o *ObjectUserOperation) Create(namespace string, userid string, displayName string, store string) error { logger.Infof("creating the object store user via CRD") - if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStoreUser(namespace, userid, displayName, store)); err != nil { + if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStoreUser(userid, displayName, store)); err != nil { return err } return nil diff --git a/tests/framework/clients/pool.go b/tests/framework/clients/pool.go index 6bfed39bd..8571a4ad4 100644 --- a/tests/framework/clients/pool.go +++ b/tests/framework/clients/pool.go @@ -50,7 +50,7 @@ func (p *PoolOperation) Update(name, namespace string, replicas int) error { } func (p *PoolOperation) createOrUpdatePool(name, namespace, action string, replicas int) error { - return p.k8sh.ResourceOperation(action, p.manifests.GetBlockPoolDef(name, namespace, strconv.Itoa(replicas))) + return p.k8sh.ResourceOperation(action, p.manifests.GetBlockPool(name, strconv.Itoa(replicas))) } func (p *PoolOperation) ListCephPools(clusterInfo *client.ClusterInfo) ([]client.CephStoragePoolSummary, error) { @@ -147,5 +147,5 @@ func (p *PoolOperation) DeletePool(blockClient *BlockOperation, clusterInfo *cli return err } - return p.k8sh.WaitForCustomResourceDeletion(clusterInfo.Namespace, crdCheckerFunc) + return p.k8sh.WaitForCustomResourceDeletion(clusterInfo.Namespace, poolName, crdCheckerFunc) } diff --git a/tests/framework/clients/rbd-mirror.go b/tests/framework/clients/rbd-mirror.go index 2ae2aeedb..5e9d0bc2e 100644 --- a/tests/framework/clients/rbd-mirror.go +++ b/tests/framework/clients/rbd-mirror.go @@ -41,7 +41,7 @@ func CreateRBDMirrorOperation(k8sh *utils.K8sHelper, manifests installer.CephMan func (r *RBDMirrorOperation) Create(namespace, name string, daemonCount int) error { logger.Infof("creating the RBDMirror daemons via CRD") - if err := r.k8sh.ResourceOperation("apply", r.manifests.GetRBDMirror(namespace, name, daemonCount)); err != nil { + if err := r.k8sh.ResourceOperation("apply", r.manifests.GetRBDMirror(name, daemonCount)); err != nil { return err } diff --git a/tests/framework/installer/cassandra_installer.go b/tests/framework/installer/cassandra_installer.go index 4e0f6c858..a18216a2f 100644 --- a/tests/framework/installer/cassandra_installer.go +++ b/tests/framework/installer/cassandra_installer.go @@ -131,7 +131,7 @@ func (ci *CassandraInstaller) DeleteCassandraCluster(namespace string) { _, err := ci.k8sHelper.RookClientset.CassandraV1alpha1().Clusters(namespace).Get(ctx, namespace, metav1.GetOptions{}) return err } - err = ci.k8sHelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc) + err = ci.k8sHelper.WaitForCustomResourceDeletion(namespace, namespace, crdCheckerFunc) assert.NoError(ci.T(), err) // Delete Namespace diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index cbd786029..1e65cc2a2 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -28,6 +28,7 @@ import ( "time" "github.com/google/uuid" + "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/cluster" @@ -59,33 +60,25 @@ var ( // CephInstaller wraps installing and uninstalling rook on a platform type CephInstaller struct { + settings *TestCephSettings Manifests CephManifests k8shelper *utils.K8sHelper hostPathToDelete string helmHelper *utils.HelmHelper - useHelm bool - clusterName string k8sVersion string changeHostnames bool - CephVersion cephv1.CephVersionSpec T func() *testing.T - cleanupHost bool } // CreateCephOperator creates rook-operator via kubectl -func (h *CephInstaller) CreateCephOperator(namespace string) (err error) { +func (h *CephInstaller) CreateCephOperator() (err error) { logger.Infof("Starting Rook Operator") // creating clusterrolebinding for kubeadm env. h.k8shelper.CreateAnonSystemClusterBinding() - var v1ExtensionsSupported bool - minVersion := "v1.16.0" - if h.k8shelper.VersionAtLeast(minVersion) { - v1ExtensionsSupported = true - } // creating rook resources logger.Info("Creating Rook CRDs") - resources := h.Manifests.GetRookCRDs(v1ExtensionsSupported) + resources := h.Manifests.GetCRDs(h.k8shelper) if _, err = h.k8shelper.KubectlWithStdin(resources, createFromStdinArgs...); err != nil { return err } @@ -96,47 +89,46 @@ func (h *CephInstaller) CreateCephOperator(namespace string) (err error) { assert.NoError(h.T(), err) } - err = h.k8shelper.CreateNamespace(namespace) - if err != nil { - if kerrors.IsAlreadyExists(err) { - logger.Warningf("Namespace %q already exists!!!", namespace) - } else { - return fmt.Errorf("failed to create namespace %q. %v", namespace, err) - } + // The operator namespace needs to be created explicitly, while the cluster namespace is created with the common.yaml + if err := h.k8shelper.CreateNamespace(h.settings.OperatorNamespace); err != nil { + return err } - // disable admission controller for Kubernetes version older than v1.16.0 and for upgrade test as api version v1 require minimum v0.7 controller runtime and upgrade test still using - // older version of controller runtime. - if !utils.IsPlatformOpenShift() && namespace != "upgrade-ns-system" { - err = h.startAdmissionController(namespace) - if err != nil { - return fmt.Errorf("Failed to start admission controllers: %v", err) - } - } - - rookOperator := h.Manifests.GetRookOperator(namespace) - _, err = h.k8shelper.KubectlWithStdin(rookOperator, createFromStdinArgs...) + // Create the namespace and RBAC before starting the operator + _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetCommon(), createFromStdinArgs...) if err != nil { return fmt.Errorf("Failed to create rook-operator pod: %v ", err) } - logger.Infof("Rook Operator started") + err = h.startAdmissionController() + if err != nil { + return fmt.Errorf("Failed to start admission controllers: %v", err) + } + _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetOperator(), createFromStdinArgs...) + if err != nil { + return fmt.Errorf("Failed to create rook-operator pod: %v", err) + } + + logger.Infof("Rook operator started") return nil } -func (h *CephInstaller) startAdmissionController(namespace string) error { - err := h.k8shelper.CreateNamespace(namespace) - if err != nil { - if kerrors.IsAlreadyExists(err) { - logger.Warningf("Namespace %q already exists!!!", namespace) - } else { - return fmt.Errorf("failed to create namespace %q. %v", namespace, err) - } - } +func (h *CephInstaller) startAdmissionController() error { if !h.k8shelper.VersionAtLeast("v1.16.0") { + logger.Info("skipping the admission controller on K8s version older than v1.16") return nil } + if !h.settings.EnableAdmissionController { + logger.Info("skipping admission controller for this test suite") + return nil + } + // disable admission controller for Kubernetes version older than v1.16.0 + if utils.IsPlatformOpenShift() { + logger.Info("skipping the admission controller on OpenShift") + return nil + } + rootPath, err := utils.FindRookRoot() if err != nil { return fmt.Errorf("failed to find rook root. %v", err) @@ -146,7 +138,7 @@ func (h *CephInstaller) startAdmissionController(namespace string) error { return fmt.Errorf("failed to find user home directory. %v", err) } scriptPath := path.Join(rootPath, "tests/scripts/deploy_admission_controller_test.sh") - err = h.k8shelper.MakeContext().Executor.ExecuteCommandWithEnv([]string{fmt.Sprintf("NAMESPACE=%s", namespace), fmt.Sprintf("HOME=%s", userHome)}, "bash", scriptPath) + err = h.k8shelper.MakeContext().Executor.ExecuteCommandWithEnv([]string{fmt.Sprintf("NAMESPACE=%s", h.settings.OperatorNamespace), fmt.Sprintf("HOME=%s", userHome)}, "bash", scriptPath) if err != nil { return err } @@ -155,41 +147,51 @@ func (h *CephInstaller) startAdmissionController(namespace string) error { } // CreateRookOperatorViaHelm creates rook operator via Helm chart named local/rook present in local repo -func (h *CephInstaller) CreateRookOperatorViaHelm(namespace, chartSettings string) error { +func (h *CephInstaller) CreateRookOperatorViaHelm(chartSettings string) error { // creating clusterrolebinding for kubeadm env. h.k8shelper.CreateAnonSystemClusterBinding() - // disable admission controller for Kubernetes version older than v1.16.0 - if !utils.IsPlatformOpenShift() { - if err := h.startAdmissionController(namespace); err != nil { - return fmt.Errorf("Failed to start admission controllers: %v", err) - } + // create the operator namespace before the admission controller is created + if err := h.k8shelper.CreateNamespace(h.settings.OperatorNamespace); err != nil { + return fmt.Errorf("failed to create namespace %s. %v", h.settings.Namespace, err) } - - if err := h.helmHelper.InstallLocalRookHelmChart(namespace, chartSettings); err != nil { + if err := h.startAdmissionController(); err != nil { + return fmt.Errorf("failed to start admission controllers: %v", err) + } + if err := h.helmHelper.InstallLocalRookHelmChart(h.settings.Namespace, chartSettings); err != nil { return fmt.Errorf("failed to install rook operator via helm, err : %v", err) } + // create the cluster namespace since it wasn't created by common.yaml + if err := h.k8shelper.CreateNamespace(h.settings.Namespace); err != nil { + return fmt.Errorf("failed to create namespace %s. %v", h.settings.Namespace, err) + } return nil } // CreateRookToolbox creates rook-ceph-tools via kubectl -func (h *CephInstaller) CreateRookToolbox(namespace string) (err error) { +func (h *CephInstaller) CreateRookToolbox(manifests CephManifests) (err error) { logger.Infof("Starting Rook toolbox") - rookToolbox := h.Manifests.GetRookToolBox(namespace) - - _, err = h.k8shelper.KubectlWithStdin(rookToolbox, createFromStdinArgs...) - + _, err = h.k8shelper.KubectlWithStdin(manifests.GetToolbox(), createFromStdinArgs...) if err != nil { - return fmt.Errorf("Failed to create rook-toolbox pod : %v ", err) + return errors.Wrap(err, "failed to create rook-toolbox pod") } - if !h.k8shelper.IsPodRunning("rook-ceph-tools", namespace) { - return fmt.Errorf("Rook Toolbox couldn't start") + if err := h.k8shelper.WaitForLabeledPodsToRun("app=rook-ceph-tools", h.settings.Namespace); err != nil { + return errors.Wrap(err, "Rook Toolbox couldn't start") } logger.Infof("Rook Toolbox started") + podNames, err := h.k8shelper.GetPodNamesForApp("rook-ceph-tools", manifests.Settings().Namespace) + assert.NoError(h.T(), err) + for _, podName := range podNames { + // All e2e tests should run ceph commands in the toolbox since we are not inside a container + logger.Infof("setting active toolbox pod to %q", podName) + client.RunAllCephCommandsInToolboxPod = podName + break + } + return nil } @@ -205,24 +207,16 @@ func (h *CephInstaller) Execute(command string, parameters []string, namespace s return nil, result } -// CreateRookCluster creates rook cluster via kubectl -func (h *CephInstaller) CreateRookCluster(namespace, systemNamespace, storeType string, usePVC bool, storageClassName string, - mon cephv1.MonSpec, startWithAllNodes, multipleMgrs, skipOSDCreation bool, useCrashPruner bool, cephVersion cephv1.CephVersionSpec) error { +// CreateCephCluster creates rook cluster via kubectl +func (h *CephInstaller) CreateCephCluster() error { ctx := context.TODO() - dataDirHostPath, err := h.initTestDir(namespace) + var err error + h.settings.DataDirHostPath, err = h.initTestDir(h.settings.Namespace) if err != nil { return fmt.Errorf("failed to create test dir. %+v", err) } - logger.Infof("Creating cluster: namespace=%s, systemNamespace=%s, storeType=%s, dataDirHostPath=%s, usePVC=%v, storageClassName=%s, startWithAllNodes=%t, mons=%+v", - namespace, systemNamespace, storeType, dataDirHostPath, usePVC, storageClassName, startWithAllNodes, mon) - - logger.Infof("Creating namespace %s", namespace) - ns := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}} - _, err = h.k8shelper.Clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) - if err != nil && !kerrors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create namespace %s. %+v", namespace, err) - } + logger.Infof("Creating cluster with settings: %+v", h.settings) logger.Infof("Creating custom ceph.conf settings") customSettings := map[string]string{ @@ -233,63 +227,53 @@ osd_pool_default_size = 1 customCM := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "rook-config-override", - Namespace: namespace, + Namespace: h.settings.Namespace, }, Data: customSettings, } - if _, err := h.k8shelper.Clientset.CoreV1().ConfigMaps(namespace).Create(ctx, customCM, metav1.CreateOptions{}); err != nil { + if _, err := h.k8shelper.Clientset.CoreV1().ConfigMaps(h.settings.Namespace).Create(ctx, customCM, metav1.CreateOptions{}); err != nil { return fmt.Errorf("failed to create custom ceph.conf. %+v", err) } - // Skip this step since the helm chart already includes the roles and bindings - if !h.useHelm { - logger.Infof("Creating cluster roles") - roles := h.Manifests.GetClusterRoles(namespace, systemNamespace) - if _, err := h.k8shelper.KubectlWithStdin(roles, createFromStdinArgs...); err != nil { - return fmt.Errorf("Failed to create cluster roles. %+v", err) - } - } - logger.Info("Starting Rook Cluster with yaml") - settings := &clusterSettings{h.clusterName, namespace, storeType, dataDirHostPath, mon.Count, multipleMgrs, 0, usePVC, storageClassName, skipOSDCreation, cephVersion, useCrashPruner} - rookCluster := h.Manifests.GetRookCluster(settings) + rookCluster := h.Manifests.GetCephCluster() logger.Info(rookCluster) - maxTry := 3 + maxTry := 10 for i := 0; i < maxTry; i++ { _, err := h.k8shelper.KubectlWithStdin(rookCluster, createFromStdinArgs...) if err == nil { break } if i == maxTry-1 { - return fmt.Errorf("failed to create rook cluster. %v ", err) + return fmt.Errorf("failed to create rook cluster. %v", err) } logger.Infof("failed to create rook cluster, trying again... %v", err) time.Sleep(5 * time.Second) } - if err := h.k8shelper.WaitForPodCount("app=rook-ceph-mon", namespace, mon.Count); err != nil { + if err := h.k8shelper.WaitForPodCount("app=rook-ceph-mon", h.settings.Namespace, h.settings.Mons); err != nil { return err } - if err := h.k8shelper.WaitForPodCount("app=rook-ceph-mgr", namespace, 1); err != nil { + if err := h.k8shelper.WaitForPodCount("app=rook-ceph-mgr", h.settings.Namespace, 1); err != nil { return err } - if !skipOSDCreation { - if err := h.k8shelper.WaitForPodCount("app=rook-ceph-osd", namespace, 1); err != nil { + if !h.settings.SkipOSDCreation { + if err := h.k8shelper.WaitForPodCount("app=rook-ceph-osd", h.settings.Namespace, 1); err != nil { return err } } - if useCrashPruner { - if err := h.k8shelper.WaitForCronJob("rook-ceph-crashcollector-pruner", namespace); err != nil { + if h.settings.UseCrashPruner { + if err := h.k8shelper.WaitForCronJob("rook-ceph-crashcollector-pruner", h.settings.Namespace); err != nil { return err } } logger.Infof("Rook Cluster started") - if !skipOSDCreation { - err = h.k8shelper.WaitForLabeledPodsToRun("app=rook-ceph-osd", namespace) + if !h.settings.SkipOSDCreation { + err = h.k8shelper.WaitForLabeledPodsToRun("app=rook-ceph-osd", h.settings.Namespace) return err } @@ -297,43 +281,39 @@ osd_pool_default_size = 1 } // CreateRookExternalCluster creates rook external cluster via kubectl -func (h *CephInstaller) CreateRookExternalCluster(namespace, firstClusterNamespace string) error { - ctx := context.TODO() - dataDirHostPath, err := h.initTestDir(namespace) +func (h *CephInstaller) CreateRookExternalCluster(externalManifests CephManifests) error { + var err error + externalSettings := externalManifests.Settings() + externalSettings.DataDirHostPath, err = h.initTestDir(externalSettings.Namespace) if err != nil { return fmt.Errorf("failed to create test dir. %+v", err) } - logger.Infof("Creating external cluster: namespace=%q, firstClusterNamespace=%q", namespace, firstClusterNamespace) - logger.Infof("Creating namespace %s", namespace) - ns := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}} - _, err = h.k8shelper.Clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) - if err != nil && !kerrors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create namespace %q. %v", namespace, err) - } + logger.Infof("Creating external cluster %q with core storage namespace %q", externalSettings.Namespace, h.settings.Namespace) - // Skip this step since the helm chart already includes the roles and bindings - if !h.useHelm { - logger.Infof("Creating external cluster roles") - roles := h.Manifests.GetClusterExternalRoles(namespace, firstClusterNamespace) - if _, err := h.k8shelper.KubectlWithStdin(roles, createFromStdinArgs...); err != nil { - return fmt.Errorf("failed to create cluster roles. %v", err) - } + logger.Infof("Creating external cluster roles") + roles := externalManifests.GetCommonExternal() + if _, err := h.k8shelper.KubectlWithStdin(roles, createFromStdinArgs...); err != nil { + return errors.Wrap(err, "failed to create cluster roles") } // Inject connection information from the first cluster logger.Info("Injecting cluster connection information") - err = h.InjectRookExternalClusterInfo(namespace, firstClusterNamespace) + err = h.injectRookExternalClusterInfo(externalSettings) if err != nil { - return fmt.Errorf("failed to inject cluster information into the external cluster. %v", err) + return errors.Wrap(err, "failed to inject cluster information into the external cluster") } // Start the external cluster logger.Infof("Starting Rook External Cluster with yaml") - settings := &clusterExternalSettings{namespace, dataDirHostPath} - rookCluster := h.Manifests.GetRookExternalCluster(settings) + rookCluster := externalManifests.GetExternalCephCluster() if _, err := h.k8shelper.KubectlWithStdin(rookCluster, createFromStdinArgs...); err != nil { - return fmt.Errorf("failed to create rook external cluster. %v ", err) + return errors.Wrap(err, "failed to create rook external cluster") + } + + logger.Infof("Running toolbox on external namespace %q", externalSettings.Namespace) + if err := h.CreateRookToolbox(externalManifests); err != nil { + return errors.Wrap(err, "failed to start toolbox on external cluster") } logger.Info("Rook external cluster started") @@ -341,28 +321,28 @@ func (h *CephInstaller) CreateRookExternalCluster(namespace, firstClusterNamespa } // InjectRookExternalClusterInfo inject connection information for an external cluster -func (h *CephInstaller) InjectRookExternalClusterInfo(namespace, firstClusterNamespace string) error { +func (h *CephInstaller) injectRookExternalClusterInfo(externalSettings *TestCephSettings) error { ctx := context.TODO() // get config map - cm, err := h.GetRookExternalClusterMonConfigMap(firstClusterNamespace) + cm, err := h.GetRookExternalClusterMonConfigMap() if err != nil { return fmt.Errorf("failed to get configmap. %v", err) } // create config map - _, err = h.k8shelper.Clientset.CoreV1().ConfigMaps(namespace).Create(ctx, cm, metav1.CreateOptions{}) + _, err = h.k8shelper.Clientset.CoreV1().ConfigMaps(externalSettings.Namespace).Create(ctx, cm, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("failed to create configmap. %v", err) } // get secret - secret, err := h.GetRookExternalClusterMonSecret(firstClusterNamespace) + secret, err := h.GetRookExternalClusterMonSecret() if err != nil { return fmt.Errorf("failed to get secret. %v", err) } // create secret - _, err = h.k8shelper.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + _, err = h.k8shelper.Clientset.CoreV1().Secrets(externalSettings.Namespace).Create(ctx, secret, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("failed to create secret. %v", err) } @@ -371,10 +351,10 @@ func (h *CephInstaller) InjectRookExternalClusterInfo(namespace, firstClusterNam } // GetRookExternalClusterMonConfigMap gets the monitor kubernetes configmap of the external cluster -func (h *CephInstaller) GetRookExternalClusterMonConfigMap(namespace string) (*v1.ConfigMap, error) { +func (h *CephInstaller) GetRookExternalClusterMonConfigMap() (*v1.ConfigMap, error) { ctx := context.TODO() configMapName := "rook-ceph-mon-endpoints" - externalCM, err := h.k8shelper.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, configMapName, metav1.GetOptions{}) + externalCM, err := h.k8shelper.Clientset.CoreV1().ConfigMaps(h.settings.Namespace).Get(ctx, configMapName, metav1.GetOptions{}) if err != nil { return nil, fmt.Errorf("failed to get secret. %v", err) } @@ -386,11 +366,11 @@ func (h *CephInstaller) GetRookExternalClusterMonConfigMap(namespace string) (*v } // GetRookExternalClusterMonSecret gets the monitor kubernetes secret of the external cluster -func (h *CephInstaller) GetRookExternalClusterMonSecret(namespace string) (*v1.Secret, error) { +func (h *CephInstaller) GetRookExternalClusterMonSecret() (*v1.Secret, error) { ctx := context.TODO() secretName := "rook-ceph-mon" //nolint:gosec // We safely suppress gosec in tests file - externalSecret, err := h.k8shelper.Clientset.CoreV1().Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{}) + externalSecret, err := h.k8shelper.Clientset.CoreV1().Secrets(h.settings.Namespace).Get(ctx, secretName, metav1.GetOptions{}) if err != nil { return nil, fmt.Errorf("failed to get secret. %v", err) } @@ -444,57 +424,53 @@ func (h *CephInstaller) GetNodeHostnames() ([]string, error) { return names, nil } -// InstallRook installs rook on k8s -func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, storageClassName string, - mon cephv1.MonSpec, startWithAllNodes bool, rbdMirrorWorkers int, skipOSDCreation bool, rookVersion string) (bool, error) { +func (h *CephInstaller) InstallRook() (bool, error) { + + if h.settings.RookVersion != VersionMaster { + // make sure we have the images from a previous release locally so the test doesn't hit a timeout + assert.NoError(h.T(), h.k8shelper.GetDockerImage("rook/ceph:"+h.settings.RookVersion)) + } + + assert.NoError(h.T(), h.k8shelper.GetDockerImage(h.settings.CephVersion.Image)) ctx := context.TODO() var err error k8sversion := h.k8shelper.GetK8sServerVersion() - logger.Infof("Installing rook on k8s %s", k8sversion) + logger.Infof("Installing rook on K8s %s", k8sversion) - startDiscovery := true - onamespace := namespace + startDiscovery := false // Create rook operator - if h.useHelm { + if h.settings.UseHelm { // disable the discovery daemonset with the helm chart - settings := "enableDiscoveryDaemon=false,image.tag=master" - startDiscovery = false - err = h.CreateRookOperatorViaHelm(namespace, settings) + startDiscovery = true + err = h.CreateRookOperatorViaHelm("enableDiscoveryDaemon=true,image.tag=master") if err != nil { - logger.Errorf("Rook Operator not installed ,error -> %v", err) + logger.Errorf("helm install failed. %v", err) return false, err - } } else { - onamespace = SystemNamespace(namespace) - err := h.CreateCephOperator(onamespace) + err := h.CreateCephOperator() if err != nil { - logger.Errorf("Rook Operator not installed ,error -> %v", err) + logger.Errorf("operator install failed. %v", err) return false, err } } - if !h.k8shelper.IsPodInExpectedState("rook-ceph-operator", onamespace, "Running") { + if !h.k8shelper.IsPodInExpectedState("rook-ceph-operator", h.settings.OperatorNamespace, "Running") { logger.Error("rook-ceph-operator is not running") - h.k8shelper.GetLogsFromNamespace(onamespace, "test-setup", utils.TestEnvName()) + h.k8shelper.GetLogsFromNamespace(h.settings.OperatorNamespace, "test-setup", utils.TestEnvName()) logger.Error("rook-ceph-operator is not Running, abort!") return false, err } - useCrashPruner := rookVersion == VersionMaster - // Create rook cluster - multipleManagers := false - err = h.CreateRookCluster(namespace, onamespace, storeType, usePVC, storageClassName, - cephv1.MonSpec{Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode}, startWithAllNodes, - multipleManagers, skipOSDCreation, useCrashPruner, h.CephVersion) + err = h.CreateCephCluster() if err != nil { - logger.Errorf("Rook cluster %s not installed, error -> %v", namespace, err) + logger.Errorf("Cluster %q install failed. %v", h.settings.Namespace, err) return false, err } - discovery, err := h.k8shelper.Clientset.AppsV1().DaemonSets(onamespace).Get(ctx, "rook-discover", metav1.GetOptions{}) + discovery, err := h.k8shelper.Clientset.AppsV1().DaemonSets(h.settings.OperatorNamespace).Get(ctx, "rook-discover", metav1.GetOptions{}) if startDiscovery { assert.NoError(h.T(), err) assert.NotNil(h.T(), discovery) @@ -504,17 +480,16 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st } // Create rook client - err = h.CreateRookToolbox(namespace) + err = h.CreateRookToolbox(h.Manifests) if err != nil { - logger.Errorf("Rook toolbox in cluster %s not installed, error -> %v", namespace, err) + logger.Errorf("Rook toolbox in cluster %s not installed. %v", h.settings.Namespace, err) return false, err } - logger.Infof("installed rook operator and cluster : %s on k8s %s", namespace, h.k8sVersion) + logger.Infof("installed rook operator and cluster %s on k8s %s", h.settings.Namespace, h.k8sVersion) - // disable admission controller test for Kubernetes version older than v1.16.0 and for upgrade test as api version v1 require minimum v0.7 controller runtime and upgrade test still using - // older version of controller runtime. - if !utils.IsPlatformOpenShift() && h.k8shelper.VersionAtLeast("v1.16.0") && namespace != "upgrade-ns" { - if !h.k8shelper.IsPodInExpectedState("rook-ceph-admission-controller", onamespace, "Running") { + // disable admission controller test for Kubernetes version older than v1.16.0 + if h.settings.EnableAdmissionController && !utils.IsPlatformOpenShift() && h.k8shelper.VersionAtLeast("v1.16.0") { + if !h.k8shelper.IsPodInExpectedState("rook-ceph-admission-controller", h.settings.OperatorNamespace, "Running") { assert.Fail(h.T(), "admission controller is not running") } } @@ -523,34 +498,41 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st } // UninstallRook uninstalls rook from k8s -func (h *CephInstaller) UninstallRook(namespace string) { - h.UninstallRookFromMultipleNS(SystemNamespace(namespace), namespace) +func (h *CephInstaller) UninstallRook() { + h.UninstallRookFromMultipleNS(h.Manifests) } // UninstallRookFromMultipleNS uninstalls rook from multiple namespaces in k8s -func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, namespaces ...string) { +func (h *CephInstaller) UninstallRookFromMultipleNS(manifests ...CephManifests) { ctx := context.TODO() + var clusterNamespaces []string + for _, manifest := range manifests { + clusterNamespaces = append(clusterNamespaces, manifest.Settings().Namespace) + } + // Gather logs after status checks - h.GatherAllRookLogs(h.T().Name(), append([]string{systemNamespace}, namespaces...)...) + h.GatherAllRookLogs(h.T().Name(), append([]string{h.settings.OperatorNamespace}, clusterNamespaces...)...) logger.Infof("Uninstalling Rook") var err error - for clusterNum, namespace := range namespaces { - if h.cleanupHost { - //Add cleanup policy to the ceph cluster - // Add cleanup policy to the ceph cluster but NOT the external one - if clusterNum == 0 { - err = h.addCleanupPolicy(namespace) - assert.NoError(h.T(), err) - } - } + for _, manifest := range manifests { + namespace := manifest.Settings().Namespace + clusterName := manifest.Settings().ClusterName - if !h.T().Failed() { - // Only check the Ceph status for the first cluster - // The second cluster is external so the check won't work since the first cluster is gone - if clusterNum == 0 { - // if the test passed, check that the ceph status is HEALTH_OK before we tear the cluster down - h.checkCephHealthStatus(namespace) + if !manifest.Settings().IsExternal { + // Add cleanup policy to the core ceph cluster + err = h.addCleanupPolicy(namespace) + if err != nil { + assert.NoError(h.T(), err) + // no need to check for cleanup policy later if it already failed + testCleanupPolicy = false + } + + // if the test passed, check that the ceph status is HEALTH_OK before we tear the cluster down + if !h.T().Failed() { + // Only check the Ceph status for the core cluster + // The check won't work for an external cluster since the core cluster is already gone + h.checkCephHealthStatus() } } @@ -561,155 +543,114 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, name logger.Infof("found pools: %v", pools) assert.Fail(h.T(), fmt.Sprintf("pool %q still exists", pool.Name)) // Get the operator log - h.GatherAllRookLogs(h.T().Name()+"poolcheck", systemNamespace) + h.GatherAllRookLogs(h.T().Name()+"poolcheck", h.settings.OperatorNamespace) } - err = h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cephcluster", h.clusterName) + err = h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cephcluster", clusterName) checkError(h.T(), err, fmt.Sprintf("cannot remove cluster %s", namespace)) - if h.cleanupHost { - // The second cluster is external so the cleanup pod will never exist! - if clusterNum == 0 { - err = h.waitForCleanupJobs(namespace) + clusterDeleteRetries := 0 + crdCheckerFunc := func() error { + _, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(ctx, clusterName, metav1.GetOptions{}) + clusterDeleteRetries++ + if clusterDeleteRetries > 10 { + // If the operator really isn't going to remove the finalizer, just force remove it + h.removeClusterFinalizers(namespace) + } + return err + } + err = h.k8shelper.WaitForCustomResourceDeletion(namespace, clusterName, crdCheckerFunc) + checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace)) + + if !manifest.Settings().IsExternal { + err = h.waitForCleanupJobs(namespace) + if err != nil { assert.NoError(h.T(), err) + h.GatherAllRookLogs(h.T().Name()+"cleanup-job", append([]string{h.settings.OperatorNamespace}, clusterNamespaces...)...) } } - roles := h.Manifests.GetClusterRoles(namespace, systemNamespace) - _, err = h.k8shelper.KubectlWithStdin(roles, deleteFromStdinArgs...) - if err != nil { - logger.Errorf("failed to delete cluster roles. %v ", err) - } - - crdCheckerFunc := func() error { - _, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(ctx, h.clusterName, metav1.GetOptions{}) - // ensure the finalizer(s) are removed - h.removeClusterFinalizers(namespace) - return err - } - err = h.k8shelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc) - checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace)) - - if h.useHelm { + // helm cleanup + if h.settings.UseHelm { err = h.helmHelper.DeleteLocalRookHelmChart(namespace, utils.HelmDeployName) checkError(h.T(), err, "cannot uninstall helm chart") + + // delete the entire namespace (in non-helm installs it's removed with the common.yaml) + err = h.k8shelper.DeleteResourceAndWait(false, "namespace", namespace) + checkError(h.T(), err, fmt.Sprintf("cannot delete namespace %s", namespace)) + continue } - // delete the entire namespace - err = h.k8shelper.DeleteResourceAndWait(false, "namespace", namespace) - checkError(h.T(), err, fmt.Sprintf("cannot delete namespace %s", namespace)) + // non-helm cleanup + if manifest.Settings().IsExternal { + logger.Infof("Deleting all the resources in the common external manifest") + _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetCommonExternal(), deleteFromStdinArgs...) + if err != nil { + logger.Warningf("failed to remove common external resources. %v", err) + } else { + logger.Infof("done deleting all the resources in the common external manifest") + } + } else { + h.k8shelper.PrintResources(namespace, "cephblockpools.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephclients.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephclusters.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephfilesystemmirrors.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephfilesystems.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephnfses.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephobjectrealms.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephobjectstores.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephobjectstoreusers.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephobjectzonegroups.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephobjectzones.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephrbdmirrors.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "objectbucketclaims.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "objectbuckets.ceph.rook.io") + h.k8shelper.PrintPodStatus(namespace) + h.k8shelper.PrintPVs(true) + logger.Infof("Deleting all the resources in the common manifest") + _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetCommon(), deleteFromStdinArgs...) + if err != nil { + logger.Warningf("failed to remove common manifest. %v", err) + } else { + logger.Infof("done deleting all the resources in the common manifest") + } + } } - err = h.k8shelper.DeleteResourceAndWait(false, "namespace", systemNamespace) - checkError(h.T(), err, fmt.Sprintf("cannot delete system namespace %s", systemNamespace)) - - logger.Infof("removing the operator from namespace %s", systemNamespace) - err = h.k8shelper.DeleteResource( - "crd", - "cephclusters.ceph.rook.io", - "cephblockpools.ceph.rook.io", - "cephobjectstores.ceph.rook.io", - "cephobjectstoreusers.ceph.rook.io", - "cephobjectrealms.ceph.rook.io", - "cephobjectzonegroups.ceph.rook.io", - "cephobjectzones.ceph.rook.io", - "cephfilesystems.ceph.rook.io", - "cephnfses.ceph.rook.io", - "cephclients.ceph.rook.io", - "volumes.rook.io", - "objectbuckets.objectbucket.io", - "objectbucketclaims.objectbucket.io", - "cephrbdmirrors.ceph.rook.io", - "cephfilesystemmirrors.ceph.rook.io") - checkError(h.T(), err, "cannot delete CRDs") - - if !h.useHelm { + if !h.settings.UseHelm { logger.Infof("Deleting all the resources in the operator manifest") - rookOperator := h.Manifests.GetRookOperator(systemNamespace) - _, err = h.k8shelper.KubectlWithStdin(rookOperator, deleteFromStdinArgs...) - logger.Infof("DONE deleting all the resources in the operator manifest") - checkError(h.T(), err, "cannot uninstall rook-operator") + _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetOperator(), deleteFromStdinArgs...) + if err != nil { + logger.Errorf("failed to remove operator resources. %v", err) + } else { + logger.Infof("done deleting all the resources in the operator manifest") + } } + logger.Info("removing the CRDs") + _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetCRDs(h.k8shelper), deleteFromStdinArgs...) + if err != nil { + logger.Errorf("failed to remove CRDS. %v", err) + } else { + logger.Infof("done deleting all the CRDs") + } + + err = h.k8shelper.DeleteResourceAndWait(false, "namespace", h.settings.OperatorNamespace) + checkError(h.T(), err, fmt.Sprintf("cannot delete operator namespace %s", h.settings.OperatorNamespace)) + err = h.k8shelper.Clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, "rook-ceph-webhook", metav1.DeleteOptions{}) checkError(h.T(), err, "failed to delete webhook configuration") - err = h.k8shelper.Clientset.RbacV1().RoleBindings(systemNamespace).Delete(ctx, "rook-ceph-system", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rook-ceph-global", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rook-ceph-mgr-cluster", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete role binding") - err = h.k8shelper.Clientset.CoreV1().ServiceAccounts(systemNamespace).Delete(ctx, "rook-ceph-system", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete service account") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "rook-ceph-cluster-mgmt", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "rook-ceph-mgr-cluster", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "rook-ceph-mgr-system", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "rook-ceph-global", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - err = h.k8shelper.Clientset.RbacV1().Roles(systemNamespace).Delete(ctx, "rook-ceph-system", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete role") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rbd-csi-nodeplugin", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "rbd-csi-nodeplugin", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rbd-csi-provisioner-role", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "rbd-external-provisioner-runner", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "cephfs-csi-nodeplugin", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "cephfs-csi-nodeplugin", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "cephfs-csi-provisioner-role", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "cephfs-external-provisioner-runner", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - - err = h.k8shelper.Clientset.PolicyV1beta1().PodSecurityPolicies().Delete(ctx, "rook-privileged", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete policy") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "psp:rook", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rook-ceph-system-psp", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rook-csi-rbd-provisioner-sa-psp", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rook-csi-rbd-plugin-sa-psp", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rook-csi-cephfs-provisioner-sa-psp", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rook-csi-cephfs-plugin-sa-psp", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - - err = h.k8shelper.Clientset.CoreV1().ConfigMaps(systemNamespace).Delete(ctx, "csi-rbd-config", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete config map") - err = h.k8shelper.Clientset.CoreV1().ConfigMaps(systemNamespace).Delete(ctx, "csi-cephfs-config", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete config map") - - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rook-ceph-object-bucket", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "rook-ceph-object-bucket", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - - logger.Infof("done removing the operator from namespace %s", systemNamespace) + logger.Infof("done removing the operator from namespace %s", h.settings.OperatorNamespace) logger.Infof("removing host data dir %s", h.hostPathToDelete) // removing data dir if exists if h.hostPathToDelete != "" { nodes, err := h.GetNodeHostnames() checkError(h.T(), err, "cannot get node names") for _, node := range nodes { - if h.cleanupHost { - err = h.verifyDirCleanup(node, h.hostPathToDelete) - logger.Infof("verifying clean up of %s from node %s. err=%v", h.hostPathToDelete, node, err) - assert.NoError(h.T(), err) - } else { - err = h.cleanupDir(node, h.hostPathToDelete) - logger.Infof("removing %s from node %s. err=%v", h.hostPathToDelete, node, err) - } + err = h.verifyDirCleanup(node, h.hostPathToDelete) + logger.Infof("verified cleanup of %s from node %s", h.hostPathToDelete, node) + assert.NoError(h.T(), err) } } if h.changeHostnames { @@ -720,12 +661,12 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, name // wait a bit longer for the system namespace to be cleaned up after their deletion for i := 0; i < 15; i++ { - _, err := h.k8shelper.Clientset.CoreV1().Namespaces().Get(ctx, systemNamespace, metav1.GetOptions{}) + _, err := h.k8shelper.Clientset.CoreV1().Namespaces().Get(ctx, h.settings.OperatorNamespace, metav1.GetOptions{}) if err != nil && kerrors.IsNotFound(err) { - logger.Infof("system namespace %q removed", systemNamespace) + logger.Infof("operator namespace %q removed", h.settings.OperatorNamespace) break } - logger.Infof("system namespace %q still found...", systemNamespace) + logger.Infof("operator namespace %q still found...", h.settings.OperatorNamespace) time.Sleep(5 * time.Second) } } @@ -733,7 +674,7 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, name func (h *CephInstaller) removeClusterFinalizers(namespace string) { ctx := context.TODO() // Get the latest cluster instead of using the same instance in case it has been changed - cluster, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(ctx, h.clusterName, metav1.GetOptions{}) + cluster, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(ctx, h.settings.ClusterName, metav1.GetOptions{}) if err != nil { logger.Errorf("failed to remove finalizer. failed to get cluster. %+v", err) return @@ -752,9 +693,9 @@ func (h *CephInstaller) removeClusterFinalizers(namespace string) { logger.Infof("removed finalizers from cluster %s", objectMeta.Name) } -func (h *CephInstaller) checkCephHealthStatus(namespace string) { +func (h *CephInstaller) checkCephHealthStatus() { ctx := context.TODO() - clusterResource, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(ctx, h.clusterName, metav1.GetOptions{}) + clusterResource, err := h.k8shelper.RookClientset.CephV1().CephClusters(h.settings.Namespace).Get(ctx, h.settings.ClusterName, metav1.GetOptions{}) assert.Nil(h.T(), err) clusterPhase := string(clusterResource.Status.Phase) if clusterPhase != "Ready" && clusterPhase != "Connected" && clusterPhase != "Progressing" { @@ -763,9 +704,10 @@ func (h *CephInstaller) checkCephHealthStatus(namespace string) { // Depending on the tests, the health may be fluctuating with different components being started or stopped. // If needed, give it a few seconds to settle and check the status again. + logger.Infof("checking ceph cluster health in namespace %q", h.settings.Namespace) if clusterResource.Status.CephStatus.Health != "HEALTH_OK" { time.Sleep(10 * time.Second) - clusterResource, err = h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(ctx, h.clusterName, metav1.GetOptions{}) + clusterResource, err = h.k8shelper.RookClientset.CephV1().CephClusters(h.settings.Namespace).Get(ctx, h.settings.ClusterName, metav1.GetOptions{}) assert.Nil(h.T(), err) } @@ -783,24 +725,18 @@ func (h *CephInstaller) checkCephHealthStatus(namespace string) { } } -func (h *CephInstaller) cleanupDir(node, dir string) error { - resources := h.GetCleanupPod(node, dir) - _, err := h.k8shelper.KubectlWithStdin(resources, createFromStdinArgs...) - return err -} - func (h *CephInstaller) verifyDirCleanup(node, dir string) error { resources := h.GetCleanupVerificationPod(node, dir) _, err := h.k8shelper.KubectlWithStdin(resources, createFromStdinArgs...) return err } -func (h *CephInstaller) CollectOperatorLog(suiteName, testName, namespace string) { +func (h *CephInstaller) CollectOperatorLog(suiteName, testName string) { if !h.T().Failed() && TestLogCollectionLevel() != "all" { return } name := fmt.Sprintf("%s_%s", suiteName, testName) - h.k8shelper.CollectPodLogsFromLabel(cephOperatorLabel, namespace, name, utils.TestEnvName()) + h.k8shelper.CollectPodLogsFromLabel(cephOperatorLabel, h.settings.OperatorNamespace, name, utils.TestEnvName()) } func (h *CephInstaller) GatherAllRookLogs(testName string, namespaces ...string) { @@ -816,18 +752,14 @@ func (h *CephInstaller) GatherAllRookLogs(testName string, namespaces ...string) } // NewCephInstaller creates new instance of CephInstaller -func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, useHelm bool, clusterName, rookVersion string, - cephVersion cephv1.CephVersionSpec, cleanupHost bool) *CephInstaller { +func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, settings *TestCephSettings) *CephInstaller { // By default set a cluster name that is different from the namespace so we don't rely on the namespace // in expected places - if clusterName == "" { - clusterName = defaultclusterName + if settings.ClusterName == "" { + settings.ClusterName = defaultclusterName } - // All e2e tests should run ceph commands in the toolbox since we are not inside a container - client.RunAllCephCommandsInToolbox = true - version, err := clientset.ServerVersion() if err != nil { logger.Infof("failed to get kubectl server version. %+v", err) @@ -837,17 +769,14 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, useH if err != nil { panic("failed to get kubectl client :" + err.Error()) } - logger.Infof("Rook Version: %s", rookVersion) - logger.Infof("Ceph Version: %s", cephVersion.Image) + logger.Infof("Rook Version: %s", settings.RookVersion) + logger.Infof("Ceph Version: %s", settings.CephVersion.Image) h := &CephInstaller{ - Manifests: NewCephManifests(rookVersion), + settings: settings, + Manifests: NewCephManifests(settings), k8shelper: k8shelp, helmHelper: utils.NewHelmHelper(testHelmPath()), - useHelm: useHelm, - clusterName: clusterName, k8sVersion: version.String(), - CephVersion: cephVersion, - cleanupHost: cleanupHost, changeHostnames: k8shelp.VersionAtLeast("v1.18.0"), T: t, } @@ -921,18 +850,26 @@ spec: } func (h *CephInstaller) addCleanupPolicy(namespace string) error { - ctx := context.TODO() - cluster, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(ctx, h.clusterName, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("failed to get ceph cluster. %+v", err) + // Retry updating the CR a few times in case of random failure + var returnErr error + for i := 0; i < 3; i++ { + ctx := context.TODO() + cluster, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(ctx, h.settings.ClusterName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get ceph cluster. %+v", err) + } + cluster.Spec.CleanupPolicy.Confirmation = cephv1.DeleteDataDirOnHostsConfirmation + cluster.Spec.CleanupPolicy.AllowUninstallWithVolumes = true + _, err = h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Update(ctx, cluster, metav1.UpdateOptions{}) + if err != nil { + returnErr = fmt.Errorf("failed to add clean up policy to the cluster. %+v", err) + logger.Warningf("could not add cleanup policy, trying again... %v", err) + } else { + logger.Info("successfully added cleanup policy to the ceph cluster and skipping checks for existing volumes") + return nil + } } - cluster.Spec.CleanupPolicy.Confirmation = cephv1.DeleteDataDirOnHostsConfirmation - _, err = h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Update(ctx, cluster, metav1.UpdateOptions{}) - if err != nil { - return fmt.Errorf("failed to add clean up policy to the cluster. %+v", err) - } - logger.Info("successfully added cleanup policy to the ceph cluster") - return nil + return returnErr } func (h *CephInstaller) waitForCleanupJobs(namespace string) error { diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index c0a1e781a..283072307 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -18,1292 +18,130 @@ package installer import ( "fmt" - "io/ioutil" - "path" "strconv" "strings" - "github.com/pkg/errors" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/tests/framework/utils" ) type CephManifests interface { - GetRookCRDs(v1ExtensionsSupported bool) string - GetRookOperator(namespace string) string - GetClusterRoles(namespace, operatorNamespace string) string - GetClusterExternalRoles(namespace, operatorNamespace string) string - GetRookCluster(settings *clusterSettings) string - GetRookExternalCluster(settings *clusterExternalSettings) string - GetRookToolBox(namespace string) string - GetBlockPoolDef(poolName, namespace, replicaSize string) string - GetBlockStorageClassDef(csi bool, poolName, storageClassName, reclaimPolicy, namespace, operatorNamespace string) string - GetFileStorageClassDef(fsName, storageClassName, operatorNamespace, namespace string) string - GetPVC(claimName, namespace, storageClassName, accessModes, size string) string - GetBlockSnapshotClass(snapshotClassName, namespace, operatorNamespace, reclaimPolicy string) string - GetFileStorageSnapshotClass(snapshotClassName, namespace, operatorNamespace, reclaimPolicy string) string - GetPVCRestore(claimName, snapshotName, namespace, storageClassName, accessModes, size string) string - GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, accessModes, size string) string - GetSnapshot(snapshotName, claimName, snapshotClassName, namespace string) string - GetPod(podName, claimName, namespace, mountPoint string, readOnly bool) string - GetFilesystem(namespace, name string, activeCount int) string - GetNFS(namespace, name, pool string, daemonCount int) string - GetRBDMirror(namespace, name string, daemonCount int) string - GetObjectStore(namespace, name string, replicaCount, port int) string - GetObjectStoreUser(namespace, name, displayName, store string) string - GetBucketStorageClass(namespace, storeName, storageClassName, reclaimPolicy, region string) string - GetObc(obcName, storageClassName, bucketName string, maxObject string, createBucket bool) string - GetClient(namespace, name string, caps map[string]string) string -} - -type clusterSettings struct { - ClusterName string - Namespace string - StoreType string - DataDirHostPath string - Mons int - MultipleMgrs bool - RBDMirrorWorkers int - UsePVCs bool - StorageClassName string - skipOSDCreation bool - CephVersion cephv1.CephVersionSpec - useCrashPruner bool -} - -// clusterExternalSettings represents the settings of an external cluster -type clusterExternalSettings struct { - Namespace string - DataDirHostPath string + Settings() *TestCephSettings + GetCRDs(k8shelper *utils.K8sHelper) string + GetOperator() string + GetCommon() string + GetCommonExternal() string + GetCephCluster() string + GetExternalCephCluster() string + GetToolbox() string + GetBlockPool(poolName, replicaSize string) string + GetBlockStorageClass(poolName, storageClassName, reclaimPolicy string) string + GetFileStorageClass(fsName, storageClassName string) string + GetBlockSnapshotClass(snapshotClassName, reclaimPolicy string) string + GetFileStorageSnapshotClass(snapshotClassName, reclaimPolicy string) string + GetFilesystem(name string, activeCount int) string + GetNFS(name, pool string, daemonCount int) string + GetRBDMirror(name string, daemonCount int) string + GetObjectStore(name string, replicaCount, port int) string + GetObjectStoreUser(name, displayName, store string) string + GetBucketStorageClass(storeName, storageClassName, reclaimPolicy, region string) string + GetOBC(obcName, storageClassName, bucketName string, maxObject string, createBucket bool) string + GetClient(name string, caps map[string]string) string } // CephManifestsMaster wraps rook yaml definitions type CephManifestsMaster struct { - imageTag string + settings *TestCephSettings } // NewCephManifests gets the manifest type depending on the Rook version desired -func NewCephManifests(version string) CephManifests { - switch version { +func NewCephManifests(settings *TestCephSettings) CephManifests { + switch settings.RookVersion { case VersionMaster: - return &CephManifestsMaster{imageTag: VersionMaster} + return &CephManifestsMaster{settings} case Version1_5: - return &CephManifestsV1_5{imageTag: Version1_5} + return &CephManifestsV1_5{settings} } - panic(fmt.Errorf("unrecognized ceph manifest version: %s", version)) + panic(fmt.Errorf("unrecognized ceph manifest version: %s", settings.RookVersion)) } -func readManifest(filename string) string { - rootDir, err := utils.FindRookRoot() - if err != nil { - panic(err) - } - manifest := path.Join(rootDir, "cluster/examples/kubernetes/ceph", filename) - logger.Infof("Reading manifest: %s", manifest) - contents, err := ioutil.ReadFile(manifest) - if err != nil { - panic(errors.Wrapf(err, "failed to read manifest at %s", manifest)) - } - return string(contents) +func (m *CephManifestsMaster) Settings() *TestCephSettings { + return m.settings } -func (m *CephManifestsMaster) GetRookCRDs(v1ExtensionsSupported bool) string { - if v1ExtensionsSupported { - return readManifest("crds.yaml") +func (m *CephManifestsMaster) GetCRDs(k8shelper *utils.K8sHelper) string { + if k8shelper.VersionAtLeast("v1.16.0") { + return m.settings.readManifest("crds.yaml") } - return readManifest("pre-k8s-1.16/crds.yaml") + return m.settings.readManifest("pre-k8s-1.16/crds.yaml") } -func getOpenshiftSCC(namespace string) string { - manifest := readManifest("scc.yaml") - return strings.ReplaceAll(manifest, "system:serviceaccount:rook-ceph:", fmt.Sprintf("system:serviceaccount:%s:", namespace)) -} - -// GetRookOperator returns rook Operator manifest -func (m *CephManifestsMaster) GetRookOperator(operatorNamespace string) string { - var operatorManifest string - openshiftEnv := "" - +func (m *CephManifestsMaster) GetOperator() string { + var manifest string if utils.IsPlatformOpenShift() { - openshiftEnv = ` - - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED - value: "true" - - name: FLEXVOLUME_DIR_PATH - value: "/etc/kubernetes/kubelet-plugins/volume/exec"` - operatorManifest = getOpenshiftSCC(operatorNamespace) + manifest = m.settings.readManifest("operator-openshift.yaml") + } else { + manifest = m.settings.readManifest("operator.yaml") } - - operatorManifest = operatorManifest + `--- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: rook-ceph-system - namespace: ` + operatorNamespace + ` - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - - apps - - extensions - resources: - - pods - - configmaps - - services - - daemonsets - - statefulsets - - deployments - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - batch - resources: - - cronjobs - verbs: - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-cluster-mgmt - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - - secrets - - pods - - pods/log - - services - - configmaps - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - apps - - extensions - resources: - - deployments - - daemonsets - verbs: - - get - - list - - watch - - create - - update - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-global - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - # Pod access is needed for fencing - - pods - # Node access is needed for determining nodes where mons should run - - nodes - - nodes/proxy - - services - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - # PVs and PVCs are managed by the Rook provisioner - - persistentvolumes - - persistentvolumeclaims - - endpoints - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - batch - resources: - - jobs - - cronjobs - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - policy - - apps - - extensions - resources: - # This is for the clusterdisruption controller - - poddisruptionbudgets - # This is for both clusterdisruption and nodedrain controllers - - deployments - - replicasets - verbs: - - "*" -- apiGroups: - - healthchecking.openshift.io - resources: - - machinedisruptionbudgets - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - machine.openshift.io - resources: - - machines - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - csidrivers - verbs: - - create - - delete - - get - - update -- apiGroups: - - k8s.cni.cncf.io - resources: - - network-attachment-definitions - verbs: - - get ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-cluster -rules: -- apiGroups: - - "" - resources: - - configmaps - - nodes - - nodes/proxy - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - list - - get - - watch ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-osd -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list ---- -# Aspects of Rook Ceph Agent that require access to secrets -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-agent-mount - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get ---- -# Aspects of ceph-mgr that require access to the system namespace -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-system -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-object-bucket - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - verbs: - - "*" - resources: - - secrets - - configmaps -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - "objectbucket.io" - verbs: - - "*" - resources: - - "*" ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-object-bucket -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-object-bucket -subjects: - - kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-system - namespace: ` + operatorNamespace + ` - labels: - operator: rook - storage-backend: ceph ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-system - namespace: ` + operatorNamespace + ` - labels: - operator: rook - storage-backend: ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-system -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-global - labels: - operator: rook - storage-backend: ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-global -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-rbd-plugin-sa - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-plugin-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: rbd-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-rbd-provisioner-sa - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-provisioner-runner -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-provisioner-role -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: rbd-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - namespace: ` + operatorNamespace + ` - name: rbd-external-provisioner-cfg -rules: - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "delete", "update"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-provisioner-role-cfg - namespace: ` + operatorNamespace + ` -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: Role - name: rbd-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-cephfs-plugin-sa - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-plugin-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: cephfs-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-cephfs-provisioner-sa - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-provisioner-runner -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-provisioner-role -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: cephfs-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - namespace: ` + operatorNamespace + ` - name: cephfs-external-provisioner-cfg -rules: - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "create", "delete"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-provisioner-role-cfg - namespace: ` + operatorNamespace + ` -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: Role - name: cephfs-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: 00-rook-privileged - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' - seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' -spec: - privileged: true - allowedCapabilities: - # required by CSI - - SYS_ADMIN - # fsGroup - the flexVolume agent has fsGroup capabilities and could potentially be any group - fsGroup: - rule: RunAsAny - # runAsUser, supplementalGroups - Rook needs to run some pods as root - # Ceph pods could be run as the Ceph user, but that user isn't always known ahead of time - runAsUser: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - # seLinux - seLinux context is unknown ahead of time; set if this is well-known - seLinux: - rule: RunAsAny - volumes: - # recommended minimum set - - configMap - - downwardAPI - - emptyDir - - persistentVolumeClaim - - secret - - projected - # required for Rook - - hostPath - - flexVolume - # allowedHostPaths can be set to Rook's known host volume mount points when they are fully-known - # allowedHostPaths: - # - /run/udev # for OSD prep - # - /dev # for OSD prep - # - /var/lib/rook # or whatever the dataDirHostPath value is set to - # Ceph requires host IPC for setting up encrypted devices - hostIPC: true - # Ceph OSDs need to share the same PID namespace - hostPID: true - # hostNetwork can be set to 'false' if host networking isn't used - hostNetwork: true - hostPorts: - # Ceph messenger protocol v1 - - min: 6789 - max: 6790 # <- support old default port - # Ceph messenger protocol v2 - - min: 3300 - max: 3300 - # Ceph RADOS ports for OSDs, MDSes - - min: 6800 - max: 7300 - # # Ceph dashboard port HTTP (not recommended) - # - min: 7000 - # max: 7000 - # Ceph dashboard port HTTPS - - min: 8443 - max: 8443 - # Ceph mgr Prometheus Metrics - - min: 9283 - max: 9283 ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: 'psp:rook' -rules: - - apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - 00-rook-privileged - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-ceph-system-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-rbd-provisioner-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: ` + operatorNamespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-rbd-plugin-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-plugin-sa - namespace: ` + operatorNamespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-cephfs-provisioner-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: ` + operatorNamespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-cephfs-plugin-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-plugin-sa - namespace: ` + operatorNamespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-admission-controller - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-admission-controller-role -rules: - - apiGroups: ["ceph.rook.io"] - resources: ["*"] - verbs: ["get", "watch", "list"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-admission-controller-rolebinding -subjects: - - kind: ServiceAccount - name: rook-ceph-admission-controller - apiGroup: "" - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: rook-ceph-admission-controller-role - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: rook-ceph-operator - namespace: ` + operatorNamespace + ` - labels: - operator: rook - storage-backend: ceph -spec: - replicas: 1 - selector: - matchLabels: - app: rook-ceph-operator - template: - metadata: - labels: - app: rook-ceph-operator - spec: - serviceAccountName: rook-ceph-system - containers: - - name: rook-ceph-operator - image: rook/ceph:` + m.imageTag + ` - args: ["ceph", "operator"] - env: - - name: ROOK_LOG_LEVEL - value: INFO - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ROOK_ENABLE_FLEX_DRIVER - value: "true" - - name: ROOK_CURRENT_NAMESPACE_ONLY - value: "false"` + openshiftEnv + ` - - name: ROOK_ENABLE_DISCOVERY_DAEMON - value: "true" - volumeMounts: - - mountPath: /var/lib/rook - name: rook-config - - mountPath: /etc/ceph - name: default-config-dir - volumes: - - name: rook-config - emptyDir: {} - - name: default-config-dir - emptyDir: {} ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: rook-ceph-operator-config - namespace: ` + operatorNamespace + ` -data: - ROOK_CSI_ENABLE_CEPHFS: "true" - ROOK_CSI_ENABLE_RBD: "true" - ROOK_CSI_ENABLE_GRPC_METRICS: "false" - ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true" - CSI_LOG_LEVEL: "5" -` - return operatorManifest + return m.settings.replaceOperatorSettings(manifest) } -// GetClusterRoles returns rook-cluster manifest -func (m *CephManifestsMaster) GetClusterRoles(namespace, operatorNamespace string) string { - return `apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-osd - namespace: ` + namespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-osd - namespace: ` + namespace + ` -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: [ "get", "list", "watch", "create", "update", "delete" ] -- apiGroups: ["ceph.rook.io"] - resources: ["cephclusters", "cephclusters/finalizers"] - verbs: [ "get", "list", "create", "update", "delete" ] ---- -# Aspects of ceph-mgr that operate within the cluster's namespace -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr - namespace: ` + namespace + ` -rules: -- apiGroups: - - "" - resources: - - pods - - services - - pods/log - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" ---- -# Allow the ceph osd to access cluster-wide resources necessary for determining their topology location -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-osd-` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-osd -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: ` + namespace + ` ---- -# Allow the ceph mgr to access cluster-wide resources necessary for the mgr modules -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-cluster-` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-mgr-cluster -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -rules: -- apiGroups: - - "" - resources: - - pods - - configmaps - verbs: - - get - - list - - watch - - create - - update - - delete ---- -# Allow the operator to create resources in this cluster's namespace -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cluster-mgmt - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-cluster-mgmt -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -# Allow the osd pods in this namespace to work with configmaps -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-osd - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-osd -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: ` + namespace + ` ---- -# Allow the ceph mgr to access the cluster-specific resources necessary for the mgr modules -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-mgr -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -# Allow the ceph mgr to access the rook system resources necessary for the mgr modules -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-system ` + namespace + ` - namespace: ` + operatorNamespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-mgr-system -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-cmd-reporter -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-default-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: default - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-osd-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-mgr-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-cmd-reporter-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -` +func (m *CephManifestsMaster) GetCommonExternal() string { + return m.settings.readManifest("common-external.yaml") } -// GetRookCluster returns rook-cluster manifest -func (m *CephManifestsMaster) GetRookCluster(settings *clusterSettings) string { - store := "# storeType not specified; Rook will use default store types" - if settings.StoreType != "" { - store = `storeType: "` + settings.StoreType + `"` - } +func (m *CephManifestsMaster) GetCommon() string { + return m.settings.readManifest("common.yaml") +} +func (m *CephManifestsMaster) GetToolbox() string { + manifest := strings.ReplaceAll(m.settings.readManifest("direct-mount.yaml"), "name: rook-direct-mount", "name: rook-ceph-tools") + return strings.ReplaceAll(manifest, "app: rook-direct-mount", "app: rook-ceph-tools") +} + +//********************************************************************************** +//********************************************************************************** +// After a release, copy the methods below this separator into the versioned file +// such as ceph_manifests_v1.6.go. Methods above this separator do not need to be +// copied since the versioned implementation will load them directly from github. +//********************************************************************************** +//********************************************************************************** + +func (m *CephManifestsMaster) GetCephCluster() string { crushRoot := "# crushRoot not specified; Rook will use `default`" - if settings.Mons == 1 { + if m.settings.Mons == 1 { crushRoot = `crushRoot: "custom-root"` } pruner := "# daysToRetain not specified;" - if settings.useCrashPruner { + if m.settings.UseCrashPruner { pruner = "daysToRetain: 5" } mgrCount := 1 - if settings.MultipleMgrs { + if m.settings.MultipleMgrs { mgrCount = 2 } - if settings.UsePVCs { + if m.settings.UsePVC { return `apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: # set the name to something different from the namespace - name: ` + settings.ClusterName + ` - namespace: ` + settings.Namespace + ` + name: ` + m.settings.ClusterName + ` + namespace: ` + m.settings.Namespace + ` spec: - dataDirHostPath: ` + settings.DataDirHostPath + ` + dataDirHostPath: ` + m.settings.DataDirHostPath + ` mon: - count: ` + strconv.Itoa(settings.Mons) + ` + count: ` + strconv.Itoa(m.settings.Mons) + ` allowMultiplePerNode: true volumeClaimTemplate: spec: - storageClassName: ` + settings.StorageClassName + ` + storageClassName: ` + m.settings.StorageClassName + ` resources: requests: storage: 5Gi cephVersion: - image: ` + settings.CephVersion.Image + ` - allowUnsupported: ` + strconv.FormatBool(settings.CephVersion.AllowUnsupported) + ` + image: ` + m.settings.CephVersion.Image + ` + allowUnsupported: ` + strconv.FormatBool(m.settings.CephVersion.AllowUnsupported) + ` skipUpgradeChecks: false continueUpgradeAfterChecksEvenIfNotHealthy: false mgr: @@ -1332,7 +170,7 @@ spec: resources: requests: storage: 10Gi - storageClassName: ` + settings.StorageClassName + ` + storageClassName: ` + m.settings.StorageClassName + ` volumeMode: Block accessModes: - ReadWriteOnce @@ -1347,31 +185,30 @@ spec: return `apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: - name: ` + settings.ClusterName + ` - namespace: ` + settings.Namespace + ` + name: ` + m.settings.ClusterName + ` + namespace: ` + m.settings.Namespace + ` spec: cephVersion: - image: ` + settings.CephVersion.Image + ` - allowUnsupported: ` + strconv.FormatBool(settings.CephVersion.AllowUnsupported) + ` - dataDirHostPath: ` + settings.DataDirHostPath + ` + image: ` + m.settings.CephVersion.Image + ` + allowUnsupported: ` + strconv.FormatBool(m.settings.CephVersion.AllowUnsupported) + ` + dataDirHostPath: ` + m.settings.DataDirHostPath + ` network: hostNetwork: false crashCollector: disable: false ` + pruner + ` mon: - count: ` + strconv.Itoa(settings.Mons) + ` + count: ` + strconv.Itoa(m.settings.Mons) + ` allowMultiplePerNode: true dashboard: enabled: true skipUpgradeChecks: true metadataDevice: storage: - useAllNodes: ` + strconv.FormatBool(!settings.skipOSDCreation) + ` - useAllDevices: ` + strconv.FormatBool(!settings.skipOSDCreation) + ` + useAllNodes: ` + strconv.FormatBool(!m.settings.SkipOSDCreation) + ` + useAllDevices: ` + strconv.FormatBool(!m.settings.SkipOSDCreation) + ` deviceFilter: ` + getDeviceFilter() + ` config: - ` + store + ` databaseSizeMB: "1024" journalSizeMB: "1024" mgr: @@ -1391,174 +228,44 @@ spec: interval: 5s` } -// GetRookToolBox returns rook-toolbox manifest -func (m *CephManifestsMaster) GetRookToolBox(namespace string) string { - return `apiVersion: v1 -kind: Pod -metadata: - name: rook-ceph-tools - namespace: ` + namespace + ` -spec: - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: rook-ceph-tools - image: rook/ceph:` + m.imageTag + ` - imagePullPolicy: IfNotPresent - command: ["/tini"] - args: ["-g", "--", "/usr/local/bin/toolbox.sh"] - env: - - name: ROOK_CEPH_USERNAME - valueFrom: - secretKeyRef: - name: rook-ceph-mon - key: ceph-username - - name: ROOK_CEPH_SECRET - valueFrom: - secretKeyRef: - name: rook-ceph-mon - key: ceph-secret - securityContext: - privileged: true - volumeMounts: - - mountPath: /dev - name: dev - - mountPath: /sys/bus - name: sysbus - - mountPath: /lib/modules - name: libmodules - - name: mon-endpoint-volume - mountPath: /etc/rook - hostNetwork: false - volumes: - - name: dev - hostPath: - path: /dev - - name: sysbus - hostPath: - path: /sys/bus - - name: libmodules - hostPath: - path: /lib/modules - - name: mon-endpoint-volume - configMap: - name: rook-ceph-mon-endpoints - items: - - key: data - path: mon-endpoints` -} - -func (m *CephManifestsMaster) GetBlockSnapshotClass(snapshotClassName, namespace, operatorNamespace, reclaimPolicy string) string { +func (m *CephManifestsMaster) GetBlockSnapshotClass(snapshotClassName, reclaimPolicy string) string { // Create a CSI driver snapshotclass return ` apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshotClass metadata: name: ` + snapshotClassName + ` -driver: ` + operatorNamespace + `.rbd.csi.ceph.com +driver: ` + m.settings.OperatorNamespace + `.rbd.csi.ceph.com deletionPolicy: ` + reclaimPolicy + ` parameters: - clusterID: ` + namespace + ` + clusterID: ` + m.settings.Namespace + ` csi.storage.k8s.io/snapshotter-secret-name: rook-csi-rbd-provisioner - csi.storage.k8s.io/snapshotter-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/snapshotter-secret-namespace: ` + m.settings.Namespace + ` ` } -func (m *CephManifestsMaster) GetFileStorageSnapshotClass(snapshotClassName, namespace, operatorNamespace, reclaimPolicy string) string { +func (m *CephManifestsMaster) GetFileStorageSnapshotClass(snapshotClassName, reclaimPolicy string) string { // Create a CSI driver snapshotclass return ` apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshotClass metadata: name: ` + snapshotClassName + ` -driver: ` + operatorNamespace + `.cephfs.csi.ceph.com +driver: ` + m.settings.OperatorNamespace + `.cephfs.csi.ceph.com deletionPolicy: ` + reclaimPolicy + ` parameters: - clusterID: ` + namespace + ` + clusterID: ` + m.settings.Namespace + ` csi.storage.k8s.io/snapshotter-secret-name: rook-csi-cephfs-provisioner - csi.storage.k8s.io/snapshotter-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/snapshotter-secret-namespace: ` + m.settings.Namespace + ` ` } -func (m *CephManifestsMaster) GetPVCRestore(claimName, snapshotName, namespace, storageClassName, accessModes, size string) string { - return `apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ` + claimName + ` - namespace: ` + namespace + ` -spec: - storageClassName: ` + storageClassName + ` - dataSource: - name: ` + snapshotName + ` - kind: VolumeSnapshot - apiGroup: snapshot.storage.k8s.io - accessModes: - - ` + accessModes + ` - resources: - requests: - storage: ` + size -} - -func (m *CephManifestsMaster) GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, accessModes, size string) string { - return `apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ` + cloneClaimName + ` - namespace: ` + namespace + ` -spec: - storageClassName: ` + storageClassName + ` - dataSource: - name: ` + parentClaimName + ` - kind: PersistentVolumeClaim - accessModes: - - ` + accessModes + ` - resources: - requests: - storage: ` + size -} - -func (m *CephManifestsMaster) GetSnapshot(snapshotName, claimName, snapshotClassName, namespace string) string { - return `apiVersion: snapshot.storage.k8s.io/v1beta1 -kind: VolumeSnapshot -metadata: - name: ` + snapshotName + ` - namespace: ` + namespace + ` -spec: - volumeSnapshotClassName: ` + snapshotClassName + ` - source: - persistentVolumeClaimName: ` + claimName -} - -func (m *CephManifestsMaster) GetPod(podName, claimName, namespace, mountPath string, readOnly bool) string { - return ` -apiVersion: v1 -kind: Pod -metadata: - name: ` + podName + ` - namespace: ` + namespace + ` -spec: - containers: - - name: ` + podName + ` - image: busybox - command: ["/bin/sleep", "infinity"] - imagePullPolicy: IfNotPresent - volumeMounts: - - mountPath: ` + mountPath + ` - name: csivol - volumes: - - name: csivol - persistentVolumeClaim: - claimName: ` + claimName + ` - readOnly: ` + strconv.FormatBool(readOnly) + ` - restartPolicy: Never -` -} - -func (m *CephManifestsMaster) GetBlockPoolDef(poolName string, namespace string, replicaSize string) string { +func (m *CephManifestsMaster) GetBlockPool(poolName string, replicaSize string) string { return `apiVersion: ceph.rook.io/v1 kind: CephBlockPool metadata: name: ` + poolName + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: replicated: size: ` + replicaSize + ` @@ -1577,23 +284,23 @@ spec: interval: 10s` } -func (m *CephManifestsMaster) GetBlockStorageClassDef(csi bool, poolName, storageClassName, reclaimPolicy, namespace, operatorNamespace string) string { +func (m *CephManifestsMaster) GetBlockStorageClass(poolName, storageClassName, reclaimPolicy string) string { // Create a CSI driver storage class - if csi { + if m.settings.UseCSI { return ` apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ` + storageClassName + ` -provisioner: ` + operatorNamespace + `.rbd.csi.ceph.com +provisioner: ` + m.settings.OperatorNamespace + `.rbd.csi.ceph.com reclaimPolicy: ` + reclaimPolicy + ` parameters: pool: ` + poolName + ` - clusterID: ` + namespace + ` + clusterID: ` + m.settings.Namespace + ` csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner - csi.storage.k8s.io/provisioner-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/provisioner-secret-namespace: ` + m.settings.Namespace + ` csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node - csi.storage.k8s.io/node-stage-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/node-stage-secret-namespace: ` + m.settings.Namespace + ` imageFeatures: layering csi.storage.k8s.io/fstype: ext4 ` @@ -1608,10 +315,10 @@ allowVolumeExpansion: true reclaimPolicy: ` + reclaimPolicy + ` parameters: blockPool: ` + poolName + ` - clusterNamespace: ` + namespace + clusterNamespace: ` + m.settings.Namespace } -func (m *CephManifestsMaster) GetFileStorageClassDef(fsName, storageClassName, operatorNamespace, namespace string) string { +func (m *CephManifestsMaster) GetFileStorageClass(fsName, storageClassName string) string { // Create a CSI driver storage class csiCephFSNodeSecret := "rook-csi-cephfs-node" //nolint:gosec // We safely suppress gosec in tests file csiCephFSProvisionerSecret := "rook-csi-cephfs-provisioner" //nolint:gosec // We safely suppress gosec in tests file @@ -1620,40 +327,25 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ` + storageClassName + ` -provisioner: ` + operatorNamespace + `.cephfs.csi.ceph.com +provisioner: ` + m.settings.OperatorNamespace + `.cephfs.csi.ceph.com parameters: - clusterID: ` + namespace + ` + clusterID: ` + m.settings.Namespace + ` fsName: ` + fsName + ` pool: ` + fsName + `-data0 csi.storage.k8s.io/provisioner-secret-name: ` + csiCephFSProvisionerSecret + ` - csi.storage.k8s.io/provisioner-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/provisioner-secret-namespace: ` + m.settings.Namespace + ` csi.storage.k8s.io/node-stage-secret-name: ` + csiCephFSNodeSecret + ` - csi.storage.k8s.io/node-stage-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/node-stage-secret-namespace: ` + m.settings.Namespace + ` ` } -func (m *CephManifestsMaster) GetPVC(claimName, namespace, storageClassName, accessModes, size string) string { - return `apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ` + claimName + ` - namespace: ` + namespace + ` -spec: - storageClassName: ` + storageClassName + ` - accessModes: - - ` + accessModes + ` - resources: - requests: - storage: ` + size -} - // GetFilesystem returns the manifest to create a Rook filesystem resource with the given config. -func (m *CephManifestsMaster) GetFilesystem(namespace, name string, activeCount int) string { +func (m *CephManifestsMaster) GetFilesystem(name string, activeCount int) string { return `apiVersion: ceph.rook.io/v1 kind: CephFilesystem metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: metadataPool: replicated: @@ -1670,12 +362,12 @@ spec: } // GetFilesystem returns the manifest to create a Rook Ceph NFS resource with the given config. -func (m *CephManifestsMaster) GetNFS(namespace, name, pool string, count int) string { +func (m *CephManifestsMaster) GetNFS(name, pool string, count int) string { return `apiVersion: ceph.rook.io/v1 kind: CephNFS metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: rados: pool: ` + pool + ` @@ -1684,12 +376,12 @@ spec: active: ` + strconv.Itoa(count) } -func (m *CephManifestsMaster) GetObjectStore(namespace, name string, replicaCount, port int) string { +func (m *CephManifestsMaster) GetObjectStore(name string, replicaCount, port int) string { return `apiVersion: ceph.rook.io/v1 kind: CephObjectStore metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: metadataPool: replicated: @@ -1711,33 +403,33 @@ spec: ` } -func (m *CephManifestsMaster) GetObjectStoreUser(namespace, name string, displayName string, store string) string { +func (m *CephManifestsMaster) GetObjectStoreUser(name string, displayName string, store string) string { return `apiVersion: ceph.rook.io/v1 kind: CephObjectStoreUser metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: displayName: ` + displayName + ` store: ` + store } //GetBucketStorageClass returns the manifest to create object bucket -func (m *CephManifestsMaster) GetBucketStorageClass(storeNameSpace string, storeName string, storageClassName string, reclaimPolicy string, region string) string { +func (m *CephManifestsMaster) GetBucketStorageClass(storeName, storageClassName, reclaimPolicy, region string) string { return `apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ` + storageClassName + ` -provisioner: ` + storeNameSpace + `.ceph.rook.io/bucket +provisioner: ` + m.settings.Namespace + `.ceph.rook.io/bucket reclaimPolicy: ` + reclaimPolicy + ` parameters: objectStoreName: ` + storeName + ` - objectStoreNamespace: ` + storeNameSpace + ` + objectStoreNamespace: ` + m.settings.Namespace + ` region: ` + region } -//GetObc returns the manifest to create object bucket claim -func (m *CephManifestsMaster) GetObc(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string { +//GetOBC returns the manifest to create object bucket claim +func (m *CephManifestsMaster) GetOBC(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string { bucketParameter := "generateBucketName" if varBucketName { bucketParameter = "bucketName" @@ -1753,7 +445,7 @@ spec: maxObjects: "` + maxObject + `"` } -func (m *CephManifestsMaster) GetClient(namespace string, claimName string, caps map[string]string) string { +func (m *CephManifestsMaster) GetClient(claimName string, caps map[string]string) string { clientCaps := []string{} for name, cap := range caps { str := name + ": " + cap @@ -1763,87 +455,31 @@ func (m *CephManifestsMaster) GetClient(namespace string, claimName string, caps kind: CephClient metadata: name: ` + claimName + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: caps: ` + strings.Join(clientCaps, "\n ") } -func (m *CephManifestsMaster) GetClusterExternalRoles(namespace, firstClusterNamespace string) string { - return `apiVersion: v1 -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cluster-mgmt - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-cluster-mgmt -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: ` + firstClusterNamespace + ` ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-cmd-reporter -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -rules: -- apiGroups: - - "" - resources: - - pods - - configmaps - verbs: - - get - - list - - watch - - create - - update - - delete` -} - -func (m *CephManifestsMaster) GetRookExternalCluster(settings *clusterExternalSettings) string { +func (m *CephManifestsMaster) GetExternalCephCluster() string { return `apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: - name: ` + settings.Namespace + ` - namespace: ` + settings.Namespace + ` + name: ` + m.settings.ClusterName + ` + namespace: ` + m.settings.Namespace + ` spec: external: enable: true - dataDirHostPath: ` + settings.DataDirHostPath + `` + dataDirHostPath: ` + m.settings.DataDirHostPath + `` } // GetRBDMirror returns the manifest to create a Rook Ceph RBD Mirror resource with the given config. -func (m *CephManifestsMaster) GetRBDMirror(namespace, name string, count int) string { +func (m *CephManifestsMaster) GetRBDMirror(name string, count int) string { return `apiVersion: ceph.rook.io/v1 kind: CephRBDMirror metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: count: ` + strconv.Itoa(count) } diff --git a/tests/framework/installer/ceph_manifests_v1.5.go b/tests/framework/installer/ceph_manifests_v1.5.go index 848009981..5376b0233 100644 --- a/tests/framework/installer/ceph_manifests_v1.5.go +++ b/tests/framework/installer/ceph_manifests_v1.5.go @@ -17,13 +17,9 @@ limitations under the License. package installer import ( - "fmt" - "io/ioutil" - "net/http" "strconv" "strings" - "github.com/pkg/errors" "github.com/rook/rook/tests/framework/utils" ) @@ -34,1201 +30,82 @@ const ( // CephManifestsV1_4 wraps rook yaml definitions type CephManifestsV1_5 struct { - imageTag string + settings *TestCephSettings } -func readManifestFromGithub(filename string) string { - url := fmt.Sprintf("https://raw.githubusercontent.com/rook/rook/%s/cluster/examples/kubernetes/ceph/%s", Version1_5, filename) - logger.Infof("Retrieving manifest: %s", url) - // #nosec G107 This is only test code and is expected to read from a url - response, err := http.Get(url) - - if err != nil { - panic(errors.Wrapf(err, "failed to read manifest from %s", url)) - } - defer response.Body.Close() - - content, err := ioutil.ReadAll(response.Body) - if err != nil { - panic(errors.Wrapf(err, "failed to read content from %s", url)) - } - return string(content) +func (m *CephManifestsV1_5) Settings() *TestCephSettings { + return m.settings } -func (m *CephManifestsV1_5) GetRookCRDs(v1ExtensionsSupported bool) string { - if v1ExtensionsSupported { - return readManifestFromGithub("crds.yaml") +func (m *CephManifestsV1_5) GetCRDs(k8shelper *utils.K8sHelper) string { + if k8shelper.VersionAtLeast("v1.16.0") { + return m.settings.readManifestFromGithub("crds.yaml") } - return readManifestFromGithub("pre-k8s-1.16/crds.yaml") + return m.settings.readManifestFromGithub("pre-k8s-1.16/crds.yaml") } // GetRookOperator returns rook Operator manifest -func (m *CephManifestsV1_5) GetRookOperator(operatorNamespace string) string { - var operatorManifest string - openshiftEnv := "" - +func (m *CephManifestsV1_5) GetOperator() string { + var manifest string if utils.IsPlatformOpenShift() { - openshiftEnv = ` - - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED - value: "true" - - name: FLEXVOLUME_DIR_PATH - value: "/etc/kubernetes/kubelet-plugins/volume/exec"` - operatorManifest = getOpenshiftSCC(operatorNamespace) + manifest = m.settings.readManifestFromGithub("operator-openshift.yaml") + } else { + manifest = m.settings.readManifestFromGithub("operator.yaml") } - - operatorManifest = operatorManifest + `--- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: rook-ceph-system - namespace: ` + operatorNamespace + ` - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - - apps - - extensions - resources: - - pods - - configmaps - - services - - daemonsets - - statefulsets - - deployments - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - k8s.cni.cncf.io - resources: - - network-attachment-definitions - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-cluster-mgmt - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - - secrets - - pods - - pods/log - - services - - configmaps - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - apps - - extensions - resources: - - deployments - - daemonsets - verbs: - - get - - list - - watch - - create - - update - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-global - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - # Pod access is needed for fencing - - pods - # Node access is needed for determining nodes where mons should run - - nodes - - nodes/proxy - - services - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - # PVs and PVCs are managed by the Rook provisioner - - persistentvolumes - - persistentvolumeclaims - - endpoints - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - policy - - apps - - extensions - resources: - # This is for the clusterdisruption controller - - poddisruptionbudgets - # This is for both clusterdisruption and nodedrain controllers - - deployments - - replicasets - verbs: - - "*" -- apiGroups: - - healthchecking.openshift.io - resources: - - machinedisruptionbudgets - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - machine.openshift.io - resources: - - machines - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - csidrivers - verbs: - - create - - delete - - get - - update ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-cluster -rules: -- apiGroups: - - "" - resources: - - configmaps - - nodes - - nodes/proxy - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - list - - get - - watch ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-osd -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list ---- -# Aspects of Rook Ceph Agent that require access to secrets -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-agent-mount - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get ---- -# Aspects of ceph-mgr that require access to the system namespace -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-system -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-object-bucket - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - verbs: - - "*" - resources: - - secrets - - configmaps -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - "objectbucket.io" - verbs: - - "*" - resources: - - "*" ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-object-bucket -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-object-bucket -subjects: - - kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-system - namespace: ` + operatorNamespace + ` - labels: - operator: rook - storage-backend: ceph ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-system - namespace: ` + operatorNamespace + ` - labels: - operator: rook - storage-backend: ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-system -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-global - labels: - operator: rook - storage-backend: ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-global -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-rbd-plugin-sa - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-plugin-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: rbd-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-rbd-provisioner-sa - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-provisioner-runner -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-provisioner-role -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: rbd-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - namespace: ` + operatorNamespace + ` - name: rbd-external-provisioner-cfg -rules: - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "delete", "update"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-provisioner-role-cfg - namespace: ` + operatorNamespace + ` -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: Role - name: rbd-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-cephfs-plugin-sa - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-plugin-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: cephfs-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-cephfs-provisioner-sa - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-provisioner-runner -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-provisioner-role -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: cephfs-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - namespace: ` + operatorNamespace + ` - name: cephfs-external-provisioner-cfg -rules: - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "create", "delete"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-provisioner-role-cfg - namespace: ` + operatorNamespace + ` -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: Role - name: cephfs-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: 00-rook-privileged - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' - seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' -spec: - privileged: true - allowedCapabilities: - # required by CSI - - SYS_ADMIN - # fsGroup - the flexVolume agent has fsGroup capabilities and could potentially be any group - fsGroup: - rule: RunAsAny - # runAsUser, supplementalGroups - Rook needs to run some pods as root - # Ceph pods could be run as the Ceph user, but that user isn't always known ahead of time - runAsUser: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - # seLinux - seLinux context is unknown ahead of time; set if this is well-known - seLinux: - rule: RunAsAny - volumes: - # recommended minimum set - - configMap - - downwardAPI - - emptyDir - - persistentVolumeClaim - - secret - - projected - # required for Rook - - hostPath - - flexVolume - # allowedHostPaths can be set to Rook's known host volume mount points when they are fully-known - # allowedHostPaths: - # - /run/udev # for OSD prep - # - /dev # for OSD prep - # - /var/lib/rook # or whatever the dataDirHostPath value is set to - # Ceph requires host IPC for setting up encrypted devices - hostIPC: true - # Ceph OSDs need to share the same PID namespace - hostPID: true - # hostNetwork can be set to 'false' if host networking isn't used - hostNetwork: true - hostPorts: - # Ceph messenger protocol v1 - - min: 6789 - max: 6790 # <- support old default port - # Ceph messenger protocol v2 - - min: 3300 - max: 3300 - # Ceph RADOS ports for OSDs, MDSes - - min: 6800 - max: 7300 - # # Ceph dashboard port HTTP (not recommended) - # - min: 7000 - # max: 7000 - # Ceph dashboard port HTTPS - - min: 8443 - max: 8443 - # Ceph mgr Prometheus Metrics - - min: 9283 - max: 9283 ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: 'psp:rook' -rules: - - apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - 00-rook-privileged - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-ceph-system-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-rbd-provisioner-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: ` + operatorNamespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-rbd-plugin-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-plugin-sa - namespace: ` + operatorNamespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-cephfs-provisioner-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: ` + operatorNamespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-cephfs-plugin-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-plugin-sa - namespace: ` + operatorNamespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-admission-controller - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-admission-controller-role -rules: - - apiGroups: ["ceph.rook.io"] - resources: ["*"] - verbs: ["get", "watch", "list"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-admission-controller-rolebinding -subjects: - - kind: ServiceAccount - name: rook-ceph-admission-controller - apiGroup: "" - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: rook-ceph-admission-controller-role - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: rook-ceph-operator - namespace: ` + operatorNamespace + ` - labels: - operator: rook - storage-backend: ceph -spec: - replicas: 1 - selector: - matchLabels: - app: rook-ceph-operator - template: - metadata: - labels: - app: rook-ceph-operator - spec: - serviceAccountName: rook-ceph-system - containers: - - name: rook-ceph-operator - image: rook/ceph:` + m.imageTag + ` - args: ["ceph", "operator"] - env: - - name: ROOK_LOG_LEVEL - value: INFO - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ROOK_ENABLE_FLEX_DRIVER - value: "true" - - name: ROOK_CURRENT_NAMESPACE_ONLY - value: "false"` + openshiftEnv + ` - - name: ROOK_ENABLE_DISCOVERY_DAEMON - value: "true" - volumeMounts: - - mountPath: /var/lib/rook - name: rook-config - - mountPath: /etc/ceph - name: default-config-dir - volumes: - - name: rook-config - emptyDir: {} - - name: default-config-dir - emptyDir: {} ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: rook-ceph-operator-config - namespace: ` + operatorNamespace + ` -data: - ROOK_CSI_ENABLE_CEPHFS: "true" - ROOK_CSI_ENABLE_RBD: "true" - ROOK_CSI_ENABLE_GRPC_METRICS: "true" - ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true" - CSI_LOG_LEVEL: "5" -` - return operatorManifest + return m.settings.replaceOperatorSettings(manifest) } -// GetClusterRoles returns rook-cluster manifest -func (m *CephManifestsV1_5) GetClusterRoles(namespace, operatorNamespace string) string { - return `apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-osd - namespace: ` + namespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-osd - namespace: ` + namespace + ` -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: [ "get", "list", "watch", "create", "update", "delete" ] -- apiGroups: ["ceph.rook.io"] - resources: ["cephclusters", "cephclusters/finalizers"] - verbs: [ "get", "list", "create", "update", "delete" ] ---- -# Aspects of ceph-mgr that operate within the cluster's namespace -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr - namespace: ` + namespace + ` -rules: -- apiGroups: - - "" - resources: - - pods - - services - - pods/log - verbs: - - get - - list - - watch - - delete -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" ---- -# Allow the ceph osd to access cluster-wide resources necessary for determining their topology location -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-osd-` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-osd -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: ` + namespace + ` ---- -# Allow the ceph mgr to access cluster-wide resources necessary for the mgr modules -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-cluster-` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-mgr-cluster -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -rules: -- apiGroups: - - "" - resources: - - pods - - configmaps - verbs: - - get - - list - - watch - - create - - update - - delete ---- -# Allow the operator to create resources in this cluster's namespace -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cluster-mgmt - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-cluster-mgmt -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -# Allow the osd pods in this namespace to work with configmaps -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-osd - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-osd -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: ` + namespace + ` ---- -# Allow the ceph mgr to access the cluster-specific resources necessary for the mgr modules -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-mgr -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -# Allow the ceph mgr to access the rook system resources necessary for the mgr modules -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-system ` + namespace + ` - namespace: ` + operatorNamespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-mgr-system -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-cmd-reporter -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-default-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: default - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-osd-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-mgr-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-cmd-reporter-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -` +// GetCommon returns rook-cluster manifest +func (m *CephManifestsV1_5) GetCommon() string { + return m.settings.readManifestFromGithub("common.yaml") } +// GetRookToolBox returns rook-toolbox manifest +func (m *CephManifestsV1_5) GetToolbox() string { + manifest := strings.ReplaceAll(m.settings.readManifestFromGithub("direct-mount.yaml"), "name: rook-direct-mount", "name: rook-ceph-tools") + return strings.ReplaceAll(manifest, "app: rook-direct-mount", "app: rook-ceph-tools") +} + +func (m *CephManifestsV1_5) GetCommonExternal() string { + return m.settings.readManifestFromGithub("common-external.yaml") +} + +//********************************************************************************** +//********************************************************************************** +// After a release, replace the methods below this separator from the +// ceph_manifests.go. Methods above this separator do not need to be +// copied since they will load them directly from github. +//********************************************************************************** +//********************************************************************************** + // GetRookCluster returns rook-cluster manifest -func (m *CephManifestsV1_5) GetRookCluster(settings *clusterSettings) string { - store := "# storeType not specified; Rook will use default store types" - if settings.StoreType != "" { - store = `storeType: "` + settings.StoreType + `"` - } - +func (m *CephManifestsV1_5) GetCephCluster() string { crushRoot := "# crushRoot not specified; Rook will use `default`" - if settings.Mons == 1 { + if m.settings.Mons == 1 { crushRoot = `crushRoot: "custom-root"` } - if settings.UsePVCs { + if m.settings.UsePVC { return `apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: # set the name to something different from the namespace - name: ` + settings.ClusterName + ` - namespace: ` + settings.Namespace + ` + name: ` + m.settings.ClusterName + ` + namespace: ` + m.settings.Namespace + ` spec: - dataDirHostPath: ` + settings.DataDirHostPath + ` + dataDirHostPath: ` + m.settings.DataDirHostPath + ` mon: - count: ` + strconv.Itoa(settings.Mons) + ` + count: ` + strconv.Itoa(m.settings.Mons) + ` allowMultiplePerNode: true volumeClaimTemplate: spec: - storageClassName: ` + settings.StorageClassName + ` + storageClassName: ` + m.settings.StorageClassName + ` resources: requests: storage: 5Gi cephVersion: - image: ` + settings.CephVersion.Image + ` - allowUnsupported: ` + strconv.FormatBool(settings.CephVersion.AllowUnsupported) + ` + image: ` + m.settings.CephVersion.Image + ` + allowUnsupported: ` + strconv.FormatBool(m.settings.CephVersion.AllowUnsupported) + ` skipUpgradeChecks: false continueUpgradeAfterChecksEvenIfNotHealthy: false dashboard: @@ -1253,7 +130,7 @@ spec: resources: requests: storage: 10Gi - storageClassName: ` + settings.StorageClassName + ` + storageClassName: ` + m.settings.StorageClassName + ` volumeMode: Block accessModes: - ReadWriteOnce @@ -1268,28 +145,27 @@ spec: return `apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: - name: ` + settings.ClusterName + ` - namespace: ` + settings.Namespace + ` + name: ` + m.settings.ClusterName + ` + namespace: ` + m.settings.Namespace + ` spec: cephVersion: - image: ` + settings.CephVersion.Image + ` - allowUnsupported: ` + strconv.FormatBool(settings.CephVersion.AllowUnsupported) + ` - dataDirHostPath: ` + settings.DataDirHostPath + ` + image: ` + m.settings.CephVersion.Image + ` + allowUnsupported: ` + strconv.FormatBool(m.settings.CephVersion.AllowUnsupported) + ` + dataDirHostPath: ` + m.settings.DataDirHostPath + ` network: hostNetwork: false mon: - count: ` + strconv.Itoa(settings.Mons) + ` + count: ` + strconv.Itoa(m.settings.Mons) + ` allowMultiplePerNode: true dashboard: enabled: true skipUpgradeChecks: true metadataDevice: storage: - useAllNodes: ` + strconv.FormatBool(!settings.skipOSDCreation) + ` - useAllDevices: ` + strconv.FormatBool(!settings.skipOSDCreation) + ` + useAllNodes: ` + strconv.FormatBool(!m.settings.SkipOSDCreation) + ` + useAllDevices: ` + strconv.FormatBool(!m.settings.SkipOSDCreation) + ` deviceFilter: '' config: - ` + store + ` databaseSizeMB: "1024" journalSizeMB: "1024" mgr: @@ -1309,174 +185,44 @@ spec: interval: 5s` } -// GetRookToolBox returns rook-toolbox manifest -func (m *CephManifestsV1_5) GetRookToolBox(namespace string) string { - return `apiVersion: v1 -kind: Pod -metadata: - name: rook-ceph-tools - namespace: ` + namespace + ` -spec: - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: rook-ceph-tools - image: rook/ceph:` + m.imageTag + ` - imagePullPolicy: IfNotPresent - command: ["/tini"] - args: ["-g", "--", "/usr/local/bin/toolbox.sh"] - env: - - name: ROOK_CEPH_USERNAME - valueFrom: - secretKeyRef: - name: rook-ceph-mon - key: ceph-username - - name: ROOK_CEPH_SECRET - valueFrom: - secretKeyRef: - name: rook-ceph-mon - key: ceph-secret - securityContext: - privileged: true - volumeMounts: - - mountPath: /dev - name: dev - - mountPath: /sys/bus - name: sysbus - - mountPath: /lib/modules - name: libmodules - - name: mon-endpoint-volume - mountPath: /etc/rook - hostNetwork: false - volumes: - - name: dev - hostPath: - path: /dev - - name: sysbus - hostPath: - path: /sys/bus - - name: libmodules - hostPath: - path: /lib/modules - - name: mon-endpoint-volume - configMap: - name: rook-ceph-mon-endpoints - items: - - key: data - path: mon-endpoints` -} - -func (m *CephManifestsV1_5) GetBlockSnapshotClass(snapshotClassName, namespace, operatorNamespace, reclaimPolicy string) string { +func (m *CephManifestsV1_5) GetBlockSnapshotClass(snapshotClassName, reclaimPolicy string) string { // Create a CSI driver snapshotclass return ` apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshotClass metadata: name: ` + snapshotClassName + ` -driver: ` + operatorNamespace + `.rbd.csi.ceph.com +driver: ` + m.settings.OperatorNamespace + `.rbd.csi.ceph.com deletionPolicy: ` + reclaimPolicy + ` parameters: - clusterID: ` + namespace + ` + clusterID: ` + m.settings.Namespace + ` csi.storage.k8s.io/snapshotter-secret-name: rook-csi-rbd-provisioner - csi.storage.k8s.io/snapshotter-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/snapshotter-secret-namespace: ` + m.settings.Namespace + ` ` } -func (m *CephManifestsV1_5) GetFileStorageSnapshotClass(snapshotClassName, namespace, operatorNamespace, reclaimPolicy string) string { +func (m *CephManifestsV1_5) GetFileStorageSnapshotClass(snapshotClassName, reclaimPolicy string) string { // Create a CSI driver snapshotclass return ` apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshotClass metadata: name: ` + snapshotClassName + ` -driver: ` + operatorNamespace + `.cephfs.csi.ceph.com +driver: ` + m.settings.OperatorNamespace + `.cephfs.csi.ceph.com deletionPolicy: ` + reclaimPolicy + ` parameters: - clusterID: ` + namespace + ` + clusterID: ` + m.settings.Namespace + ` csi.storage.k8s.io/snapshotter-secret-name: rook-csi-cephfs-provisioner - csi.storage.k8s.io/snapshotter-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/snapshotter-secret-namespace: ` + m.settings.Namespace + ` ` } -func (m *CephManifestsV1_5) GetPVCRestore(claimName, snapshotName, namespace, storageClassName, accessModes, size string) string { - return `apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ` + claimName + ` - namespace: ` + namespace + ` -spec: - storageClassName: ` + storageClassName + ` - dataSource: - name: ` + snapshotName + ` - kind: VolumeSnapshot - apiGroup: snapshot.storage.k8s.io - accessModes: - - ` + accessModes + ` - resources: - requests: - storage: ` + size -} - -func (m *CephManifestsV1_5) GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, accessModes, size string) string { - return `apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ` + cloneClaimName + ` - namespace: ` + namespace + ` -spec: - storageClassName: ` + storageClassName + ` - dataSource: - name: ` + parentClaimName + ` - kind: PersistentVolumeClaim - accessModes: - - ` + accessModes + ` - resources: - requests: - storage: ` + size -} - -func (m *CephManifestsV1_5) GetSnapshot(snapshotName, claimName, snapshotClassName, namespace string) string { - return `apiVersion: snapshot.storage.k8s.io/v1beta1 -kind: VolumeSnapshot -metadata: - name: ` + snapshotName + ` - namespace: ` + namespace + ` -spec: - volumeSnapshotClassName: ` + snapshotClassName + ` - source: - persistentVolumeClaimName: ` + claimName -} - -func (m *CephManifestsV1_5) GetPod(podName, claimName, namespace, mountPath string, readOnly bool) string { - return ` -apiVersion: v1 -kind: Pod -metadata: - name: ` + podName + ` - namespace: ` + namespace + ` -spec: - containers: - - name: ` + podName + ` - image: busybox - command: ["/bin/sleep", "infinity"] - imagePullPolicy: IfNotPresent - volumeMounts: - - mountPath: ` + mountPath + ` - name: csivol - volumes: - - name: csivol - persistentVolumeClaim: - claimName: ` + claimName + ` - readOnly: ` + strconv.FormatBool(readOnly) + ` - restartPolicy: Never -` -} - -func (m *CephManifestsV1_5) GetBlockPoolDef(poolName string, namespace string, replicaSize string) string { +func (m *CephManifestsV1_5) GetBlockPool(poolName, replicaSize string) string { return `apiVersion: ceph.rook.io/v1 kind: CephBlockPool metadata: name: ` + poolName + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: replicated: size: ` + replicaSize + ` @@ -1495,23 +241,23 @@ spec: interval: 10s` } -func (m *CephManifestsV1_5) GetBlockStorageClassDef(csi bool, poolName, storageClassName, reclaimPolicy, namespace, operatorNamespace string) string { +func (m *CephManifestsV1_5) GetBlockStorageClass(poolName, storageClassName, reclaimPolicy string) string { // Create a CSI driver storage class - if csi { + if m.settings.UseCSI { return ` apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ` + storageClassName + ` -provisioner: ` + operatorNamespace + `.rbd.csi.ceph.com +provisioner: ` + m.settings.OperatorNamespace + `.rbd.csi.ceph.com reclaimPolicy: ` + reclaimPolicy + ` parameters: pool: ` + poolName + ` - clusterID: ` + namespace + ` + clusterID: ` + m.settings.Namespace + ` csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner - csi.storage.k8s.io/provisioner-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/provisioner-secret-namespace: ` + m.settings.Namespace + ` csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node - csi.storage.k8s.io/node-stage-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/node-stage-secret-namespace: ` + m.settings.Namespace + ` imageFeatures: layering csi.storage.k8s.io/fstype: ext4 ` @@ -1526,10 +272,10 @@ allowVolumeExpansion: true reclaimPolicy: ` + reclaimPolicy + ` parameters: blockPool: ` + poolName + ` - clusterNamespace: ` + namespace + clusterNamespace: ` + m.settings.Namespace } -func (m *CephManifestsV1_5) GetFileStorageClassDef(fsName, storageClassName, operatorNamespace, namespace string) string { +func (m *CephManifestsV1_5) GetFileStorageClass(fsName, storageClassName string) string { // Create a CSI driver storage class csiCephFSNodeSecret := "rook-csi-cephfs-node" //nolint:gosec // We safely suppress gosec in tests file csiCephFSProvisionerSecret := "rook-csi-cephfs-provisioner" //nolint:gosec // We safely suppress gosec in tests file @@ -1538,40 +284,25 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ` + storageClassName + ` -provisioner: ` + operatorNamespace + `.cephfs.csi.ceph.com +provisioner: ` + m.settings.OperatorNamespace + `.cephfs.csi.ceph.com parameters: - clusterID: ` + namespace + ` + clusterID: ` + m.settings.Namespace + ` fsName: ` + fsName + ` pool: ` + fsName + `-data0 csi.storage.k8s.io/provisioner-secret-name: ` + csiCephFSProvisionerSecret + ` - csi.storage.k8s.io/provisioner-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/provisioner-secret-namespace: ` + m.settings.Namespace + ` csi.storage.k8s.io/node-stage-secret-name: ` + csiCephFSNodeSecret + ` - csi.storage.k8s.io/node-stage-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/node-stage-secret-namespace: ` + m.settings.Namespace + ` ` } -func (m *CephManifestsV1_5) GetPVC(claimName, namespace, storageClassName, accessModes, size string) string { - return `apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ` + claimName + ` - namespace: ` + namespace + ` -spec: - storageClassName: ` + storageClassName + ` - accessModes: - - ` + accessModes + ` - resources: - requests: - storage: ` + size -} - // GetFilesystem returns the manifest to create a Rook filesystem resource with the given config. -func (m *CephManifestsV1_5) GetFilesystem(namespace, name string, activeCount int) string { +func (m *CephManifestsV1_5) GetFilesystem(name string, activeCount int) string { return `apiVersion: ceph.rook.io/v1 kind: CephFilesystem metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: metadataPool: replicated: @@ -1588,12 +319,12 @@ spec: } // GetFilesystem returns the manifest to create a Rook Ceph NFS resource with the given config. -func (m *CephManifestsV1_5) GetNFS(namespace, name, pool string, count int) string { +func (m *CephManifestsV1_5) GetNFS(name, pool string, count int) string { return `apiVersion: ceph.rook.io/v1 kind: CephNFS metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: rados: pool: ` + pool + ` @@ -1602,12 +333,12 @@ spec: active: ` + strconv.Itoa(count) } -func (m *CephManifestsV1_5) GetObjectStore(namespace, name string, replicaCount, port int) string { +func (m *CephManifestsV1_5) GetObjectStore(name string, replicaCount, port int) string { return `apiVersion: ceph.rook.io/v1 kind: CephObjectStore metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: metadataPool: replicated: @@ -1630,33 +361,33 @@ spec: ` } -func (m *CephManifestsV1_5) GetObjectStoreUser(namespace, name string, displayName string, store string) string { +func (m *CephManifestsV1_5) GetObjectStoreUser(name string, displayName string, store string) string { return `apiVersion: ceph.rook.io/v1 kind: CephObjectStoreUser metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: displayName: ` + displayName + ` store: ` + store } //GetBucketStorageClass returns the manifest to create object bucket -func (m *CephManifestsV1_5) GetBucketStorageClass(storeNameSpace string, storeName string, storageClassName string, reclaimPolicy string, region string) string { +func (m *CephManifestsV1_5) GetBucketStorageClass(storeName, storageClassName, reclaimPolicy, region string) string { return `apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ` + storageClassName + ` -provisioner: ` + storeNameSpace + `.ceph.rook.io/bucket +provisioner: ` + m.settings.Namespace + `.ceph.rook.io/bucket reclaimPolicy: ` + reclaimPolicy + ` parameters: objectStoreName: ` + storeName + ` - objectStoreNamespace: ` + storeNameSpace + ` + objectStoreNamespace: ` + m.settings.Namespace + ` region: ` + region } -//GetObc returns the manifest to create object bucket claim -func (m *CephManifestsV1_5) GetObc(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string { +//GetOBC returns the manifest to create object bucket claim +func (m *CephManifestsV1_5) GetOBC(claimName, storageClassName, objectBucketName, maxObject string, varBucketName bool) string { bucketParameter := "generateBucketName" if varBucketName { bucketParameter = "bucketName" @@ -1672,7 +403,7 @@ spec: maxObjects: "` + maxObject + `"` } -func (m *CephManifestsV1_5) GetClient(namespace string, claimName string, caps map[string]string) string { +func (m *CephManifestsV1_5) GetClient(claimName string, caps map[string]string) string { clientCaps := []string{} for name, cap := range caps { str := name + ": " + cap @@ -1682,87 +413,31 @@ func (m *CephManifestsV1_5) GetClient(namespace string, claimName string, caps m kind: CephClient metadata: name: ` + claimName + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: caps: ` + strings.Join(clientCaps, "\n ") } -func (m *CephManifestsV1_5) GetClusterExternalRoles(namespace, firstClusterNamespace string) string { - return `apiVersion: v1 -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cluster-mgmt - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-cluster-mgmt -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: ` + firstClusterNamespace + ` ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-cmd-reporter -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -rules: -- apiGroups: - - "" - resources: - - pods - - configmaps - verbs: - - get - - list - - watch - - create - - update - - delete` -} - -func (m *CephManifestsV1_5) GetRookExternalCluster(settings *clusterExternalSettings) string { +func (m *CephManifestsV1_5) GetExternalCephCluster() string { return `apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: - name: ` + settings.Namespace + ` - namespace: ` + settings.Namespace + ` + name: ` + m.settings.Namespace + ` + namespace: ` + m.settings.Namespace + ` spec: external: enable: true - dataDirHostPath: ` + settings.DataDirHostPath + `` + dataDirHostPath: ` + m.settings.DataDirHostPath + `` } // GetRBDMirror returns the manifest to create a Rook Ceph RBD Mirror resource with the given config. -func (m *CephManifestsV1_5) GetRBDMirror(namespace, name string, count int) string { +func (m *CephManifestsV1_5) GetRBDMirror(name string, count int) string { return `apiVersion: ceph.rook.io/v1 kind: CephRBDMirror metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: count: ` + strconv.Itoa(count) } diff --git a/tests/framework/installer/ceph_settings.go b/tests/framework/installer/ceph_settings.go new file mode 100644 index 000000000..e617f460d --- /dev/null +++ b/tests/framework/installer/ceph_settings.go @@ -0,0 +1,126 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installer + +import ( + "fmt" + "io/ioutil" + "net/http" + "path" + "strings" + + "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/tests/framework/utils" +) + +// TestCephSettings struct for handling panic and test suite tear down +type TestCephSettings struct { + DataDirHostPath string + ClusterName string + Namespace string + OperatorNamespace string + StorageClassName string + UseHelm bool + UsePVC bool + Mons int + UseCrashPruner bool + MultipleMgrs bool + SkipOSDCreation bool + UseCSI bool + EnableDiscovery bool + EnableAdmissionController bool + IsExternal bool + RookVersion string + CephVersion cephv1.CephVersionSpec +} + +func (s *TestCephSettings) readManifest(filename string) string { + rootDir, err := utils.FindRookRoot() + if err != nil { + panic(err) + } + manifest := path.Join(rootDir, "cluster/examples/kubernetes/ceph", filename) + logger.Infof("Reading manifest: %s", manifest) + contents, err := ioutil.ReadFile(manifest) + if err != nil { + panic(errors.Wrapf(err, "failed to read manifest at %s", manifest)) + } + return replaceNamespaces(manifest, string(contents), s.OperatorNamespace, s.Namespace) +} + +func (s *TestCephSettings) readManifestFromGithub(filename string) string { + return s.readManifestFromGithubWithClusterNamespace(filename, s.Namespace) +} + +func (s *TestCephSettings) readManifestFromGithubWithClusterNamespace(filename, clusterNamespace string) string { + url := fmt.Sprintf("https://raw.githubusercontent.com/rook/rook/%s/cluster/examples/kubernetes/ceph/%s", s.RookVersion, filename) + logger.Infof("Retrieving manifest: %s", url) + // #nosec G107 This is only test code and is expected to read from a url + response, err := http.Get(url) + if err != nil { + panic(errors.Wrapf(err, "failed to read manifest from %s", url)) + } + defer response.Body.Close() + + content, err := ioutil.ReadAll(response.Body) + if err != nil { + panic(errors.Wrapf(err, "failed to read content from %s", url)) + } + return replaceNamespaces(url, string(content), s.OperatorNamespace, clusterNamespace) +} + +func (s *TestCephSettings) replaceOperatorSettings(manifest string) string { + manifest = strings.ReplaceAll(manifest, `# CSI_LOG_LEVEL: "0"`, `CSI_LOG_LEVEL: "5"`) + manifest = strings.ReplaceAll(manifest, `ROOK_ENABLE_DISCOVERY_DAEMON: "false"`, fmt.Sprintf(`ROOK_ENABLE_DISCOVERY_DAEMON: "%t"`, s.EnableDiscovery)) + manifest = strings.ReplaceAll(manifest, `ROOK_ENABLE_FLEX_DRIVER: "false"`, fmt.Sprintf(`ROOK_ENABLE_FLEX_DRIVER: "%t"`, !s.UseCSI)) + manifest = strings.ReplaceAll(manifest, `ROOK_CSI_ENABLE_CEPHFS: "true"`, fmt.Sprintf(`ROOK_CSI_ENABLE_CEPHFS: "%t"`, s.UseCSI)) + manifest = strings.ReplaceAll(manifest, `ROOK_CSI_ENABLE_RBD: "true"`, fmt.Sprintf(`ROOK_CSI_ENABLE_RBD: "%t"`, s.UseCSI)) + return manifest +} + +func replaceNamespaces(name, manifest, operatorNamespace, clusterNamespace string) string { + // RBAC and related namespaces + manifest = strings.ReplaceAll(manifest, "rook-ceph # namespace:operator", operatorNamespace) + manifest = strings.ReplaceAll(manifest, "rook-ceph # namespace:cluster", clusterNamespace) + manifest = strings.ReplaceAll(manifest, "rook-ceph-external # namespace:cluster", clusterNamespace) + // Double space only needed for v1.5 upgrade test + manifest = strings.ReplaceAll(manifest, "rook-ceph # namespace:operator", operatorNamespace) + + // SCC namespaces for operator and Ceph daemons + manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-system # serviceaccount:namespace:operator", operatorNamespace+":rook-ceph-system") + manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-mgr # serviceaccount:namespace:cluster", clusterNamespace+":rook-ceph-mgr") + manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-osd # serviceaccount:namespace:cluster", clusterNamespace+":rook-ceph-osd") + + // SCC namespaces for CSI driver + manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-rbd-plugin-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-rbd-plugin-sa") + manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-rbd-provisioner-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-rbd-provisioner-sa") + manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-cephfs-plugin-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-cephfs-plugin-sa") + manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-cephfs-provisioner-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-cephfs-provisioner-sa") + + // CSI Drivers + manifest = strings.ReplaceAll(manifest, "rook-ceph.cephfs.csi.ceph.com # driver:namespace:operator", operatorNamespace+".cephfs.csi.ceph.com") + manifest = strings.ReplaceAll(manifest, "rook-ceph.rbd.csi.ceph.com # driver:namespace:operator", operatorNamespace+".rbd.csi.ceph.com") + + // Bucket storage class + manifest = strings.ReplaceAll(manifest, "rook-ceph.ceph.rook.io/bucket # driver:namespace:cluster", clusterNamespace+".ceph.rook.io/bucket") + if strings.Contains(manifest, "namespace:operator") || strings.Contains(manifest, "namespace:cluster") || strings.Contains(manifest, "driver:namespace:") || strings.Contains(manifest, "serviceaccount:namespace:") { + logger.Infof("BAD MANIFEST:\n%s", manifest) + panic(fmt.Sprintf("manifest %s still contains a namespace identifier", name)) + } + return manifest +} diff --git a/tests/framework/installer/general_manifests.go b/tests/framework/installer/general_manifests.go new file mode 100644 index 000000000..e544cbf78 --- /dev/null +++ b/tests/framework/installer/general_manifests.go @@ -0,0 +1,110 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installer + +import ( + "strconv" +) + +func GetPodWithVolume(podName, claimName, namespace, mountPath string, readOnly bool) string { + return ` +apiVersion: v1 +kind: Pod +metadata: + name: ` + podName + ` + namespace: ` + namespace + ` +spec: + containers: + - name: ` + podName + ` + image: busybox + command: ["/bin/sleep", "infinity"] + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: ` + mountPath + ` + name: csivol + volumes: + - name: csivol + persistentVolumeClaim: + claimName: ` + claimName + ` + readOnly: ` + strconv.FormatBool(readOnly) + ` + restartPolicy: Never +` +} + +func GetPVC(claimName, namespace, storageClassName, accessModes, size string) string { + return `apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ` + claimName + ` + namespace: ` + namespace + ` +spec: + storageClassName: ` + storageClassName + ` + accessModes: + - ` + accessModes + ` + resources: + requests: + storage: ` + size +} + +func GetPVCRestore(claimName, snapshotName, namespace, storageClassName, accessModes, size string) string { + return `apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ` + claimName + ` + namespace: ` + namespace + ` +spec: + storageClassName: ` + storageClassName + ` + dataSource: + name: ` + snapshotName + ` + kind: VolumeSnapshot + apiGroup: snapshot.storage.k8s.io + accessModes: + - ` + accessModes + ` + resources: + requests: + storage: ` + size +} + +func GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, accessModes, size string) string { + return `apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ` + cloneClaimName + ` + namespace: ` + namespace + ` +spec: + storageClassName: ` + storageClassName + ` + dataSource: + name: ` + parentClaimName + ` + kind: PersistentVolumeClaim + accessModes: + - ` + accessModes + ` + resources: + requests: + storage: ` + size +} + +func GetSnapshot(snapshotName, claimName, snapshotClassName, namespace string) string { + return `apiVersion: snapshot.storage.k8s.io/v1beta1 +kind: VolumeSnapshot +metadata: + name: ` + snapshotName + ` + namespace: ` + namespace + ` +spec: + volumeSnapshotClassName: ` + snapshotClassName + ` + source: + persistentVolumeClaimName: ` + claimName +} diff --git a/tests/framework/installer/installer.go b/tests/framework/installer/installer.go index 4c7d0f673..66bf6f567 100644 --- a/tests/framework/installer/installer.go +++ b/tests/framework/installer/installer.go @@ -45,11 +45,6 @@ var ( deleteFromStdinArgs = append(deleteArgs, "-") ) -type TestSuite interface { - Setup() - Teardown() -} - func SkipTestSuite(name string) bool { testsToRun := testStorageProvider() // jenkins passes "null" if the env var is not set. diff --git a/tests/framework/installer/nfs_installer.go b/tests/framework/installer/nfs_installer.go index 578d1548b..d70bf403b 100644 --- a/tests/framework/installer/nfs_installer.go +++ b/tests/framework/installer/nfs_installer.go @@ -166,7 +166,7 @@ func (h *NFSInstaller) UninstallNFSServer(systemNamespace, namespace string) { _, err := h.k8shelper.RookClientset.NfsV1alpha1().NFSServers(namespace).Get(ctx, namespace, metav1.GetOptions{}) return err } - err = h.k8shelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc) + err = h.k8shelper.WaitForCustomResourceDeletion(namespace, namespace, crdCheckerFunc) checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace)) err = h.k8shelper.DeleteResource("namespace", namespace) diff --git a/tests/framework/utils/helm_helper.go b/tests/framework/utils/helm_helper.go index 41832bb27..9264bd4d7 100644 --- a/tests/framework/utils/helm_helper.go +++ b/tests/framework/utils/helm_helper.go @@ -62,7 +62,7 @@ func (h *HelmHelper) InstallLocalRookHelmChart(namespace, chartSettings string) return errors.Wrap(err, "failed to find rook root") } chartDir := path.Join(rootDir, "cluster/charts/rook-ceph/") - cmdArgs := []string{"install", HelmDeployName, chartDir, "-f", path.Join(chartDir, "values.yaml")} + cmdArgs := []string{"install", "--create-namespace", HelmDeployName, chartDir, "-f", path.Join(chartDir, "values.yaml")} if namespace != "" { cmdArgs = append(cmdArgs, "--namespace", namespace) } diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index 8498a4571..25cbbc94a 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -347,7 +347,12 @@ func (k8sh *K8sHelper) ResourceOperationFromTemplate(action string, podDefinitio // ResourceOperation performs a kubectl action on a pod definition func (k8sh *K8sHelper) ResourceOperation(action string, manifest string) error { args := []string{action, "-f", "-"} - logger.Infof("kubectl %s manifest:\n%s", action, manifest) + maxManifestCharsToPrint := 4000 + if len(manifest) > maxManifestCharsToPrint { + logger.Infof("kubectl %s manifest (too long to print)", action) + } else { + logger.Infof("kubectl %s manifest:\n%s", action, manifest) + } _, err := k8sh.KubectlWithStdin(manifest, args...) if err == nil { return nil @@ -381,23 +386,23 @@ func (k8sh *K8sHelper) DeleteResource(args ...string) error { } // WaitForCustomResourceDeletion waits for the CRD deletion -func (k8sh *K8sHelper) WaitForCustomResourceDeletion(namespace string, checkerFunc func() error) error { +func (k8sh *K8sHelper) WaitForCustomResourceDeletion(namespace, name string, checkerFunc func() error) error { // wait for the operator to finalize and delete the CRD for i := 0; i < 60; i++ { err := checkerFunc() if err == nil { - logger.Infof("custom resource %s still exists", namespace) + logger.Infof("custom resource %q in namespace %q still exists", name, namespace) time.Sleep(2 * time.Second) continue } if kerrors.IsNotFound(err) { - logger.Infof("custom resource %s deleted", namespace) + logger.Infof("custom resource %q in namespace %s deleted", name, namespace) return nil } return err } - logger.Errorf("gave up deleting custom resource %s", namespace) + logger.Errorf("gave up deleting custom resource %q ", name) return nil } @@ -1218,6 +1223,7 @@ func (k8sh *K8sHelper) IsPodInExpectedState(podNamePattern string, namespace str } } + logger.Infof("waiting for pod with label app=%s in namespace %q to be in state %q...", podNamePattern, namespace, state) time.Sleep(RetryInterval * time.Second) } diff --git a/tests/integration/ceph_base_block_test.go b/tests/integration/ceph_base_block_test.go index aa68f4a38..a4e8e2ef9 100644 --- a/tests/integration/ceph_base_block_test.go +++ b/tests/integration/ceph_base_block_test.go @@ -24,7 +24,6 @@ import ( "testing" "time" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/tests/framework/clients" @@ -269,8 +268,6 @@ func runBlockCSITest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite. poolName := "replicapool" storageClassName := "rook-ceph-block" blockName := "block-pv-claim" - systemNamespace := installer.SystemNamespace(namespace) - podNameWithPVRetained := "block-test-retained" poolNameRetained := "replicapoolretained" storageClassNameRetained := "rook-ceph-block-retained" @@ -287,10 +284,10 @@ func runBlockCSITest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite. assert.Equal(s.T(), 0, len(initBlockImages), "there should not already be any images in the pool") logger.Infof("step 1: Create block storage") - err := helper.BlockClient.CreateStorageClassAndPVC(true, defaultNamespace, namespace, systemNamespace, poolName, storageClassName, "Delete", blockName, "ReadWriteOnce") + err := helper.BlockClient.CreateStorageClassAndPVC(defaultNamespace, poolName, storageClassName, "Delete", blockName, "ReadWriteOnce") require.NoError(s.T(), err) require.NoError(s.T(), retryBlockImageCountCheck(helper, clusterInfo, 1), "Make sure a new block is created") - err = helper.BlockClient.CreateStorageClassAndPVC(true, defaultNamespace, namespace, systemNamespace, poolNameRetained, storageClassNameRetained, "Retain", blockNameRetained, "ReadWriteOnce") + err = helper.BlockClient.CreateStorageClassAndPVC(defaultNamespace, poolNameRetained, storageClassNameRetained, "Retain", blockNameRetained, "ReadWriteOnce") require.NoError(s.T(), err) require.NoError(s.T(), retryBlockImageCountCheck(helper, clusterInfo, 2), "Make sure another new block is created") logger.Infof("Block Storage created successfully") @@ -405,7 +402,8 @@ func restartOSDPods(k8sh *utils.K8sHelper, s suite.Suite, namespace string) { options := metav1.DeleteOptions{} err := k8sh.Clientset.CoreV1().Pods(namespace).Delete(ctx, pod.Name, options) assert.NoError(s.T(), err) - + } + for _, pod := range pods.Items { logger.Infof("Waiting for osd pod %s to be deleted", pod.Name) deleted := k8sh.WaitUntilPodIsDeleted(pod.Name, namespace) assert.True(s.T(), deleted) @@ -417,20 +415,20 @@ func restartOSDPods(k8sh *utils.K8sHelper, s suite.Suite, namespace string) { assert.NoError(s.T(), err) } -func runBlockCSITestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, clusterNamespace, systemNamespace string, version cephv1.CephVersionSpec) { +func runBlockCSITestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings) { checkSkipCSITest(s.T(), k8sh) logger.Infof("Block Storage End to End Integration Test - create storageclass,pool and pvc") - logger.Infof("Running on Rook Cluster %s", clusterNamespace) - clusterInfo := client.AdminClusterInfo(clusterNamespace) + logger.Infof("Running on Rook Cluster %s", settings.Namespace) + clusterInfo := client.AdminClusterInfo(settings.Namespace) poolName := "rookpool" storageClassName := "rook-ceph-block-lite" blockName := "test-block-claim-lite" podName := "test-pod-lite" defer blockTestDataCleanUp(helper, k8sh, s, clusterInfo, poolName, storageClassName, blockName, podName, true) - setupBlockLite(helper, k8sh, s, clusterInfo, systemNamespace, poolName, storageClassName, blockName, podName, version) + setupBlockLite(helper, k8sh, s, clusterInfo, poolName, storageClassName, blockName, podName) if !skipSnapshotTest(k8sh) { - blockCSISnapshotTest(helper, k8sh, s, storageClassName, clusterNamespace) + blockCSISnapshotTest(helper, k8sh, s, storageClassName, settings.Namespace) } if !skipCloneTest(k8sh) { @@ -439,7 +437,7 @@ func runBlockCSITestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s su } func setupBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, clusterInfo *client.ClusterInfo, - systemNamespace, poolName, storageClassName, blockName, podName string, version cephv1.CephVersionSpec) { + poolName, storageClassName, blockName, podName string) { // Check initial number of blocks initialBlocks, err := helper.BlockClient.ListAllImages(clusterInfo) @@ -449,7 +447,7 @@ func setupBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S logger.Infof("step : Create Pool,StorageClass and PVC") - err = helper.BlockClient.CreateStorageClassAndPVC(true, defaultNamespace, clusterInfo.Namespace, systemNamespace, poolName, storageClassName, "Delete", blockName, "ReadWriteOnce") + err = helper.BlockClient.CreateStorageClassAndPVC(defaultNamespace, poolName, storageClassName, "Delete", blockName, "ReadWriteOnce") require.NoError(s.T(), err) require.True(s.T(), k8sh.WaitUntilPVCIsBound(defaultNamespace, blockName)) diff --git a/tests/integration/ceph_base_deploy_test.go b/tests/integration/ceph_base_deploy_test.go index 5971a707c..204de0e86 100644 --- a/tests/integration/ceph_base_deploy_test.go +++ b/tests/integration/ceph_base_deploy_test.go @@ -25,7 +25,6 @@ import ( "testing" "github.com/coreos/pkg/capnslog" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" @@ -87,36 +86,15 @@ func checkIfRookClusterIsHealthy(s suite.Suite, testClient *clients.TestClient, require.Nil(s.T(), err) } -func HandlePanics(r interface{}, op installer.TestSuite, t func() *testing.T) { +func HandlePanics(r interface{}, uninstaller func(), t func() *testing.T) { if r != nil { logger.Infof("unexpected panic occurred during test %s, --> %v", t().Name(), r) t().Fail() - op.Teardown() + uninstaller() t().FailNow() } } -// TestCluster struct for handling panic and test suite tear down -type TestCluster struct { - installer *installer.CephInstaller - kh *utils.K8sHelper - helper *clients.TestClient - T func() *testing.T - clusterName string - namespace string - storeType string - storageClassName string - useHelm bool - usePVC bool - mons int - rbdMirrorWorkers int - rookCephCleanup bool - skipOSDCreation bool - minimalMatrixK8sVersion string - rookVersion string - cephVersion cephv1.CephVersionSpec -} - func checkIfShouldRunForMinimalTestMatrix(t func() *testing.T, k8sh *utils.K8sHelper, version string) { testArgs := os.Getenv("TEST_ARGUMENTS") if !strings.Contains(testArgs, "min-test-matrix") { @@ -140,50 +118,27 @@ func checkIfShouldRunForMinimalTestMatrix(t func() *testing.T, k8sh *utils.K8sHe logger.Infof("Running test suite since kube version is %q", kubeVersion) } -// StartTestCluster creates new instance of TestCluster struct -func StartTestCluster(t func() *testing.T, cluster *TestCluster) (*TestCluster, *utils.K8sHelper) { - kh, err := utils.CreateK8sHelper(t) +// StartTestCluster creates new instance of TestCephSettings struct +func StartTestCluster(t func() *testing.T, settings *installer.TestCephSettings, minimalMatrixK8sVersion string) (*installer.CephInstaller, *utils.K8sHelper) { + k8shelper, err := utils.CreateK8sHelper(t) require.NoError(t(), err) - checkIfShouldRunForMinimalTestMatrix(t, kh, cluster.minimalMatrixK8sVersion) + checkIfShouldRunForMinimalTestMatrix(t, k8shelper, minimalMatrixK8sVersion) - cluster.installer = installer.NewCephInstaller(t, kh.Clientset, cluster.useHelm, cluster.clusterName, cluster.rookVersion, cluster.cephVersion, cluster.rookCephCleanup) - cluster.kh = kh - cluster.helper = nil - cluster.T = t - - if cluster.rookVersion != installer.VersionMaster { - // make sure we have the images from a previous release locally so the test doesn't hit a timeout - assert.NoError(t(), kh.GetDockerImage("rook/ceph:"+cluster.rookVersion)) - } - - assert.NoError(t(), kh.GetDockerImage(cluster.cephVersion.Image)) - - cluster.Setup() - return cluster, kh -} - -// Setup is a wrapper for setting up rook -func (op *TestCluster) Setup() { // Turn on DEBUG logging capnslog.SetGlobalLogLevel(capnslog.DEBUG) - isRookInstalled, err := op.installer.InstallRook(op.namespace, op.storeType, op.usePVC, op.storageClassName, - cephv1.MonSpec{Count: op.mons, AllowMultiplePerNode: true}, false /* startWithAllNodes */, op.rbdMirrorWorkers, op.skipOSDCreation, op.rookVersion) + + installer := installer.NewCephInstaller(t, k8shelper.Clientset, settings) + isRookInstalled, err := installer.InstallRook() if !isRookInstalled || err != nil { logger.Errorf("Rook was not installed successfully: %v", err) - if !op.installer.T().Failed() { - op.installer.GatherAllRookLogs(op.installer.T().Name(), op.namespace, installer.SystemNamespace(op.namespace)) + if !installer.T().Failed() { + installer.GatherAllRookLogs(t().Name(), settings.Namespace, settings.OperatorNamespace) } - op.T().Fail() - op.Teardown() - op.T().FailNow() + t().Fail() + installer.UninstallRook() + t().FailNow() } -} -// SetInstallData updates the installer helper based on the version of Rook desired -func (op *TestCluster) SetInstallData(version string) {} - -// Teardown is a wrapper for tearDown after Suite -func (op *TestCluster) Teardown() { - op.installer.UninstallRook(op.namespace) + return installer, k8shelper } diff --git a/tests/integration/ceph_base_file_test.go b/tests/integration/ceph_base_file_test.go index dad175090..d7b3fd07a 100644 --- a/tests/integration/ceph_base_file_test.go +++ b/tests/integration/ceph_base_file_test.go @@ -239,83 +239,83 @@ func fileSystemCSISnapshotTest(helper *clients.TestClient, k8sh *utils.K8sHelper // Smoke Test for File System Storage - Test check the following operations on Filesystem Storage in order // Create,Mount,Write,Read,Unmount and Delete. -func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName string, useCSI bool, preserveFilesystemOnDelete bool) { - if useCSI { +func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string, preserveFilesystemOnDelete bool) { + if settings.UseCSI { checkSkipCSITest(s.T(), k8sh) } - defer fileTestDataCleanUp(helper, k8sh, s, filePodName, namespace, filesystemName) - logger.Infof("Running on Rook Cluster %s", namespace) + defer fileTestDataCleanUp(helper, k8sh, s, filePodName, settings.Namespace, filesystemName) + logger.Infof("Running on Rook Cluster %s", settings.Namespace) logger.Infof("File Storage End To End Integration Test - create, mount, write to, read from, and unmount") activeCount := 2 - createFilesystem(helper, k8sh, s, namespace, filesystemName, activeCount) + createFilesystem(helper, k8sh, s, settings, filesystemName, activeCount) if preserveFilesystemOnDelete { - _, err := k8sh.Kubectl("-n", namespace, "patch", "CephFilesystem", filesystemName, "--type=merge", "-p", `{"spec": {"preserveFilesystemOnDelete": true}}`) + _, err := k8sh.Kubectl("-n", settings.Namespace, "patch", "CephFilesystem", filesystemName, "--type=merge", "-p", `{"spec": {"preserveFilesystemOnDelete": true}}`) assert.NoError(s.T(), err) } // Create a test pod where CephFS is consumed without user creds storageClassName := "cephfs-storageclass" - err := helper.FSClient.CreateStorageClass(filesystemName, installer.SystemNamespace(namespace), namespace, storageClassName) + err := helper.FSClient.CreateStorageClass(filesystemName, settings.OperatorNamespace, settings.Namespace, storageClassName) assert.NoError(s.T(), err) - createFilesystemConsumerPod(helper, k8sh, s, namespace, filesystemName, storageClassName, useCSI) + createFilesystemConsumerPod(helper, k8sh, s, settings, filesystemName, storageClassName) // Test reading and writing to the first pod - err = writeAndReadToFilesystem(helper, k8sh, s, namespace, filePodName, "test_file") + err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, filePodName, "test_file") assert.NoError(s.T(), err) // TODO: Also mount with user credentials with the CSI driver - if !useCSI { + if !settings.UseCSI { // Create a test pod where CephFS is consumed with a mountUser and mountSecret specified. - createFilesystemMountCephCredentials(helper, k8sh, s, namespace, filesystemName) - createFilesystemMountUserConsumerPod(helper, k8sh, s, namespace, filesystemName, storageClassName) + createFilesystemMountCephCredentials(helper, k8sh, s, settings, filesystemName) + createFilesystemMountUserConsumerPod(helper, k8sh, s, settings, filesystemName, storageClassName) // Test reading and writing to the second pod - err = writeAndReadToFilesystem(helper, k8sh, s, namespace, fileMountUserPodName, "canttouchthis") + err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, fileMountUserPodName, "canttouchthis") assert.Error(s.T(), err, "we should not be able to write to file canttouchthis on CephFS `/`") - err = writeAndReadToFilesystem(helper, k8sh, s, namespace, fileMountUserPodName, "foo/test_file") + err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, fileMountUserPodName, "foo/test_file") assert.NoError(s.T(), err, "we should be able to write to the `/foo` directory on CephFS") - cleanupFilesystemConsumer(helper, k8sh, s, namespace, fileMountUserPodName) + cleanupFilesystemConsumer(helper, k8sh, s, settings.Namespace, fileMountUserPodName) assert.NoError(s.T(), err) } // Start the NFS daemons - testNFSDaemons(helper, k8sh, s, namespace, filesystemName) + testNFSDaemons(helper, k8sh, s, settings, filesystemName) // Cleanup the filesystem and its clients - cleanupFilesystemConsumer(helper, k8sh, s, namespace, filePodName) + cleanupFilesystemConsumer(helper, k8sh, s, settings.Namespace, filePodName) assert.NoError(s.T(), err) - downscaleMetadataServers(helper, k8sh, s, namespace, filesystemName) - cleanupFilesystem(helper, k8sh, s, namespace, filesystemName) + downscaleMetadataServers(helper, k8sh, s, settings.Namespace, filesystemName) + cleanupFilesystem(helper, k8sh, s, settings.Namespace, filesystemName) err = helper.FSClient.DeleteStorageClass(storageClassName) assertNoErrorUnlessNotFound(s, err) if preserveFilesystemOnDelete { - fses, err := helper.FSClient.List(namespace) + fses, err := helper.FSClient.List(settings.Namespace) assert.NoError(s.T(), err) assert.Len(s.T(), fses, 1) assert.Equal(s.T(), fses[0].Name, filesystemName) - err = helper.FSClient.Delete(filesystemName, namespace) + err = helper.FSClient.Delete(filesystemName, settings.Namespace) assert.NoError(s.T(), err) } } -func testNFSDaemons(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) { +func testNFSDaemons(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) { name := "my-nfs" - err := helper.NFSClient.Create(namespace, name, filesystemName+"-data0", 2) + err := helper.NFSClient.Create(settings.Namespace, name, filesystemName+"-data0", 2) require.Nil(s.T(), err) - err = helper.NFSClient.Delete(namespace, name) + err = helper.NFSClient.Delete(settings.Namespace, name) assert.Nil(s.T(), err) } -func createFilesystemConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName, storageClassName string, useCSI bool) { - err := createPodWithFilesystem(k8sh, s, filePodName, namespace, filesystemName, storageClassName, false, useCSI) +func createFilesystemConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName, storageClassName string) { + err := createPodWithFilesystem(k8sh, s, settings, filePodName, filesystemName, storageClassName, false) require.NoError(s.T(), err) - filePodRunning := k8sh.IsPodRunning(filePodName, namespace) + filePodRunning := k8sh.IsPodRunning(filePodName, settings.Namespace) require.True(s.T(), filePodRunning, "make sure file-test pod is in running state") logger.Infof("File system mounted successfully") } @@ -358,36 +358,36 @@ func cleanupFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suit } // Test File System Creation on Rook that was installed on a custom namespace i.e. Namespace != "rook" and delete it again -func runFileE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) { +func runFileE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) { checkSkipCSITest(s.T(), k8sh) logger.Infof("File Storage End to End Integration Test - create Filesystem and make sure mds pod is running") - logger.Infof("Running on Rook Cluster %s", namespace) + logger.Infof("Running on Rook Cluster %s", settings.Namespace) activeCount := 1 - createFilesystem(helper, k8sh, s, namespace, filesystemName, activeCount) + createFilesystem(helper, k8sh, s, settings, filesystemName, activeCount) // Create a test pod where CephFS is consumed without user creds storageClassName := "cephfs-storageclass" - err := helper.FSClient.CreateStorageClass(filesystemName, namespace, namespace, storageClassName) + err := helper.FSClient.CreateStorageClass(filesystemName, settings.OperatorNamespace, settings.Namespace, storageClassName) assert.NoError(s.T(), err) assert.NoError(s.T(), err) if !skipSnapshotTest(k8sh) { - fileSystemCSISnapshotTest(helper, k8sh, s, storageClassName, namespace) + fileSystemCSISnapshotTest(helper, k8sh, s, storageClassName, settings.Namespace) } if !skipCloneTest(k8sh) { - fileSystemCSICloneTest(helper, k8sh, s, storageClassName, namespace) + fileSystemCSICloneTest(helper, k8sh, s, storageClassName, settings.Namespace) } - cleanupFilesystem(helper, k8sh, s, namespace, filesystemName) + cleanupFilesystem(helper, k8sh, s, settings.Namespace, filesystemName) err = helper.FSClient.DeleteStorageClass(storageClassName) assertNoErrorUnlessNotFound(s, err) } -func createFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName string, activeCount int) { +func createFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string, activeCount int) { logger.Infof("Create file System") - fscErr := helper.FSClient.Create(filesystemName, namespace, activeCount) + fscErr := helper.FSClient.Create(filesystemName, settings.Namespace, activeCount) require.Nil(s.T(), fscErr) logger.Infof("File system %s created", filesystemName) - filesystemList, _ := helper.FSClient.List(namespace) + filesystemList, _ := helper.FSClient.List(settings.Namespace) require.Equal(s.T(), 1, len(filesystemList), "There should be one shared file system present") } @@ -399,13 +399,12 @@ func fileTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s su assert.NoError(s.T(), err) } -func createPodWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podName, namespace, filesystemName, storageClassName string, mountUser, useCSI bool) error { +func createPodWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, podName, filesystemName, storageClassName string, mountUser bool) error { var testPodManifest string - if useCSI { - testPodManifest = getFilesystemCSITestPod(podName, namespace, storageClassName) + if settings.UseCSI { + testPodManifest = getFilesystemCSITestPod(settings, podName, storageClassName) } else { - driverName := installer.SystemNamespace(namespace) - testPodManifest = getFilesystemFlexTestPod(podName, namespace, filesystemName, driverName, mountUser) + testPodManifest = getFilesystemFlexTestPod(settings, podName, filesystemName, mountUser) } if err := k8sh.ResourceOperation("create", testPodManifest); err != nil { return fmt.Errorf("failed to create pod -- %s. %+v", testPodManifest, err) @@ -413,7 +412,7 @@ func createPodWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podName, name return nil } -func getFilesystemFlexTestPod(podName, namespace, filesystemName, driverName string, mountUser bool) string { +func getFilesystemFlexTestPod(settings *installer.TestCephSettings, podName, filesystemName string, mountUser bool) string { mountUserInsert := "" if mountUser { mountUserInsert = ` @@ -425,7 +424,7 @@ func getFilesystemFlexTestPod(podName, namespace, filesystemName, driverName str kind: Pod metadata: name: ` + podName + ` - namespace: ` + namespace + ` + namespace: ` + settings.Namespace + ` spec: containers: - name: ` + podName + ` @@ -444,23 +443,23 @@ spec: volumes: - name: ` + filesystemName + ` flexVolume: - driver: ceph.rook.io/` + driverName + ` + driver: ceph.rook.io/` + settings.OperatorNamespace + ` fsType: ceph options: fsName: ` + filesystemName + ` - clusterNamespace: ` + namespace + mountUserInsert + ` + clusterNamespace: ` + settings.Namespace + mountUserInsert + ` restartPolicy: Always ` } -func getFilesystemCSITestPod(podName, namespace, storageClassName string) string { +func getFilesystemCSITestPod(settings *installer.TestCephSettings, podName, storageClassName string) string { claimName := podName return ` apiVersion: v1 kind: PersistentVolumeClaim metadata: name: ` + claimName + ` - namespace: ` + namespace + ` + namespace: ` + settings.Namespace + ` spec: accessModes: - ReadWriteOnce @@ -473,7 +472,7 @@ apiVersion: v1 kind: Pod metadata: name: ` + podName + ` - namespace: ` + namespace + ` + namespace: ` + settings.Namespace + ` spec: containers: - name: ` + podName + ` @@ -496,20 +495,20 @@ spec: ` } -func createFilesystemMountCephCredentials(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) { +func createFilesystemMountCephCredentials(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) { ctx := context.TODO() // Create agent binding for access to Secrets - err := k8sh.ResourceOperation("apply", getFilesystemAgentMountSecretsBinding(namespace)) + err := k8sh.ResourceOperation("apply", getFilesystemAgentMountSecretsBinding(settings.Namespace)) require.Nil(s.T(), err) // Mount CephFS in toolbox and create /foo directory on it logger.Info("Creating /foo directory on CephFS") - _, err = k8sh.Exec(namespace, "rook-ceph-tools", "mkdir", []string{"-p", utils.TestMountPath}) + _, err = k8sh.Exec(settings.Namespace, client.RunAllCephCommandsInToolboxPod, "mkdir", []string{"-p", utils.TestMountPath}) require.Nil(s.T(), err) - _, err = k8sh.ExecWithRetry(10, namespace, "rook-ceph-tools", "bash", []string{"-c", fmt.Sprintf("mount -t ceph -o mds_namespace=%s,name=admin,secret=$(grep key /etc/ceph/keyring | awk '{print $3}') $(grep mon_host /etc/ceph/ceph.conf | awk '{print $3}'):/ %s", filesystemName, utils.TestMountPath)}) + _, err = k8sh.ExecWithRetry(10, settings.Namespace, client.RunAllCephCommandsInToolboxPod, "bash", []string{"-c", fmt.Sprintf("mount -t ceph -o mds_namespace=%s,name=admin,secret=$(grep key /etc/ceph/keyring | awk '{print $3}') $(grep mon_host /etc/ceph/ceph.conf | awk '{print $3}'):/ %s", filesystemName, utils.TestMountPath)}) require.Nil(s.T(), err) - _, err = k8sh.Exec(namespace, "rook-ceph-tools", "mkdir", []string{"-p", fmt.Sprintf("%s/foo", utils.TestMountPath)}) + _, err = k8sh.Exec(settings.Namespace, client.RunAllCephCommandsInToolboxPod, "mkdir", []string{"-p", fmt.Sprintf("%s/foo", utils.TestMountPath)}) require.Nil(s.T(), err) - _, err = k8sh.Exec(namespace, "rook-ceph-tools", "umount", []string{utils.TestMountPath}) + _, err = k8sh.Exec(settings.Namespace, client.RunAllCephCommandsInToolboxPod, "umount", []string{utils.TestMountPath}) require.Nil(s.T(), err) logger.Info("Created /foo directory on CephFS") @@ -523,15 +522,15 @@ func createFilesystemMountCephCredentials(helper *clients.TestClient, k8sh *util ), } logger.Infof("ceph credentials command args: %s", commandArgs[1]) - result, err := k8sh.Exec(namespace, "rook-ceph-tools", "bash", commandArgs) + result, err := k8sh.Exec(settings.Namespace, client.RunAllCephCommandsInToolboxPod, "bash", commandArgs) logger.Infof("Ceph filesystem credentials output: %s", result) logger.Info("Created Ceph credentials") require.Nil(s.T(), err) // Save Ceph credentials to Kubernetes - _, err = k8sh.Clientset.CoreV1().Secrets(namespace).Create(ctx, &v1.Secret{ + _, err = k8sh.Clientset.CoreV1().Secrets(settings.Namespace).Create(ctx, &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: fileMountSecret, - Namespace: namespace, + Namespace: settings.Namespace, }, Data: map[string][]byte{ "mykey": []byte(result), @@ -541,12 +540,11 @@ func createFilesystemMountCephCredentials(helper *clients.TestClient, k8sh *util logger.Info("Created Ceph credentials Secret in Kubernetes") } -func createFilesystemMountUserConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName, storageClassName string) { +func createFilesystemMountUserConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName, storageClassName string) { // TODO: Mount with user credentials for the CSI driver - useCSI := false - mtfsErr := createPodWithFilesystem(k8sh, s, fileMountUserPodName, namespace, filesystemName, storageClassName, true, useCSI) + mtfsErr := createPodWithFilesystem(k8sh, s, settings, fileMountUserPodName, filesystemName, storageClassName, true) require.Nil(s.T(), mtfsErr) - filePodRunning := k8sh.IsPodRunning(fileMountUserPodName, namespace) + filePodRunning := k8sh.IsPodRunning(fileMountUserPodName, settings.Namespace) require.True(s.T(), filePodRunning, "make sure file-mountuser-test pod is in running state") logger.Infof("File system mounted successfully") } diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go index 3cbb6c19a..4cb263334 100644 --- a/tests/integration/ceph_base_object_test.go +++ b/tests/integration/ceph_base_object_test.go @@ -24,9 +24,9 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" rgw "github.com/rook/rook/pkg/operator/ceph/object" "github.com/rook/rook/tests/framework/clients" + "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -59,11 +59,11 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite logger.Infof("Step 0 : Create Object Store User") cosuErr := helper.ObjectUserClient.Create(namespace, userid, userdisplayname, storeName) - require.Nil(s.T(), cosuErr) + assert.Nil(s.T(), cosuErr) logger.Infof("Step 1 : Create Object Store") cobsErr := helper.ObjectClient.Create(namespace, storeName, 3) - require.Nil(s.T(), cobsErr) + assert.Nil(s.T(), cobsErr) // check that ObjectStore is created logger.Infof("Check that RGW pods are Running") @@ -86,7 +86,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite assert.True(s.T(), helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid)) userInfo, err := helper.ObjectUserClient.GetUser(namespace, storeName, userid) - require.NoError(s.T(), err) + assert.NoError(s.T(), err) assert.Equal(s.T(), userid, userInfo.UserID) assert.Equal(s.T(), userdisplayname, *userInfo.DisplayName) logger.Infof("Done creating object store user") @@ -107,11 +107,11 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite assert.NotEmpty(s.T(), objectStore.Status.Info["endpoint"]) break } - require.NotEqual(s.T(), i, 4) + assert.NotEqual(s.T(), i, 4) logger.Infof("Step 2 : Test Deleting User") dosuErr := helper.ObjectUserClient.Delete(namespace, userid) - require.Nil(s.T(), dosuErr) + assert.Nil(s.T(), dosuErr) logger.Infof("Object store user deleted successfully") logger.Infof("Checking to see if the user secret has been deleted") for i = 0; i < 4 && helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid) == true; i++ { @@ -119,7 +119,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite time.Sleep(5 * time.Second) } assert.False(s.T(), helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid)) - require.NotEqual(s.T(), i, 4) + assert.NotEqual(s.T(), i, 4) logger.Infof("Check that MGRs are not in a crashloop") assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mgr", namespace, 1, "Running")) @@ -129,14 +129,14 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite logger.Infof("Step 3 : Create Object Bucket Claim with reclaim policy delete") bucketStorageClassName := "rook-smoke-delete-bucket" cobErr := helper.BucketClient.CreateBucketStorageClass(namespace, storeName, bucketStorageClassName, "Delete", region) - require.Nil(s.T(), cobErr) + assert.Nil(s.T(), cobErr) cobcErr := helper.BucketClient.CreateObc(obcName, bucketStorageClassName, bucketname, maxObject, true) - require.Nil(s.T(), cobcErr) + assert.Nil(s.T(), cobcErr) created := utils.Retry(12, 2*time.Second, "OBC is created", func() bool { return helper.BucketClient.CheckOBC(obcName, "bound") }) - require.True(s.T(), created) + assert.True(s.T(), created) logger.Infof("Check if bucket was created") context := k8sh.MakeContext() @@ -153,7 +153,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite time.Sleep(5 * time.Second) } assert.NotEqual(s.T(), i, 4) - require.Equal(s.T(), bucketname, bkt.Name) + assert.Equal(s.T(), bucketname, bkt.Name) logger.Infof("OBC, Secret and ConfigMap created") logger.Infof("Step 4 : Create s3 client") @@ -161,37 +161,37 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName) s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName) s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true) - require.Nil(s.T(), err) + assert.Nil(s.T(), err) logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey) logger.Infof("Step 5 : Put Object on bucket") _, poErr := s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey1, contentType) - require.Nil(s.T(), poErr) + assert.Nil(s.T(), poErr) logger.Infof("Step 6 : Get Object from bucket") read, err := s3client.GetObjectInBucket(bucketname, ObjectKey1) - require.Nil(s.T(), err) - require.Equal(s.T(), ObjBody, read) + assert.Nil(s.T(), err) + assert.Equal(s.T(), ObjBody, read) logger.Infof("Object Created and Retrieved on bucket successfully") logger.Infof("Step 7 : Testing Quota for the OBC") logger.Infof("Adding one more object to the bucket") _, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey2, contentType) - require.Nil(s.T(), poErr) + assert.Nil(s.T(), poErr) logger.Infof("Testing the max object limit") _, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey3, contentType) - require.Error(s.T(), poErr) + assert.Error(s.T(), poErr) logger.Infof("Step 8 : Delete objects on bucket") _, delobjErr := s3client.DeleteObjectInBucket(bucketname, ObjectKey1) - require.Nil(s.T(), delobjErr) + assert.Nil(s.T(), delobjErr) _, delobjErr = s3client.DeleteObjectInBucket(bucketname, ObjectKey2) - require.Nil(s.T(), delobjErr) + assert.Nil(s.T(), delobjErr) logger.Infof("Objects deleted on bucket successfully") logger.Infof("Step 9 : Delete Object Bucket Claim") dobcErr := helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketname, maxObject, true) - require.Nil(s.T(), dobcErr) + assert.Nil(s.T(), dobcErr) logger.Infof("Checking to see if the obc, secret and cm have all been deleted") for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "deleted"); i++ { logger.Infof("(%d) obc deleted check, sleeping for 5 seconds ...", i) @@ -220,27 +220,27 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite } // Test Object StoreCreation on Rook that was installed via helm -func runObjectE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, name string, replicaSize int, deleteStore bool) { +func runObjectE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, name string, replicaSize int, deleteStore bool) { logger.Infof("Object Storage End To End Integration Test - Create Object Store and check if rgw service is Running") - logger.Infof("Running on Rook Cluster %s", namespace) + logger.Infof("Running on Rook Cluster %s", settings.Namespace) logger.Infof("Step 1 : Create Object Store") - err := helper.ObjectClient.Create(namespace, name, int32(replicaSize)) - require.Nil(s.T(), err) + err := helper.ObjectClient.Create(settings.Namespace, name, int32(replicaSize)) + assert.Nil(s.T(), err) logger.Infof("Step 2 : check rook-ceph-rgw service status and count") - require.True(s.T(), k8sh.IsPodInExpectedState("rook-ceph-rgw", namespace, "Running"), + assert.True(s.T(), k8sh.IsPodInExpectedState("rook-ceph-rgw", settings.Namespace, "Running"), "Make sure rook-ceph-rgw is in running state") - assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-rgw", namespace, replicaSize, "Running"), + assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-rgw", settings.Namespace, replicaSize, "Running"), "Make sure all rook-ceph-rgw pods are in Running state") - require.True(s.T(), k8sh.IsServiceUp("rook-ceph-rgw-"+name, namespace)) + assert.True(s.T(), k8sh.IsServiceUp("rook-ceph-rgw-"+name, settings.Namespace)) if deleteStore { logger.Infof("Delete Object Store") - err = helper.ObjectClient.Delete(namespace, name) - require.Nil(s.T(), err) + err = helper.ObjectClient.Delete(settings.Namespace, name) + assert.Nil(s.T(), err) logger.Infof("Done deleting object store") } } diff --git a/tests/integration/ceph_flex_test.go b/tests/integration/ceph_flex_test.go index e00618e70..0937e090e 100644 --- a/tests/integration/ceph_flex_test.go +++ b/tests/integration/ceph_flex_test.go @@ -61,7 +61,7 @@ func TestCephFlexSuite(t *testing.T) { s := new(CephFlexDriverSuite) defer func(s *CephFlexDriverSuite) { - HandlePanics(recover(), s.op, s.T) + HandlePanics(recover(), s.TearDownSuite, s.T) }(s) suite.Run(t, s) } @@ -72,47 +72,45 @@ type CephFlexDriverSuite struct { clusterInfo *client.ClusterInfo bc *clients.BlockOperation kh *utils.K8sHelper - namespace string + installer *installer.CephInstaller + settings *installer.TestCephSettings pvcNameRWO string pvcNameRWX string - op *TestCluster } func (s *CephFlexDriverSuite) SetupSuite() { - s.namespace = "flex-ns" + namespace := "flex-ns" s.pvcNameRWO = "block-persistent-rwo" s.pvcNameRWX = "block-persistent-rwx" - - flexTestCluster := TestCluster{ - namespace: s.namespace, - storeType: "bluestore", - storageClassName: "", - useHelm: false, - usePVC: false, - mons: 1, - rbdMirrorWorkers: 1, - rookCephCleanup: true, - skipOSDCreation: false, - minimalMatrixK8sVersion: flexDriverMinimalTestVersion, - rookVersion: installer.VersionMaster, - cephVersion: installer.OctopusVersion, + s.settings = &installer.TestCephSettings{ + ClusterName: "flex-test", + Namespace: namespace, + OperatorNamespace: installer.SystemNamespace(namespace), + StorageClassName: "", + UseHelm: false, + UsePVC: false, + Mons: 1, + SkipOSDCreation: false, + UseCSI: false, + DirectMountToolbox: true, + RookVersion: installer.VersionMaster, + CephVersion: installer.OctopusVersion, } - s.clusterInfo = client.AdminClusterInfo(s.namespace) - s.op, s.kh = StartTestCluster(s.T, &flexTestCluster) - s.testClient = clients.CreateTestClient(s.kh, s.op.installer.Manifests) + s.clusterInfo = client.AdminClusterInfo(namespace) + s.installer, s.kh = StartTestCluster(s.T, s.settings, flexDriverMinimalTestVersion) + s.testClient = clients.CreateTestClient(s.kh, s.installer.Manifests) s.bc = s.testClient.BlockClient } func (s *CephFlexDriverSuite) AfterTest(suiteName, testName string) { - s.op.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(s.namespace)) + s.installer.CollectOperatorLog(suiteName, testName) } func (s *CephFlexDriverSuite) TestFileSystem() { - useCSI := false preserveFilesystemOnDelete := false - runFileE2ETest(s.testClient, s.kh, s.Suite, s.namespace, "smoke-test-fs", useCSI, preserveFilesystemOnDelete) + runFileE2ETest(s.testClient, s.kh, s.Suite, s.settings, "flex-fs", preserveFilesystemOnDelete) } func (s *CephFlexDriverSuite) TestBlockStorageMountUnMountForStatefulSets() { @@ -127,9 +125,9 @@ func (s *CephFlexDriverSuite) TestBlockStorageMountUnMountForStatefulSets() { logger.Infof("Test case when block persistent volumes are scaled up and down along with StatefulSet") logger.Info("Step 1: Create pool and storageClass") - err := s.testClient.PoolClient.Create(poolName, s.namespace, 1) + err := s.testClient.PoolClient.Create(poolName, s.settings.Namespace, 1) assert.Nil(s.T(), err) - err = s.testClient.BlockClient.CreateStorageClass(false, poolName, storageClassName, reclaimPolicy, s.namespace) + err = s.testClient.BlockClient.CreateStorageClass(false, poolName, storageClassName, reclaimPolicy, s.settings.Namespace) assert.Nil(s.T(), err) logger.Info("Step 2 : Deploy statefulSet with 1X replication") service, statefulset := getBlockStatefulSetAndServiceDefinition(defaultNamespace, statefulSetName, statefulPodsName, storageClassName) @@ -190,11 +188,9 @@ func (s *CephFlexDriverSuite) setupPVCs() { logger.Infof("creating the test PVCs") poolNameRWO := "block-pool-rwo" storageClassNameRWO := "rook-ceph-block-rwo" - systemNamespace := installer.SystemNamespace(s.namespace) // Create PVCs - useCSI := false - err := s.testClient.BlockClient.CreateStorageClassAndPVC(useCSI, defaultNamespace, s.namespace, systemNamespace, poolNameRWO, storageClassNameRWO, "Delete", s.pvcNameRWO, "ReadWriteOnce") + err := s.testClient.BlockClient.CreateStorageClassAndPVC(defaultNamespace, poolNameRWO, storageClassNameRWO, "Delete", s.pvcNameRWO, "ReadWriteOnce") require.Nil(s.T(), err) require.True(s.T(), s.kh.WaitUntilPVCIsBound(defaultNamespace, s.pvcNameRWO), "Make sure PVC is Bound") @@ -207,13 +203,13 @@ func (s *CephFlexDriverSuite) setupPVCs() { require.Nil(s.T(), err) crdName, err := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWO) require.Nil(s.T(), err) - s.kh.IsVolumeResourcePresent(systemNamespace, crdName) + s.kh.IsVolumeResourcePresent(s.settings.OperatorNamespace, crdName) err = s.bc.CreateClientPod(getFlexBlockPodDefinition("setup-block-rwx", s.pvcNameRWX, false)) require.Nil(s.T(), err) crdName, err = s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWX) require.Nil(s.T(), err) - s.kh.IsVolumeResourcePresent(systemNamespace, crdName) + s.kh.IsVolumeResourcePresent(s.settings.OperatorNamespace, crdName) require.True(s.T(), s.kh.IsPodRunning("setup-block-rwo", defaultNamespace), "make sure setup-block-rwo pod is in running state") require.True(s.T(), s.kh.IsPodRunning("setup-block-rwx", defaultNamespace), "make sure setup-block-rwx pod is in running state") @@ -239,9 +235,9 @@ func (s *CephFlexDriverSuite) TearDownSuite() { "setup-block-rwo", "setup-block-rwx", "rwo-block-rw-one", "rwo-block-rw-two", "rwo-block-ro-one", "rwo-block-ro-two", "rwx-block-rw-one", "rwx-block-rw-two", "rwx-block-ro-one", "rwx-block-ro-two") assert.NoError(s.T(), err) - err = s.testClient.BlockClient.DeletePVC(s.namespace, s.pvcNameRWO) + err = s.testClient.BlockClient.DeletePVC(s.settings.Namespace, s.pvcNameRWO) assertNoErrorUnlessNotFound(s.Suite, err) - err = s.testClient.BlockClient.DeletePVC(s.namespace, s.pvcNameRWX) + err = s.testClient.BlockClient.DeletePVC(s.settings.Namespace, s.pvcNameRWX) assertNoErrorUnlessNotFound(s.Suite, err) err = s.testClient.BlockClient.DeleteStorageClass("rook-ceph-block-rwo") assert.NoError(s.T(), err) @@ -251,7 +247,7 @@ func (s *CephFlexDriverSuite) TearDownSuite() { assert.NoError(s.T(), err) err = s.testClient.PoolClient.DeletePool(s.testClient.BlockClient, s.clusterInfo, "block-pool-rwx") assert.NoError(s.T(), err) - s.op.Teardown() + s.installer.UninstallRook() } func (s *CephFlexDriverSuite) TestBlockStorageMountUnMountForDifferentAccessModes() { @@ -267,12 +263,12 @@ func (s *CephFlexDriverSuite) TestBlockStorageMountUnMountForDifferentAccessMode require.Nil(s.T(), err) crdName, err := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWO) assert.Nil(s.T(), err) - assert.True(s.T(), s.kh.IsVolumeResourcePresent(installer.SystemNamespace(s.namespace), crdName), fmt.Sprintf("make sure Volume %s is created", crdName)) + assert.True(s.T(), s.kh.IsVolumeResourcePresent(s.settings.OperatorNamespace, crdName), fmt.Sprintf("make sure Volume %s is created", crdName)) assert.True(s.T(), s.kh.IsPodRunning("rwo-block-rw-one", defaultNamespace), "make sure block-rw-one pod is in running state") crdName, err = s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWX) assert.Nil(s.T(), err) - assert.True(s.T(), s.kh.IsVolumeResourcePresent(installer.SystemNamespace(s.namespace), crdName), fmt.Sprintf("make sure Volume %s is created", crdName)) + assert.True(s.T(), s.kh.IsVolumeResourcePresent(s.settings.OperatorNamespace, crdName), fmt.Sprintf("make sure Volume %s is created", crdName)) assert.True(s.T(), s.kh.IsPodRunning("rwx-block-rw-one", defaultNamespace), "make sure rwx-block-rw-one pod is in running state") logger.Infof("Step 2: Check if previously persisted data is readable from ReadWriteOnce and ReadWriteMany PVC") diff --git a/tests/integration/ceph_helm_test.go b/tests/integration/ceph_helm_test.go index a50420cdb..028a88d50 100644 --- a/tests/integration/ceph_helm_test.go +++ b/tests/integration/ceph_helm_test.go @@ -47,7 +47,7 @@ func TestCephHelmSuite(t *testing.T) { s := new(HelmSuite) defer func(s *HelmSuite) { - HandlePanics(recover(), s.op, s.T) + HandlePanics(recover(), s.TearDownSuite, s.T) }(s) suite.Run(t, s) } @@ -55,56 +55,56 @@ func TestCephHelmSuite(t *testing.T) { type HelmSuite struct { suite.Suite helper *clients.TestClient - kh *utils.K8sHelper - op *TestCluster - namespace string + installer *installer.CephInstaller + settings *installer.TestCephSettings + k8shelper *utils.K8sHelper } -func (hs *HelmSuite) SetupSuite() { - hs.namespace = "helm-ns" - helmTestCluster := TestCluster{ - namespace: hs.namespace, - storeType: "bluestore", - storageClassName: "", - useHelm: true, - usePVC: false, - mons: 1, - rbdMirrorWorkers: 1, - rookCephCleanup: true, - skipOSDCreation: false, - minimalMatrixK8sVersion: helmMinimalTestVersion, - rookVersion: installer.VersionMaster, - cephVersion: installer.NautilusVersion, +func (h *HelmSuite) SetupSuite() { + namespace := "helm-ns" + h.settings = &installer.TestCephSettings{ + Namespace: namespace, + OperatorNamespace: namespace, + StorageClassName: "", + UseHelm: true, + UsePVC: false, + Mons: 1, + UseCSI: true, + SkipOSDCreation: false, + EnableAdmissionController: true, + EnableDiscovery: true, + RookVersion: installer.VersionMaster, + CephVersion: installer.OctopusVersion, } - hs.op, hs.kh = StartTestCluster(hs.T, &helmTestCluster) - hs.helper = clients.CreateTestClient(hs.kh, hs.op.installer.Manifests) + h.installer, h.k8shelper = StartTestCluster(h.T, h.settings, helmMinimalTestVersion) + h.helper = clients.CreateTestClient(h.k8shelper, h.installer.Manifests) } -func (hs *HelmSuite) TearDownSuite() { - hs.op.Teardown() +func (h *HelmSuite) TearDownSuite() { + h.installer.UninstallRook() } -func (hs *HelmSuite) AfterTest(suiteName, testName string) { - hs.op.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(hs.namespace)) +func (h *HelmSuite) AfterTest(suiteName, testName string) { + h.installer.CollectOperatorLog(suiteName, testName) } // Test to make sure all rook components are installed and Running -func (hs *HelmSuite) TestARookInstallViaHelm() { - checkIfRookClusterIsInstalled(hs.Suite, hs.kh, hs.namespace, hs.namespace, 1) +func (h *HelmSuite) TestARookInstallViaHelm() { + checkIfRookClusterIsInstalled(h.Suite, h.k8shelper, h.settings.Namespace, h.settings.Namespace, 1) } // Test BlockCreation on Rook that was installed via Helm -func (hs *HelmSuite) TestBlockStoreOnRookInstalledViaHelm() { - runBlockCSITestLite(hs.helper, hs.kh, hs.Suite, hs.namespace, hs.namespace, hs.op.installer.CephVersion) +func (h *HelmSuite) TestBlockStoreOnRookInstalledViaHelm() { + runBlockCSITestLite(h.helper, h.k8shelper, h.Suite, h.settings) } // Test File System Creation on Rook that was installed via helm -func (hs *HelmSuite) TestFileStoreOnRookInstalledViaHelm() { - runFileE2ETestLite(hs.helper, hs.kh, hs.Suite, hs.namespace, "testfs") +func (h *HelmSuite) TestFileStoreOnRookInstalledViaHelm() { + runFileE2ETestLite(h.helper, h.k8shelper, h.Suite, h.settings, "testfs") } // Test Object StoreCreation on Rook that was installed via helm -func (hs *HelmSuite) TestObjectStoreOnRookInstalledViaHelm() { - runObjectE2ETestLite(hs.helper, hs.kh, hs.Suite, hs.namespace, "default", 3, true) +func (h *HelmSuite) TestObjectStoreOnRookInstalledViaHelm() { + runObjectE2ETestLite(h.helper, h.k8shelper, h.Suite, h.settings, "default", 3, true) } diff --git a/tests/integration/ceph_mgr_test.go b/tests/integration/ceph_mgr_test.go index 35b849357..cc5abd077 100644 --- a/tests/integration/ceph_mgr_test.go +++ b/tests/integration/ceph_mgr_test.go @@ -55,15 +55,16 @@ func TestCephMgrSuite(t *testing.T) { s := new(CephMgrSuite) defer func(s *CephMgrSuite) { - HandlePanics(recover(), s.cluster, s.T) + HandlePanics(recover(), s.TearDownSuite, s.T) }(s) suite.Run(t, s) } type CephMgrSuite struct { suite.Suite - cluster *TestCluster + settings *installer.TestCephSettings k8sh *utils.K8sHelper + installer *installer.CephInstaller namespace string } @@ -87,46 +88,44 @@ type service struct { Status serviceStatus } -func (suite *CephMgrSuite) SetupSuite() { - suite.namespace = "mgr-ns" +func (s *CephMgrSuite) SetupSuite() { + s.namespace = "mgr-ns" - mgrTestCluster := TestCluster{ - clusterName: suite.namespace, - namespace: suite.namespace, - storeType: "bluestore", - storageClassName: "", - useHelm: false, - usePVC: false, - mons: 1, - rbdMirrorWorkers: 0, - rookCephCleanup: true, - skipOSDCreation: true, - minimalMatrixK8sVersion: cephMasterSuiteMinimalTestVersion, - rookVersion: installer.VersionMaster, - cephVersion: installer.MasterVersion, + s.settings = &installer.TestCephSettings{ + ClusterName: s.namespace, + OperatorNamespace: installer.SystemNamespace(s.namespace), + Namespace: s.namespace, + StorageClassName: "", + UseHelm: false, + UsePVC: false, + Mons: 1, + UseCSI: true, + SkipOSDCreation: true, + RookVersion: installer.VersionMaster, + CephVersion: installer.MasterVersion, } - suite.cluster, suite.k8sh = StartTestCluster(suite.T, &mgrTestCluster) - suite.waitForOrchestrationModule() + s.installer, s.k8sh = StartTestCluster(s.T, s.settings, cephMasterSuiteMinimalTestVersion) + s.waitForOrchestrationModule() } -func (suite *CephMgrSuite) AfterTest(suiteName, testName string) { - suite.cluster.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(suite.namespace)) +func (s *CephMgrSuite) AfterTest(suiteName, testName string) { + s.installer.CollectOperatorLog(suiteName, testName) } -func (suite *CephMgrSuite) TearDownSuite() { - suite.cluster.Teardown() +func (s *CephMgrSuite) TearDownSuite() { + s.installer.UninstallRook() } -func (suite *CephMgrSuite) execute(command []string) (error, string) { +func (s *CephMgrSuite) execute(command []string) (error, string) { orchCommand := append([]string{"orch"}, command...) - return suite.cluster.installer.Execute("ceph", orchCommand, suite.namespace) + return s.installer.Execute("ceph", orchCommand, s.namespace) } -func (suite *CephMgrSuite) waitForOrchestrationModule() { +func (s *CephMgrSuite) waitForOrchestrationModule() { var err error for timeout := 0; timeout < 30; timeout++ { - err, output := suite.execute([]string{"status"}) + err, output := s.execute([]string{"status"}) logger.Infof("%s", output) if err == nil { logger.Info("Rook Toolbox ready to execute commands") @@ -135,30 +134,30 @@ func (suite *CephMgrSuite) waitForOrchestrationModule() { time.Sleep(2 * time.Second) } logger.Error("Giving up waiting for Rook Toolbox to be ready") - assert.Nil(suite.T(), err) + assert.Nil(s.T(), err) } -func (suite *CephMgrSuite) TestDeviceLs() { +func (s *CephMgrSuite) TestDeviceLs() { logger.Info("Testing .... ") - err, device_list := suite.execute([]string{"device", "ls"}) - assert.Nil(suite.T(), err) + err, device_list := s.execute([]string{"device", "ls"}) + assert.Nil(s.T(), err) logger.Infof("output = %s", device_list) } -func (suite *CephMgrSuite) TestStatus() { +func (s *CephMgrSuite) TestStatus() { logger.Info("Testing .... ") - err, status := suite.execute([]string{"status"}) - assert.Nil(suite.T(), err) + err, status := s.execute([]string{"status"}) + assert.Nil(s.T(), err) logger.Infof("output = %s", status) - assert.Equal(suite.T(), status, "Backend: rook\nAvailable: True") + assert.Equal(s.T(), status, "Backend: rook\nAvailable: True") } -func (suite *CephMgrSuite) TestHostLs() { +func (s *CephMgrSuite) TestHostLs() { logger.Info("Testing .... ") // Get the orchestrator hosts - err, output := suite.execute([]string{"host", "ls", "json"}) - assert.Nil(suite.T(), err) + err, output := s.execute([]string{"host", "ls", "json"}) + assert.Nil(s.T(), err) logger.Infof("output = %s", output) hosts := []byte(output) @@ -166,7 +165,7 @@ func (suite *CephMgrSuite) TestHostLs() { err = json.Unmarshal(hosts, &hostsList) if err != nil { - assert.Nil(suite.T(), err) + assert.Nil(s.T(), err) } var hostOutput []string @@ -176,8 +175,8 @@ func (suite *CephMgrSuite) TestHostLs() { sort.Strings(hostOutput) // get the k8s nodes - nodes, err := k8sutil.GetNodeHostNames(suite.k8sh.Clientset) - assert.Nil(suite.T(), err) + nodes, err := k8sutil.GetNodeHostNames(s.k8sh.Clientset) + assert.Nil(s.T(), err) k8sNodes := make([]string, 0, len(nodes)) for k := range nodes { @@ -186,21 +185,21 @@ func (suite *CephMgrSuite) TestHostLs() { sort.Strings(k8sNodes) // nodes and hosts must be the same - assert.Equal(suite.T(), hostOutput, k8sNodes) + assert.Equal(s.T(), hostOutput, k8sNodes) } -func (suite *CephMgrSuite) TestCreateOSD() { +func (s *CephMgrSuite) TestCreateOSD() { logger.Info("Testing .... ") // Get the first available device - err, deviceList := suite.execute([]string{"device", "ls", "--format", "json"}) - assert.Nil(suite.T(), err) + err, deviceList := s.execute([]string{"device", "ls", "--format", "json"}) + assert.Nil(s.T(), err) logger.Infof("output = %s", deviceList) inventory := make([]map[string]interface{}, 0) err = json.Unmarshal([]byte(deviceList), &inventory) - assert.Nil(suite.T(), err) + assert.Nil(s.T(), err) selectedNode := "" selectedDevice := "" @@ -216,40 +215,40 @@ func (suite *CephMgrSuite) TestCreateOSD() { break } } - assert.NotEqual(suite.T(), "", selectedDevice, "No devices available to create test OSD") - assert.NotEqual(suite.T(), "", selectedNode, "No nodes available to create test OSD") + assert.NotEqual(s.T(), "", selectedDevice, "No devices available to create test OSD") + assert.NotEqual(s.T(), "", selectedNode, "No nodes available to create test OSD") if selectedDevice == "" || selectedNode == "" { return } // Create the OSD - err, output := suite.execute([]string{"daemon", "add", "osd", fmt.Sprintf("%s:%s", selectedNode, selectedDevice)}) + err, output := s.execute([]string{"daemon", "add", "osd", fmt.Sprintf("%s:%s", selectedNode, selectedDevice)}) - assert.Nil(suite.T(), err) + assert.Nil(s.T(), err) logger.Infof("output = %s", output) - err = suite.k8sh.WaitForPodCount("app=rook-ceph-osd", suite.namespace, 1) - assert.Nil(suite.T(), err) + err = s.k8sh.WaitForPodCount("app=rook-ceph-osd", s.namespace, 1) + assert.Nil(s.T(), err) } -func (suite *CephMgrSuite) TestServiceLs() { +func (s *CephMgrSuite) TestServiceLs() { logger.Info("Testing .... ") - err, output := suite.execute([]string{"ls", "--format", "json"}) - assert.Nil(suite.T(), err) + err, output := s.execute([]string{"ls", "--format", "json"}) + assert.Nil(s.T(), err) logger.Infof("output = %s", output) services := []byte(output) var servicesList []service err = json.Unmarshal(services, &servicesList) - assert.Nil(suite.T(), err) + assert.Nil(s.T(), err) for _, svc := range servicesList { labelFilter := fmt.Sprintf("app=rook-ceph-%s", svc.ServiceName) - k8sPods, err := k8sutil.PodsRunningWithLabel(suite.k8sh.Clientset, suite.namespace, labelFilter) + k8sPods, err := k8sutil.PodsRunningWithLabel(s.k8sh.Clientset, s.namespace, labelFilter) logger.Infof("Service: %+v", svc) logger.Infof("k8s pods for svc %q: %d", svc.ServiceName, k8sPods) - assert.Nil(suite.T(), err) - assert.Equal(suite.T(), svc.Status.Running, k8sPods, fmt.Sprintf("Wrong number of pods for kind of service <%s>", svc.ServiceName)) + assert.Nil(s.T(), err) + assert.Equal(s.T(), svc.Status.Running, k8sPods, fmt.Sprintf("Wrong number of pods for kind of service <%s>", svc.ServiceName)) } } diff --git a/tests/integration/ceph_multi_cluster_test.go b/tests/integration/ceph_multi_cluster_test.go index 7e5e2af2f..bd552453a 100644 --- a/tests/integration/ceph_multi_cluster_test.go +++ b/tests/integration/ceph_multi_cluster_test.go @@ -17,16 +17,14 @@ limitations under the License. package integration import ( - "fmt" "path/filepath" "testing" - "time" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -58,180 +56,128 @@ func TestCephMultiClusterDeploySuite(t *testing.T) { s := new(MultiClusterDeploySuite) defer func(s *MultiClusterDeploySuite) { - HandlePanics(recover(), s.op, s.T) + HandlePanics(recover(), s.TearDownSuite, s.T) }(s) suite.Run(t, s) } type MultiClusterDeploySuite struct { suite.Suite - testClient *clients.TestClient - k8sh *utils.K8sHelper - namespace1 string - namespace2 string - op *MCTestOperations - poolName string + testClient *clients.TestClient + k8sh *utils.K8sHelper + settings *installer.TestCephSettings + externalManifests installer.CephManifests + installer *installer.CephInstaller + coreToolbox string + externalToolbox string + poolName string } // Deploy Multiple Rook clusters -func (mrc *MultiClusterDeploySuite) SetupSuite() { - mrc.poolName = "multi-cluster-pool1" - mrc.namespace1 = "mrc-n1" - mrc.namespace2 = "mrc-n2" - - mrc.op, mrc.k8sh = NewMCTestOperations(mrc.T, mrc.namespace1, mrc.namespace2) - mrc.testClient = clients.CreateTestClient(mrc.k8sh, mrc.op.installer.Manifests) - mrc.createPools() -} - -func (mrc *MultiClusterDeploySuite) AfterTest(suiteName, testName string) { - mrc.op.installer.CollectOperatorLog(suiteName, testName, mrc.op.systemNamespace) -} - -func (mrc *MultiClusterDeploySuite) createPools() { - // create a test pool in each cluster so that we get some PGs - logger.Infof("Creating pool %s", mrc.poolName) - err := mrc.testClient.PoolClient.Create(mrc.poolName, mrc.namespace1, 1) - require.Nil(mrc.T(), err) -} - -func (mrc *MultiClusterDeploySuite) deletePools() { - // create a test pool in each cluster so that we get some PGs - logger.Infof("Deleting pool %s", mrc.poolName) - clusterInfo := client.AdminClusterInfo(mrc.namespace1) - if err := mrc.testClient.PoolClient.DeletePool(mrc.testClient.BlockClient, clusterInfo, mrc.poolName); err != nil { - logger.Errorf("failed to delete pool %q. %v", mrc.poolName, err) - } else { - logger.Infof("deleted pool %q", mrc.poolName) +func (s *MultiClusterDeploySuite) SetupSuite() { + s.poolName = "multi-cluster-pool1" + coreNamespace := "multi-core" + s.settings = &installer.TestCephSettings{ + ClusterName: "multi-cluster", + Namespace: coreNamespace, + OperatorNamespace: installer.SystemNamespace(coreNamespace), + StorageClassName: "manual", + UsePVC: installer.UsePVC(), + Mons: 1, + UseCSI: true, + MultipleMgrs: true, + EnableAdmissionController: true, + RookVersion: installer.VersionMaster, + CephVersion: installer.NautilusVersion, } -} - -func (mrc *MultiClusterDeploySuite) TearDownSuite() { - mrc.deletePools() - mrc.op.Teardown() -} - -// Test to make sure all rook components are installed and Running -func (mrc *MultiClusterDeploySuite) TestInstallingMultipleRookClusters() { - // Check if Rook cluster 1 is deployed successfully - checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace1, 1) - checkIfRookClusterIsHealthy(mrc.Suite, mrc.testClient, mrc.namespace1) - - // Check if Rook external cluster is deployed successfully - // Checking health status is enough to validate the connection - checkIfRookClusterIsHealthy(mrc.Suite, mrc.testClient, mrc.namespace2) -} - -// MCTestOperations struct for handling panic and test suite tear down -type MCTestOperations struct { - installer *installer.CephInstaller - kh *utils.K8sHelper - T func() *testing.T - namespace1 string - namespace2 string - systemNamespace string - storageClassName string - testOverPVC bool -} - -// NewMCTestOperations creates new instance of TestCluster struct -func NewMCTestOperations(t func() *testing.T, namespace1 string, namespace2 string) (*MCTestOperations, *utils.K8sHelper) { - - kh, err := utils.CreateK8sHelper(t) - require.NoError(t(), err) - checkIfShouldRunForMinimalTestMatrix(t, kh, multiClusterMinimalTestVersion) - - cleanupHost := false - i := installer.NewCephInstaller(t, kh.Clientset, false, "", installer.VersionMaster, installer.NautilusVersion, cleanupHost) - - op := &MCTestOperations{i, kh, t, namespace1, namespace2, installer.SystemNamespace(namespace1), "", false} - if kh.VersionAtLeast("v1.13.0") { - op.testOverPVC = true - op.storageClassName = "manual" + externalSettings := &installer.TestCephSettings{ + IsExternal: true, + ClusterName: "test-external", + Namespace: "multi-external", + OperatorNamespace: s.settings.OperatorNamespace, + RookVersion: s.settings.RookVersion, + UseCSI: true, } - op.Setup() - return op, kh -} + s.externalManifests = installer.NewCephManifests(externalSettings) -// Setup is wrapper for setting up multiple rook clusters. -func (o MCTestOperations) Setup() { - var err error - if o.testOverPVC { - root, err := utils.FindRookRoot() - require.NoError(o.T(), err, "failed to get rook root") - cmdArgs := utils.CommandArgs{Command: filepath.Join(root, localPathPVCmd), - CmdArgs: []string{installer.TestScratchDevice()}} - cmdOut := utils.ExecuteCommand(cmdArgs) - require.NoError(o.T(), cmdOut.Err) + k8sh, err := utils.CreateK8sHelper(s.T) + assert.NoError(s.T(), err) + if !k8sh.VersionAtLeast("v1.16.0") { + s.T().Skip("requires at least k8s 1.16, no need to run on older versions") } - err = o.installer.CreateCephOperator(installer.SystemNamespace(o.namespace1)) - require.NoError(o.T(), err) - - require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-operator", o.systemNamespace, "Running"), - "Make sure rook-operator is in running state") - - require.True(o.T(), o.kh.IsPodInExpectedState("rook-discover", o.systemNamespace, "Running"), - "Make sure rook-discover is in running state") - - time.Sleep(10 * time.Second) - - // start the two clusters in parallel - logger.Infof("starting two clusters, one traditional and one external") - err = o.startCluster(o.namespace1, "bluestore") - require.NoError(o.T(), err) - // Wait for the monitors to be up so that we can fetch the configmap and secrets - require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-mon", o.namespace1, "Running"), - "Make sure rook-ceph-mon is in running state") + // Start the core storage cluster + s.setupMultiClusterCore() + s.createPools() + // Start the external cluster that will connect to the core cluster // create an external cluster - err = o.startExternalCluster(o.namespace2) - require.NoError(o.T(), err) + s.startExternalCluster() logger.Infof("finished starting clusters") } -// Teardown is a wrapper for tearDown after suite -func (o MCTestOperations) Teardown() { - o.installer.UninstallRookFromMultipleNS(installer.SystemNamespace(o.namespace1), o.namespace1, o.namespace2) +func (s *MultiClusterDeploySuite) AfterTest(suiteName, testName string) { + s.installer.CollectOperatorLog(suiteName, testName) } -func (o MCTestOperations) startCluster(namespace, store string) error { - logger.Infof("starting cluster %s", namespace) - multipleManagers := true - err := o.installer.CreateRookCluster(namespace, o.systemNamespace, store, o.testOverPVC, o.storageClassName, - cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, multipleManagers, false, false, installer.NautilusVersion) +func (s *MultiClusterDeploySuite) createPools() { + // create a test pool in each cluster so that we get some PGs + logger.Infof("Creating pool %s", s.poolName) + err := s.testClient.PoolClient.Create(s.poolName, s.settings.Namespace, 1) + require.Nil(s.T(), err) +} + +func (s *MultiClusterDeploySuite) deletePools() { + // create a test pool in each cluster so that we get some PGs + clusterInfo := client.AdminClusterInfo(s.settings.Namespace) + if err := s.testClient.PoolClient.DeletePool(s.testClient.BlockClient, clusterInfo, s.poolName); err != nil { + logger.Errorf("failed to delete pool %q. %v", s.poolName, err) + } else { + logger.Infof("deleted pool %q", s.poolName) + } +} + +func (s *MultiClusterDeploySuite) TearDownSuite() { + s.deletePools() + s.installer.UninstallRookFromMultipleNS(s.installer.Manifests, s.externalManifests) +} + +// Test to make sure all rook components are installed and Running +func (s *MultiClusterDeploySuite) TestInstallingMultipleRookClusters() { + // Check if Rook cluster 1 is deployed successfully + client.RunAllCephCommandsInToolboxPod = s.coreToolbox + checkIfRookClusterIsInstalled(s.Suite, s.k8sh, s.settings.OperatorNamespace, s.settings.Namespace, 1) + checkIfRookClusterIsHealthy(s.Suite, s.testClient, s.settings.Namespace) + + // Check if Rook external cluster is deployed successfully + // Checking health status is enough to validate the connection + client.RunAllCephCommandsInToolboxPod = s.externalToolbox + checkIfRookClusterIsHealthy(s.Suite, s.testClient, s.externalManifests.Settings().Namespace) +} + +// Setup is wrapper for setting up multiple rook clusters. +func (s *MultiClusterDeploySuite) setupMultiClusterCore() { + root, err := utils.FindRookRoot() + require.NoError(s.T(), err, "failed to get rook root") + cmdArgs := utils.CommandArgs{Command: filepath.Join(root, localPathPVCmd), + CmdArgs: []string{installer.TestScratchDevice()}} + cmdOut := utils.ExecuteCommand(cmdArgs) + require.NoError(s.T(), cmdOut.Err) + + s.installer, s.k8sh = StartTestCluster(s.T, s.settings, multiClusterMinimalTestVersion) + s.testClient = clients.CreateTestClient(s.k8sh, s.installer.Manifests) + s.coreToolbox = client.RunAllCephCommandsInToolboxPod +} + +func (s *MultiClusterDeploySuite) startExternalCluster() { + err := s.installer.CreateRookExternalCluster(s.externalManifests) if err != nil { - o.T().Fail() - o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace) - return fmt.Errorf("failed to create cluster %s. %+v", namespace, err) + s.T().Fail() + s.installer.GatherAllRookLogs(s.T().Name(), s.externalManifests.Settings().Namespace) + require.NoError(s.T(), err) } - if err := o.installer.CreateRookToolbox(namespace); err != nil { - o.T().Fail() - o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace) - return fmt.Errorf("failed to create toolbox for %s. %+v", namespace, err) - } - logger.Infof("succeeded starting cluster %s", namespace) - return nil -} - -func (o MCTestOperations) startExternalCluster(namespace string) error { - logger.Infof("starting external cluster %q", namespace) - err := o.installer.CreateRookExternalCluster(namespace, o.namespace1) - if err != nil { - o.T().Fail() - o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace) - return fmt.Errorf("failed to create external cluster %s. %+v", namespace, err) - } - - logger.Infof("running toolbox on namespace %q", namespace) - if err := o.installer.CreateRookToolbox(namespace); err != nil { - o.T().Fail() - o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace) - return fmt.Errorf("failed to create toolbox for %s. %+v", namespace, err) - } - - logger.Infof("succeeded starting external cluster %s", namespace) - return nil + s.externalToolbox = client.RunAllCephCommandsInToolboxPod + logger.Infof("succeeded starting external cluster %s", s.externalManifests.Settings().Namespace) } diff --git a/tests/integration/ceph_smoke_test.go b/tests/integration/ceph_smoke_test.go index fa6981343..98c7a985c 100644 --- a/tests/integration/ceph_smoke_test.go +++ b/tests/integration/ceph_smoke_test.go @@ -72,7 +72,7 @@ func TestCephSmokeSuite(t *testing.T) { s := new(SmokeSuite) defer func(s *SmokeSuite) { - HandlePanics(recover(), s.op, s.T) + HandlePanics(recover(), s.TearDownSuite, s.T) }(s) suite.Run(t, s) } @@ -80,83 +80,84 @@ func TestCephSmokeSuite(t *testing.T) { type SmokeSuite struct { suite.Suite helper *clients.TestClient - op *TestCluster + settings *installer.TestCephSettings + installer *installer.CephInstaller k8sh *utils.K8sHelper - namespace string } -func (suite *SmokeSuite) SetupSuite() { - suite.namespace = "smoke-ns" - smokeTestCluster := TestCluster{ - namespace: suite.namespace, - storeType: "bluestore", - storageClassName: installer.StorageClassName(), - useHelm: false, - usePVC: installer.UsePVC(), - mons: 3, - rbdMirrorWorkers: 1, - rookCephCleanup: true, - skipOSDCreation: false, - minimalMatrixK8sVersion: smokeSuiteMinimalTestVersion, - rookVersion: installer.VersionMaster, - cephVersion: installer.OctopusVersion, +func (s *SmokeSuite) SetupSuite() { + namespace := "smoke-ns" + s.settings = &installer.TestCephSettings{ + ClusterName: "smoke-cluster", + Namespace: namespace, + OperatorNamespace: installer.SystemNamespace(namespace), + StorageClassName: installer.StorageClassName(), + UseHelm: false, + UsePVC: installer.UsePVC(), + Mons: 3, + SkipOSDCreation: false, + UseCSI: true, + EnableAdmissionController: true, + UseCrashPruner: true, + RookVersion: installer.VersionMaster, + CephVersion: installer.OctopusVersion, } - suite.op, suite.k8sh = StartTestCluster(suite.T, &smokeTestCluster) - suite.helper = clients.CreateTestClient(suite.k8sh, suite.op.installer.Manifests) + s.installer, s.k8sh = StartTestCluster(s.T, s.settings, smokeSuiteMinimalTestVersion) + s.helper = clients.CreateTestClient(s.k8sh, s.installer.Manifests) } -func (suite *SmokeSuite) AfterTest(suiteName, testName string) { - suite.op.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(suite.namespace)) +func (s *SmokeSuite) AfterTest(suiteName, testName string) { + s.installer.CollectOperatorLog(suiteName, testName) } -func (suite *SmokeSuite) TearDownSuite() { - suite.op.Teardown() +func (s *SmokeSuite) TearDownSuite() { + s.installer.UninstallRook() } -func (suite *SmokeSuite) TestBlockStorage_SmokeTest() { - runBlockCSITest(suite.helper, suite.k8sh, suite.Suite, suite.namespace) +func (s *SmokeSuite) TestBlockStorage_SmokeTest() { + runBlockCSITest(s.helper, s.k8sh, s.Suite, s.settings.Namespace) } -func (suite *SmokeSuite) TestFileStorage_SmokeTest() { - useCSI := true +func (s *SmokeSuite) TestFileStorage_SmokeTest() { preserveFilesystemOnDelete := true - runFileE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace, "smoke-test-fs", useCSI, preserveFilesystemOnDelete) + runFileE2ETest(s.helper, s.k8sh, s.Suite, s.settings, "smoke-test-fs", preserveFilesystemOnDelete) } -func (suite *SmokeSuite) TestObjectStorage_SmokeTest() { - if !utils.IsPlatformOpenShift() { - runObjectE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace) +func (s *SmokeSuite) TestObjectStorage_SmokeTest() { + if utils.IsPlatformOpenShift() { + s.T().Skip("object store tests skipped on openshift") } + runObjectE2ETest(s.helper, s.k8sh, s.Suite, s.settings.Namespace) } // Test to make sure all rook components are installed and Running -func (suite *SmokeSuite) TestARookClusterInstallation_SmokeTest() { - checkIfRookClusterIsInstalled(suite.Suite, suite.k8sh, installer.SystemNamespace(suite.namespace), suite.namespace, 3) +func (s *SmokeSuite) TestARookClusterInstallation_SmokeTest() { + checkIfRookClusterIsInstalled(s.Suite, s.k8sh, s.settings.OperatorNamespace, s.settings.Namespace, 3) } // Smoke Test for Mon failover - Test check the following operations for the Mon failover in order // Delete mon pod, Wait for new mon pod -func (suite *SmokeSuite) TestMonFailover() { +func (s *SmokeSuite) TestMonFailover() { ctx := context.TODO() logger.Infof("Mon Failover Smoke Test") - deployments, err := suite.getNonCanaryMonDeployments() - require.Nil(suite.T(), err) - require.Equal(suite.T(), 3, len(deployments)) + deployments, err := s.getNonCanaryMonDeployments() + require.NoError(s.T(), err) + require.Equal(s.T(), 3, len(deployments)) monToKill := deployments[0].Name logger.Infof("Killing mon %s", monToKill) propagation := metav1.DeletePropagationForeground delOptions := &metav1.DeleteOptions{PropagationPolicy: &propagation} - err = suite.k8sh.Clientset.AppsV1().Deployments(suite.namespace).Delete(ctx, monToKill, *delOptions) - require.Nil(suite.T(), err) + err = s.k8sh.Clientset.AppsV1().Deployments(s.settings.Namespace).Delete(ctx, monToKill, *delOptions) + require.NoError(s.T(), err) // Wait for the health check to start a new monitor originalMonDeleted := false for i := 0; i < 30; i++ { - deployments, err := suite.getNonCanaryMonDeployments() - require.Nil(suite.T(), err) + deployments, err := s.getNonCanaryMonDeployments() + require.NoError(s.T(), err) // Make sure the old mon is not still alive foundOldMon := false @@ -188,31 +189,31 @@ func (suite *SmokeSuite) TestMonFailover() { return } - assert.Equal(suite.T(), 2, len(deployments)) + assert.Equal(s.T(), 2, len(deployments)) } time.Sleep(5 * time.Second) } - require.Fail(suite.T(), "giving up waiting for a new monitor") + require.Fail(s.T(), "giving up waiting for a new monitor") } // Smoke Test for pool Resizing -func (suite *SmokeSuite) TestPoolResize() { +func (s *SmokeSuite) TestPoolResize() { ctx := context.TODO() logger.Infof("Pool Resize Smoke Test") poolName := "testpool" - err := suite.helper.PoolClient.Create(poolName, suite.namespace, 1) - require.Nil(suite.T(), err) + err := s.helper.PoolClient.Create(poolName, s.settings.Namespace, 1) + require.NoError(s.T(), err) poolFound := false - clusterInfo := client.AdminClusterInfo(suite.namespace) + clusterInfo := client.AdminClusterInfo(s.settings.Namespace) // Wait for pool to appear for i := 0; i < 10; i++ { - pools, err := suite.helper.PoolClient.ListCephPools(clusterInfo) - require.Nil(suite.T(), err) + pools, err := s.helper.PoolClient.ListCephPools(clusterInfo) + require.NoError(s.T(), err) for _, p := range pools { if p.Name != poolName { continue @@ -227,24 +228,24 @@ func (suite *SmokeSuite) TestPoolResize() { time.Sleep(2 * time.Second) } - require.Equal(suite.T(), true, poolFound, "pool not found") + require.Equal(s.T(), true, poolFound, "pool not found") - err = suite.helper.PoolClient.Update(poolName, suite.namespace, 2) - require.Nil(suite.T(), err) + err = s.helper.PoolClient.Update(poolName, s.settings.Namespace, 2) + require.NoError(s.T(), err) poolResized := false // Wait for pool resize to happen for i := 0; i < 10; i++ { - details, err := suite.helper.PoolClient.GetCephPoolDetails(clusterInfo, poolName) - require.Nil(suite.T(), err) + details, err := s.helper.PoolClient.GetCephPoolDetails(clusterInfo, poolName) + require.NoError(s.T(), err) if details.Size > 1 { logger.Infof("pool %s size was updated", poolName) - require.Equal(suite.T(), 2, int(details.Size)) + require.Equal(s.T(), 2, int(details.Size)) poolResized = true // resize the pool back to 1 to avoid hangs around not having enough OSDs to satisfy rbd - err = suite.helper.PoolClient.Update(poolName, suite.namespace, 1) - require.Nil(suite.T(), err) + err = s.helper.PoolClient.Update(poolName, s.settings.Namespace, 1) + require.NoError(s.T(), err) } else if poolResized && details.Size == 1 { logger.Infof("pool resized back to 1") break @@ -255,45 +256,45 @@ func (suite *SmokeSuite) TestPoolResize() { time.Sleep(2 * time.Second) } - require.Equal(suite.T(), true, poolResized, fmt.Sprintf("pool %s not found", poolName)) + require.Equal(s.T(), true, poolResized, fmt.Sprintf("pool %s not found", poolName)) // Verify the Kubernetes Secret has been created (bootstrap peer token) - pool, err := suite.k8sh.RookClientset.CephV1().CephBlockPools(suite.namespace).Get(ctx, poolName, metav1.GetOptions{}) - assert.NoError(suite.T(), err) + pool, err := s.k8sh.RookClientset.CephV1().CephBlockPools(s.settings.Namespace).Get(ctx, poolName, metav1.GetOptions{}) + assert.NoError(s.T(), err) if pool.Spec.Mirroring.Enabled { secretName := pool.Status.Info[oppool.RBDMirrorBootstrapPeerSecretName] - assert.NotEmpty(suite.T(), secretName) + assert.NotEmpty(s.T(), secretName) // now fetch the secret which contains the bootstrap peer token - s, err := suite.k8sh.Clientset.CoreV1().Secrets(suite.namespace).Get(ctx, secretName, metav1.GetOptions{}) - require.Nil(suite.T(), err) - assert.NotEmpty(suite.T(), s.Data["token"]) + secret, err := s.k8sh.Clientset.CoreV1().Secrets(s.settings.Namespace).Get(ctx, secretName, metav1.GetOptions{}) + require.NoError(s.T(), err) + assert.NotEmpty(s.T(), secret.Data["token"]) // Once we have a scenario with another Ceph cluster - needs to be added in the MultiCluster suite // We would need to add a bootstrap peer token following the below procedure // bootstrapSecretName := "bootstrap-peer-token" // token := "eyJmc2lkIjoiYzZiMDg3ZjItNzgyOS00ZGJiLWJjZmMtNTNkYzM0ZTBiMzVkIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBV1lsWmZVQ1Q2RGhBQVBtVnAwbGtubDA5YVZWS3lyRVV1NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMTExLjEwOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTA6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjEyOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTI6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjExOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTE6Njc4OV0ifQ==" - // s = oppool.GenerateBootstrapPeerSecret(bootstrapSecretName, suite.namespace, string(pool.GetUID()), []byte(token)) - // s, err = suite.k8sh.Clientset.CoreV1().Secrets(suite.namespace).Create(s) - // require.Nil(suite.T(), err, err.Error()) + // s = oppool.GenerateBootstrapPeerSecret(bootstrapSecretName, s.settings.Namespace, string(pool.GetUID()), []byte(token)) + // s, err = s.k8sh.Clientset.CoreV1().Secrets(s.settings.Namespace).Create(s) + // require.Nil(s.T(), err, err.Error()) // // update the ceph block pool cr // pool.Spec.Mirrored.PeersSecretNames = append(pool.Spec.Mirrored.PeersSecretNames, bootstrapSecretName) - // _, err = suite.k8sh.RookClientset.CephV1().CephBlockPools(suite.namespace).Update(pool) - // require.Nil(suite.T(), err, err.Error()) + // _, err = s.k8sh.RookClientset.CephV1().CephBlockPools(s.settings.Namespace).Update(pool) + // require.Nil(s.T(), err, err.Error()) - // mirrorInfo, err := client.PrintPoolMirroringInfo(suite.k8sh.MakeContext(), clusterInfo, poolName) - // require.Nil(suite.T(), err, err.Error()) - // assert.Equal(suite.T(), "image", mirrorInfo.Mode) - // assert.Equal(suite.T(), 1, len(mirrorInfo.Peers)) + // mirrorInfo, err := client.PrintPoolMirroringInfo(s.k8sh.MakeContext(), clusterInfo, poolName) + // require.Nil(s.T(), err, err.Error()) + // assert.Equal(s.T(), "image", mirrorInfo.Mode) + // assert.Equal(s.T(), 1, len(mirrorInfo.Peers)) } // clean up the pool - err = suite.helper.PoolClient.DeletePool(suite.helper.BlockClient, clusterInfo, poolName) - assert.NoError(suite.T(), err) + err = s.helper.PoolClient.DeletePool(s.helper.BlockClient, clusterInfo, poolName) + assert.NoError(s.T(), err) } // Smoke Test for Client CRD -func (suite *SmokeSuite) TestCreateClient() { +func (s *SmokeSuite) TestCreateClient() { logger.Infof("Create Client Smoke Test") clientName := "client1" @@ -302,14 +303,14 @@ func (suite *SmokeSuite) TestCreateClient() { "mgr": "allow rwx", "osd": "allow rwx", } - clusterInfo := client.AdminClusterInfo(suite.namespace) - err := suite.helper.UserClient.Create(clientName, suite.namespace, caps) - require.Nil(suite.T(), err) + clusterInfo := client.AdminClusterInfo(s.settings.Namespace) + err := s.helper.UserClient.Create(clientName, s.settings.Namespace, caps) + require.NoError(s.T(), err) clientFound := false for i := 0; i < 30; i++ { - clients, _ := suite.helper.UserClient.Get(clusterInfo, "client."+clientName) + clients, _ := s.helper.UserClient.Get(clusterInfo, "client."+clientName) if clients != "" { clientFound = true } @@ -322,7 +323,7 @@ func (suite *SmokeSuite) TestCreateClient() { time.Sleep(2 * time.Second) } - require.Equal(suite.T(), true, clientFound, "client not found") + assert.Equal(s.T(), true, clientFound, "client not found") logger.Infof("Update Client Smoke Test") newcaps := map[string]string{ @@ -330,30 +331,33 @@ func (suite *SmokeSuite) TestCreateClient() { "mgr": "allow rw", "osd": "allow *", } - caps, _ = suite.helper.UserClient.Update(clusterInfo, clientName, newcaps) + caps, _ = s.helper.UserClient.Update(clusterInfo, clientName, newcaps) - require.Equal(suite.T(), "allow r", caps["mon"], "wrong caps") - require.Equal(suite.T(), "allow rw", caps["mgr"], "wrong caps") - require.Equal(suite.T(), "allow *", caps["osd"], "wrong caps") + assert.Equal(s.T(), "allow r", caps["mon"], "wrong caps") + assert.Equal(s.T(), "allow rw", caps["mgr"], "wrong caps") + assert.Equal(s.T(), "allow *", caps["osd"], "wrong caps") + + err = s.helper.UserClient.Delete(clientName, s.settings.Namespace) + require.NoError(s.T(), err) } // Smoke Test for RBD Mirror CRD -func (suite *SmokeSuite) TestCreateRBDMirrorClient() { +func (s *SmokeSuite) TestCreateRBDMirrorClient() { logger.Infof("Create rbd-mirror Smoke Test") rbdMirrorName := "my-rbd-mirror" - err := suite.helper.RBDMirrorClient.Create(suite.namespace, rbdMirrorName, 1) - require.Nil(suite.T(), err) + err := s.helper.RBDMirrorClient.Create(s.settings.Namespace, rbdMirrorName, 1) + require.NoError(s.T(), err) - err = suite.helper.RBDMirrorClient.Delete(suite.namespace, rbdMirrorName) - require.Nil(suite.T(), err) + err = s.helper.RBDMirrorClient.Delete(s.settings.Namespace, rbdMirrorName) + require.NoError(s.T(), err) } -func (suite *SmokeSuite) getNonCanaryMonDeployments() ([]appsv1.Deployment, error) { +func (s *SmokeSuite) getNonCanaryMonDeployments() ([]appsv1.Deployment, error) { ctx := context.TODO() opts := metav1.ListOptions{LabelSelector: "app=rook-ceph-mon"} - deployments, err := suite.k8sh.Clientset.AppsV1().Deployments(suite.namespace).List(ctx, opts) + deployments, err := s.k8sh.Clientset.AppsV1().Deployments(s.settings.Namespace).List(ctx, opts) if err != nil { return nil, err } diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index 45d88f93f..4ebf92714 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -58,7 +58,7 @@ func TestCephUpgradeSuite(t *testing.T) { s := new(UpgradeSuite) defer func(s *UpgradeSuite) { - HandlePanics(recover(), s.op, s.T) + HandlePanics(recover(), s.TearDownSuite, s.T) }(s) suite.Run(t, s) } @@ -66,40 +66,37 @@ func TestCephUpgradeSuite(t *testing.T) { type UpgradeSuite struct { suite.Suite helper *clients.TestClient - op *TestCluster k8sh *utils.K8sHelper + settings *installer.TestCephSettings + installer *installer.CephInstaller namespace string } func (s *UpgradeSuite) SetupSuite() { s.namespace = "upgrade-ns" - upgradeTestCluster := TestCluster{ - clusterName: s.namespace, - namespace: s.namespace, - storeType: "", - storageClassName: "", - useHelm: false, - usePVC: false, - mons: 1, - rbdMirrorWorkers: 0, - rookCephCleanup: false, - skipOSDCreation: false, - minimalMatrixK8sVersion: upgradeMinimalTestVersion, - rookVersion: installer.Version1_5, - cephVersion: installer.NautilusVersion, + s.settings = &installer.TestCephSettings{ + ClusterName: s.namespace, + Namespace: s.namespace, + OperatorNamespace: installer.SystemNamespace(s.namespace), + StorageClassName: "", + UseHelm: false, + UsePVC: false, + Mons: 1, + UseCSI: true, + SkipOSDCreation: false, + RookVersion: installer.Version1_5, + CephVersion: installer.NautilusVersion, } - s.op, s.k8sh = StartTestCluster(s.T, &upgradeTestCluster) - s.helper = clients.CreateTestClient(s.k8sh, s.op.installer.Manifests) + s.installer, s.k8sh = StartTestCluster(s.T, s.settings, upgradeMinimalTestVersion) + s.helper = clients.CreateTestClient(s.k8sh, s.installer.Manifests) } func (s *UpgradeSuite) TearDownSuite() { - s.op.Teardown() + s.installer.UninstallRook() } func (s *UpgradeSuite) TestUpgradeToMaster() { - systemNamespace := installer.SystemNamespace(s.namespace) - // // Create block, object, and file storage before the upgrade // @@ -108,7 +105,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { blockName := "block-claim-upgrade" logger.Infof("Initializing block before the upgrade") clusterInfo := client.AdminClusterInfo(s.namespace) - setupBlockLite(s.helper, s.k8sh, s.Suite, clusterInfo, systemNamespace, poolName, storageClassName, blockName, rbdPodName, s.op.installer.CephVersion) + setupBlockLite(s.helper, s.k8sh, s.Suite, clusterInfo, poolName, storageClassName, blockName, rbdPodName) createPodWithBlock(s.helper, s.k8sh, s.Suite, s.namespace, storageClassName, rbdPodName, blockName) @@ -121,13 +118,12 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { logger.Infof("Initializing file before the upgrade") filesystemName := "upgrade-test-fs" activeCount := 1 - createFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, activeCount) + createFilesystem(s.helper, s.k8sh, s.Suite, s.settings, filesystemName, activeCount) // Start the file test client fsStorageClass := "file-upgrade" - assert.NoError(s.T(), s.helper.FSClient.CreateStorageClass(filesystemName, systemNamespace, s.namespace, fsStorageClass)) - useCSI := true - createFilesystemConsumerPod(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, fsStorageClass, useCSI) + assert.NoError(s.T(), s.helper.FSClient.CreateStorageClass(filesystemName, s.settings.OperatorNamespace, s.namespace, fsStorageClass)) + createFilesystemConsumerPod(s.helper, s.k8sh, s.Suite, s.settings, filesystemName, fsStorageClass) defer func() { cleanupFilesystemConsumer(s.helper, s.k8sh, s.Suite, s.namespace, filePodName) cleanupFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName) @@ -135,7 +131,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { logger.Infof("Initializing object before the upgrade") objectStoreName := "upgraded-object" - runObjectE2ETestLite(s.helper, s.k8sh, s.Suite, s.namespace, objectStoreName, 1, false) + runObjectE2ETestLite(s.helper, s.k8sh, s.Suite, s.settings, objectStoreName, 1, false) logger.Info("Initializing object bucket claim before the upgrade") bucketStorageClassName := "rook-smoke-delete-bucket" @@ -147,6 +143,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { defer func() { _ = s.helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketPrefix, maxObject, false) _ = s.helper.BucketClient.DeleteBucketStorageClass(s.namespace, objectStoreName, bucketStorageClassName, "Delete", region) + objectStoreCleanUp(s.Suite, s.helper, s.k8sh, s.settings.Namespace, objectStoreName) }() created := utils.Retry(12, 2*time.Second, "OBC is created", func() bool { @@ -178,7 +175,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { // Upgrade Rook from v1.5 to master // logger.Infof("*** UPGRADING ROOK FROM %s to master ***", installer.Version1_5) - s.gatherLogs(systemNamespace, "_before_master_upgrade") + s.gatherLogs(s.settings.OperatorNamespace, "_before_master_upgrade") s.upgradeToMaster() s.verifyOperatorImage(installer.VersionMaster) @@ -199,7 +196,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { // Upgrade from nautilus to octopus // logger.Infof("*** UPGRADING CEPH FROM Nautilus TO Octopus ***") - s.gatherLogs(systemNamespace, "_before_octopus_upgrade") + s.gatherLogs(s.settings.OperatorNamespace, "_before_octopus_upgrade") s.upgradeCephVersion(installer.OctopusVersion.Image, numOSDs) // Verify reading and writing to the test clients newFile = "post-octopus-upgrade-file" @@ -213,7 +210,7 @@ func (s *UpgradeSuite) gatherLogs(systemNamespace, testSuffix string) { s.k8sh.PrintPodDescribe(s.namespace) } n := strings.Replace(s.T().Name(), "/", "_", -1) + testSuffix - s.op.installer.GatherAllRookLogs(n, systemNamespace, s.namespace) + s.installer.GatherAllRookLogs(n, systemNamespace, s.namespace) } func (s *UpgradeSuite) upgradeCephVersion(newCephImage string, numOSDs int) { @@ -241,7 +238,7 @@ func (s *UpgradeSuite) verifyRookUpgrade(numOSDs int) { // Get some info about the currently deployed mons to determine later if they are all updated monDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-mon") require.NoError(s.T(), err) - require.Equal(s.T(), s.op.mons, len(monDepList.Items), monDepList.Items) + require.Equal(s.T(), s.settings.Mons, len(monDepList.Items), monDepList.Items) // Get some info about the currently deployed mgr to determine later if it is updated mgrDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-mgr") @@ -263,7 +260,7 @@ func (s *UpgradeSuite) verifyRookUpgrade(numOSDs int) { func (s *UpgradeSuite) waitForUpgradedDaemons(previousVersion, versionLabel string, numOSDs int, waitForMDS bool) { // wait for the mon(s) to be updated monsNotOldVersion := fmt.Sprintf("app=rook-ceph-mon,%s!=%s", versionLabel, previousVersion) - err := s.k8sh.WaitForDeploymentCount(monsNotOldVersion, s.namespace, s.op.mons) + err := s.k8sh.WaitForDeploymentCount(monsNotOldVersion, s.namespace, s.settings.Mons) require.NoError(s.T(), err, "mon(s) didn't update") err = s.k8sh.WaitForLabeledDeploymentsToBeReady(monsNotOldVersion, s.namespace) require.NoError(s.T(), err) @@ -336,19 +333,13 @@ func (s *UpgradeSuite) verifyFilesAfterUpgrade(fsName, newFileToWrite, messageFo // UpgradeToMaster performs the steps necessary to upgrade a Rook v1.4 cluster to master. It does not // verify the upgrade but merely starts the upgrade process. func (s *UpgradeSuite) upgradeToMaster() { - v1ExtensionsSupported := false - if s.k8sh.VersionAtLeast("v1.16.0") { - v1ExtensionsSupported = true - } - require.NoError(s.T(), s.k8sh.ResourceOperation("apply", upgradeManifestToMaster(s.namespace, v1ExtensionsSupported))) + // Apply the CRDs for the latest master + s.settings.RookVersion = installer.VersionMaster + m := installer.NewCephManifests(s.settings) + require.NoError(s.T(), s.k8sh.ResourceOperation("apply", m.GetCRDs(s.k8sh))) + + require.NoError(s.T(), s.k8sh.ResourceOperation("apply", m.GetCommon())) require.NoError(s.T(), - s.k8sh.SetDeploymentVersion(installer.SystemNamespace(s.namespace), operatorContainer, operatorContainer, installer.VersionMaster)) -} - -func upgradeManifestToMaster(namespace string, useV1Extensions bool) string { - // Always apply CRDs for the latest master - m := installer.NewCephManifests(installer.VersionMaster) - manifests := m.GetRookCRDs(useV1Extensions) - return manifests + s.k8sh.SetDeploymentVersion(s.settings.OperatorNamespace, operatorContainer, operatorContainer, installer.VersionMaster)) } diff --git a/tests/integration/nfs_test.go b/tests/integration/nfs_test.go index 6c4a2b60a..e84659532 100644 --- a/tests/integration/nfs_test.go +++ b/tests/integration/nfs_test.go @@ -45,7 +45,7 @@ func TestNfsSuite(t *testing.T) { s := new(NfsSuite) defer func(s *NfsSuite) { - HandlePanics(recover(), s, s.T) + HandlePanics(recover(), s.Teardown, s.T) }(s) suite.Run(t, s) } @@ -60,87 +60,87 @@ type NfsSuite struct { instanceCount int } -func (suite *NfsSuite) SetupSuite() { - suite.Setup() +func (s *NfsSuite) SetupSuite() { + s.Setup() } -func (suite *NfsSuite) TearDownSuite() { - suite.Teardown() +func (s *NfsSuite) TearDownSuite() { + s.Teardown() } -func (suite *NfsSuite) Setup() { - suite.namespace = "rook-nfs" - suite.systemNamespace = installer.SystemNamespace(suite.namespace) - suite.instanceCount = 1 +func (s *NfsSuite) Setup() { + s.namespace = "rook-nfs" + s.systemNamespace = installer.SystemNamespace(s.namespace) + s.instanceCount = 1 - k8shelper, err := utils.CreateK8sHelper(suite.T) + k8shelper, err := utils.CreateK8sHelper(s.T) v := version.MustParseSemantic(k8shelper.GetK8sServerVersion()) if !v.AtLeast(version.MustParseSemantic("1.14.0")) { logger.Info("Skipping NFS tests when not at least K8s v1.14") - suite.T().Skip() + s.T().Skip() } - require.NoError(suite.T(), err) - suite.k8shelper = k8shelper + require.NoError(s.T(), err) + s.k8shelper = k8shelper - k8sversion := suite.k8shelper.GetK8sServerVersion() + k8sversion := s.k8shelper.GetK8sServerVersion() logger.Infof("Installing nfs server on k8s %s", k8sversion) - suite.installer = installer.NewNFSInstaller(suite.k8shelper, suite.T) + s.installer = installer.NewNFSInstaller(s.k8shelper, s.T) - suite.rwClient = clients.CreateReadWriteOperation(suite.k8shelper) + s.rwClient = clients.CreateReadWriteOperation(s.k8shelper) - err = suite.installer.InstallNFSServer(suite.systemNamespace, suite.namespace, suite.instanceCount) + err = s.installer.InstallNFSServer(s.systemNamespace, s.namespace, s.instanceCount) if err != nil { logger.Errorf("nfs server installation failed: %+v", err) - suite.T().Fail() - suite.Teardown() - suite.T().FailNow() + s.T().Fail() + s.Teardown() + s.T().FailNow() } } -func (suite *NfsSuite) Teardown() { - suite.installer.GatherAllNFSServerLogs(suite.systemNamespace, suite.namespace, suite.T().Name()) - suite.installer.UninstallNFSServer(suite.systemNamespace, suite.namespace) +func (s *NfsSuite) Teardown() { + s.installer.GatherAllNFSServerLogs(s.systemNamespace, s.namespace, s.T().Name()) + s.installer.UninstallNFSServer(s.systemNamespace, s.namespace) } -func (suite *NfsSuite) TestNfsServerInstallation() { - logger.Infof("Verifying that nfs server pod %s is running", suite.namespace) +func (s *NfsSuite) TestNfsServerInstallation() { + logger.Infof("Verifying that nfs server pod %s is running", s.namespace) // verify nfs server operator is running OK - assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-nfs-operator", suite.systemNamespace, 1, "Running"), + assert.True(s.T(), s.k8shelper.CheckPodCountAndState("rook-nfs-operator", s.systemNamespace, 1, "Running"), "1 rook-nfs-operator must be in Running state") // verify nfs server instances are running OK - assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState(suite.namespace, suite.namespace, suite.instanceCount, "Running"), - fmt.Sprintf("%d rook-nfs pods must be in Running state", suite.instanceCount)) + assert.True(s.T(), s.k8shelper.CheckPodCountAndState(s.namespace, s.namespace, s.instanceCount, "Running"), + fmt.Sprintf("%d rook-nfs pods must be in Running state", s.instanceCount)) // verify bigger export is running OK - assert.True(suite.T(), true, suite.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim-bigger")) + assert.True(s.T(), true, s.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim-bigger")) - podList, err := suite.rwClient.CreateWriteClient("nfs-pv-claim-bigger") - require.NoError(suite.T(), err) - assert.True(suite.T(), true, suite.checkReadData(podList)) - err = suite.rwClient.Delete() - assert.NoError(suite.T(), err) + podList, err := s.rwClient.CreateWriteClient("nfs-pv-claim-bigger") + require.NoError(s.T(), err) + assert.True(s.T(), true, s.checkReadData(podList)) + err = s.rwClient.Delete() + assert.NoError(s.T(), err) // verify another smaller export is running OK - assert.True(suite.T(), true, suite.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim")) + assert.True(s.T(), true, s.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim")) - defer suite.rwClient.Delete() //nolint, delete a nfs consuming pod in rook - podList, err = suite.rwClient.CreateWriteClient("nfs-pv-claim") - require.NoError(suite.T(), err) - assert.True(suite.T(), true, suite.checkReadData(podList)) + defer s.rwClient.Delete() //nolint, delete a nfs consuming pod in rook + podList, err = s.rwClient.CreateWriteClient("nfs-pv-claim") + require.NoError(s.T(), err) + assert.True(s.T(), true, s.checkReadData(podList)) } -func (suite *NfsSuite) checkReadData(podList []string) bool { +func (s *NfsSuite) checkReadData(podList []string) bool { var result string var err error // the following for loop retries to read data from the first pod in the pod list for i := 0; i < utils.RetryLoop; i++ { // the nfs volume is mounted on "/mnt" and the data(hostname of the pod) is written in "/mnt/data" of the pod // results stores the hostname of either one of the pod which is same as the pod name, which is read from "/mnt/data" - result, err = suite.rwClient.Read(podList[0]) + result, err = s.rwClient.Read(podList[0]) logger.Infof("nfs volume read exited, err: %+v. result: %s", err, result) if err == nil { break @@ -148,7 +148,7 @@ func (suite *NfsSuite) checkReadData(podList []string) bool { logger.Warning("nfs volume read failed, will try again") time.Sleep(utils.RetryInterval * time.Second) } - require.NoError(suite.T(), err) + require.NoError(s.T(), err) // the value of result must be same as the name of pod. if result == podList[0] || result == podList[1] { return true diff --git a/tests/scripts/deploy_admission_controller.sh b/tests/scripts/deploy_admission_controller.sh index 440cb1900..6c0be04c7 100755 --- a/tests/scripts/deploy_admission_controller.sh +++ b/tests/scripts/deploy_admission_controller.sh @@ -17,7 +17,7 @@ trap cleanup SIGINT ERR SERVER_VERSION=$(kubectl version --short | awk -F "." '/Server Version/ {print $2}') MINIMUM_VERSION=16 -if [ ${SERVER_VERSION} -le ${MINIMUM_VERSION} ]; then +if [ ${SERVER_VERSION} -lt ${MINIMUM_VERSION} ]; then echo "required minimum kubernetes version 1.$MINIMUM_VERSION.0" exit fi