Merge pull request #7390 from travisn/refactor-tests

ceph: Refactor integration tests for simplification
This commit is contained in:
Travis Nielsen authored and GitHub committed 2021-03-17 13:51:37 -06:00
commit 7b3244a324
55 files changed
+1426 -4014

No files matched your search

@@ -54,4 +54,4 @@ jobs:
- name: setup tmate session for debugging
if: failure() && contains(github.event.pull_request.labels.*.name, 'debug-flex-suite')
uses: mxschmitt/action-tmate@v3
uses: mxschmitt/action-tmate@v3
@@ -45,7 +45,7 @@ jobs:
tests/scripts/minikube.sh helm
tests/scripts/helm.sh up
export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1)
go test -v -timeout 1800s -run CephHelmSuite github.com/rook/rook/tests/integration
SKIP_TEST_CLEANUP=false SKIP_CLEANUP_POLICY=false go test -v -timeout 1800s -run CephHelmSuite github.com/rook/rook/tests/integration
- name: Artifact
uses: actions/upload-artifact@v2
@@ -56,4 +56,4 @@ jobs:
- name: setup tmate session for debugging
if: failure() && contains(github.event.pull_request.labels.*.name, 'debug-helm-suite')
uses: mxschmitt/action-tmate@v3
uses: mxschmitt/action-tmate@v3
@@ -55,4 +55,4 @@ jobs:
- name: setup tmate session for debugging
if: failure() && contains(github.event.pull_request.labels.*.name, 'debug-multi-cluster-suite')
uses: mxschmitt/action-tmate@v3
uses: mxschmitt/action-tmate@v3
@@ -43,7 +43,7 @@ jobs:
- name: TestCephSmokeSuite
run: |
export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1)
go test -v -timeout 1800s -run CephSmokeSuite github.com/rook/rook/tests/integration
SKIP_CLEANUP_POLICY=false go test -v -timeout 1800s -run CephSmokeSuite github.com/rook/rook/tests/integration
- name: Artifact
uses: actions/upload-artifact@v2
@@ -54,4 +54,4 @@ jobs:
- name: setup tmate session for debugging
if: failure() && contains(github.event.pull_request.labels.*.name, 'debug-smoke-suite')
uses: mxschmitt/action-tmate@v3
uses: mxschmitt/action-tmate@v3
@@ -54,4 +54,4 @@ jobs:
- name: setup tmate session for debugging
if: failure() && contains(github.event.pull_request.labels.*.name, 'debug-upgrade-suite')
uses: mxschmitt/action-tmate@v3
uses: mxschmitt/action-tmate@v3
-2
View File
@@ -459,7 +459,6 @@ The following are the settings for Storage Class Device Sets which can be config
The following storage selection settings are specific to Ceph and do not apply to other backends. All variables are key-value pairs represented as strings.
* `metadataDevice`: Name of a device to use for the metadata of OSDs on each node. Performance can be improved by using a low latency device (such as SSD or NVMe) as the metadata device, while other spinning platter (HDD) devices on a node are used to store data. Provisioning will fail if the user specifies a `metadataDevice` but that device is not used as a metadata device by Ceph. Notably, `ceph-volume` will not use a device of the same device class (HDD, SSD, NVMe) as OSD devices for metadata, resulting in this failure.
* `storeType`: `bluestore`, the underlying storage format to use for each OSD. The default is set dynamically to `bluestore` for devices and is the only supported format at this point.
* `databaseSizeMB`: The size in MB of a bluestore database. Include quotes around the size.
* `walSizeMB`: The size in MB of a bluestore write ahead log (WAL). Include quotes around the size.
* `deviceClass`: The [CRUSH device class](https://ceph.io/community/new-luminous-crush-device-classes/) to use for this selection of storage devices. (By default, if a device's class has not already been set, OSDs will automatically set a device's class to either `hdd`, `ssd`, or `nvme` based on the hardware properties exposed by the Linux kernel.) These storage classes can then be used to select the devices backing a storage pool by specifying them as the value of [the pool spec's `deviceClass` field](ceph-pool-crd.md#spec).
@@ -764,7 +763,6 @@ spec:
- name: "sdc1" # One specific partition. Should not have a file system on it.
- name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # both device name and explicit udev links are supported
config: # configuration can be specified at the node level which overrides the cluster level config
storeType: bluestore
- name: "172.17.4.301"
deviceFilter: "^sd."
```
+1 -1
View File
@@ -12,7 +12,7 @@ Rook allows creation and customization of object stores through the custom resou
### Erasure Coded
Erasure coded pools require the OSDs to use `bluestore` for the configured [`storeType`](ceph-cluster-crd.md#osd-configuration-settings). Additionally, erasure coded pools can only be used with `dataPools`. The `metadataPool` must use a replicated pool.
Erasure coded pools can only be used with `dataPools`. The `metadataPool` must use a replicated pool.
> **NOTE**: This sample requires *at least 3 bluestore OSDs*, with each OSD located on a *different node*.
Vendored
+2
View File
@@ -232,6 +232,8 @@ def RunIntegrationTest(k, v) {
sh '''#!/bin/bash
set -o pipefail
export KUBECONFIG=$HOME/admin.conf \
SKIP_TEST_CLEANUP=false \
SKIP_CLEANUP_POLICY=false \
TEST_ENV_NAME='''+"${k}"+''' \
TEST_BASE_DIR="WORKING_DIR" \
TEST_LOG_COLLECTION_LEVEL='''+"${env.getLogs}"+''' \
+4 -4
View File
@@ -68,16 +68,16 @@ EOF
build_helm_resources() {
# Add helm annotations to all CRDS
yq w -i -d'*' "$HELM_CRDS_FILE_PATH" "metadata.annotations[helm.sh/resource-policy]" keep
# add header
sed -i '1s/^/{{- if .Values.crds.enabled }}\n/' "$HELM_CRDS_FILE_PATH"
sed -i '1s/^/{{- if semverCompare ">=1.16.0" .Capabilities.KubeVersion.GitVersion }}\n/' "$HELM_CRDS_FILE_PATH"
sed -i '1s/^/{{- if .Values.crds.enabled }}\n/' "$HELM_CRDS_FILE_PATH"
echo "Generating helm resources.yaml"
{
# add else
echo "{{- else }}"
# add footer
cat "$CRDS_BEFORE_1_16_FILE_PATH"
# DO NOT REMOVE the empty line, it is necessary
@@ -1,5 +1,5 @@
{{- if semverCompare ">=1.16.0" .Capabilities.KubeVersion.GitVersion }}
{{- if .Values.crds.enabled }}
{{- if semverCompare ">=1.16.0" .Capabilities.KubeVersion.GitVersion }}
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
@@ -218,7 +218,6 @@ spec:
# osdsPerDevice: "5"
# - name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # devices can be specified using full udev paths
# config: # configuration can be specified at the node level which overrides the cluster level config
# storeType: filestore
# - name: "172.17.4.301"
# deviceFilter: "^sd."
# The section for configuring management of daemon disruptions during upgrade or fencing.
@@ -350,6 +350,13 @@ data:
# Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used
ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true"
# Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release
# in favor of the CSI driver.
ROOK_ENABLE_FLEX_DRIVER: "false"
# Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster.
# This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs.
ROOK_ENABLE_DISCOVERY_DAEMON: "false"
# (Optional) Admission controller NodeAffinity.
# ADMISSION_CONTROLLER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
# (Optional) Admission controller tolerations list. Put here list of taints you want to tolerate in YAML format.
@@ -475,16 +482,6 @@ spec:
- name: DISCOVER_DAEMON_UDEV_BLACKLIST
value: "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+"
# Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release
# in favor of the CSI driver.
- name: ROOK_ENABLE_FLEX_DRIVER
value: "false"
# Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster.
# This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs.
- name: ROOK_ENABLE_DISCOVERY_DAEMON
value: "false"
# Whether to start machineDisruptionBudget and machineLabel controller to watch for the osd pods and MDBs.
- name: ROOK_ENABLE_MACHINE_DISRUPTION_BUDGET
value: "false"
+7 -10
View File
@@ -274,6 +274,13 @@ data:
# Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used
ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true"
# Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release
# in favor of the CSI driver.
ROOK_ENABLE_FLEX_DRIVER: "false"
# Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster.
# This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs.
ROOK_ENABLE_DISCOVERY_DAEMON: "false"
# (Optional) Admission controller NodeAffinity.
# ADMISSION_CONTROLLER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
# (Optional) Admission controller tolerations list. Put here list of taints you want to tolerate in YAML format.
@@ -424,16 +431,6 @@ spec:
- name: DISCOVER_DAEMON_UDEV_BLACKLIST
value: "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+"
# Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release
# in favor of the CSI driver.
- name: ROOK_ENABLE_FLEX_DRIVER
value: "false"
# Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster.
# This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs.
- name: ROOK_ENABLE_DISCOVERY_DAEMON
value: "false"
# Time to wait until the node controller will move Rook pods to other
# nodes after detecting an unreachable node.
# Pods affected by this setting are:
-3
View File
@@ -47,9 +47,6 @@ func init() {
operatorCmd.Flags().DurationVar(&mon.HealthCheckInterval, "mon-healthcheck-interval", mon.HealthCheckInterval, "mon health check interval (duration)")
operatorCmd.Flags().DurationVar(&mon.MonOutTimeout, "mon-out-timeout", mon.MonOutTimeout, "mon out timeout (duration)")
operatorCmd.Flags().BoolVar(&operator.EnableFlexDriver, "enable-flex-driver", true, "enable the rook flex driver")
operatorCmd.Flags().BoolVar(&operator.EnableDiscoveryDaemon, "enable-discovery-daemon", false, "enable the rook discovery daemon")
// csi deployment templates
operatorCmd.Flags().StringVar(&csi.RBDPluginTemplatePath, "csi-rbd-plugin-template-path", csi.DefaultRBDPluginTemplatePath, "path to ceph-csi rbd plugin template")
-1
View File
@@ -115,7 +115,6 @@ func addOSDConfigFlags(command *cobra.Command) {
// OSD store config flags
command.Flags().IntVar(&cfg.storeConfig.WalSizeMB, "osd-wal-size", osdcfg.WalDefaultSizeMB, "default size (MB) for OSD write ahead log (WAL) (bluestore)")
command.Flags().IntVar(&cfg.storeConfig.DatabaseSizeMB, "osd-database-size", 0, "default size (MB) for OSD database (bluestore)")
command.Flags().StringVar(&cfg.storeConfig.StoreType, "osd-store", "", "type of backing OSD store to use (bluestore or filestore)")
command.Flags().IntVar(&cfg.storeConfig.OSDsPerDevice, "osds-per-device", 1, "the number of OSDs per device")
command.Flags().BoolVar(&cfg.storeConfig.EncryptedDevice, "encrypted-device", false, "whether to encrypt the OSD with dmcrypt")
command.Flags().StringVar(&cfg.storeConfig.DeviceClass, "osd-crush-device-class", "", "The device class for all OSDs configured on this node")
+5 -4
View File
@@ -25,9 +25,10 @@ import (
"github.com/rook/rook/pkg/clusterd"
)
// RunAllCephCommandsInToolbox - when running the e2e tests, all ceph commands need to be run in the toolbox.
// RunAllCephCommandsInToolboxPod - when running the e2e tests, all ceph commands need to be run in the toolbox.
// Everywhere else, the ceph tools are assumed to be in the container where we can shell out.
var RunAllCephCommandsInToolbox = false
// This is the name of the pod.
var RunAllCephCommandsInToolboxPod string
const (
// AdminUsername is the name of the admin user
@@ -66,8 +67,8 @@ func FinalizeCephCommandArgs(command string, clusterInfo *ClusterInfo, args []st
}
// If the command should be run inside the toolbox pod, include the kubectl args to call the toolbox
if RunAllCephCommandsInToolbox {
toolArgs := []string{"exec", "-i", "rook-ceph-tools", "-n", clusterInfo.Namespace, "--", command}
if RunAllCephCommandsInToolboxPod != "" {
toolArgs := []string{"exec", "-i", RunAllCephCommandsInToolboxPod, "-n", clusterInfo.Namespace, "--", command}
return Kubectl, append(toolArgs, args...)
}
+4 -4
View File
@@ -23,7 +23,7 @@ import (
)
func TestFinalizeCephCommandArgs(t *testing.T) {
RunAllCephCommandsInToolbox = false
RunAllCephCommandsInToolboxPod = ""
configDir := "/var/lib/rook/rook-ceph"
expectedCommand := "ceph"
args := []string{"quorum_status"}
@@ -43,7 +43,7 @@ func TestFinalizeCephCommandArgs(t *testing.T) {
}
func TestFinalizeRadosGWAdminCommandArgs(t *testing.T) {
RunAllCephCommandsInToolbox = false
RunAllCephCommandsInToolboxPod = ""
configDir := "/var/lib/rook/rook-ceph"
expectedCommand := "radosgw-admin"
args := []string{
@@ -73,7 +73,7 @@ func TestFinalizeRadosGWAdminCommandArgs(t *testing.T) {
}
func TestFinalizeCephCommandArgsToolBox(t *testing.T) {
RunAllCephCommandsInToolbox = true
RunAllCephCommandsInToolboxPod = "rook-ceph-tools"
configDir := "/var/lib/rook/rook-ceph"
expectedCommand := "ceph"
args := []string{"health"}
@@ -93,5 +93,5 @@ func TestFinalizeCephCommandArgsToolBox(t *testing.T) {
cmd, args := FinalizeCephCommandArgs(expectedCommand, clusterInfo, args, configDir)
assert.Exactly(t, "kubectl", cmd)
assert.Exactly(t, expectedArgs, args)
RunAllCephCommandsInToolbox = false
RunAllCephCommandsInToolboxPod = ""
}
+2 -2
View File
@@ -38,7 +38,7 @@ func TestCephArgs(t *testing.T) {
assert.Equal(t, "--name=client.admin", args[3])
assert.Equal(t, "--keyring=/etc/a/client.admin.keyring", args[4])
RunAllCephCommandsInToolbox = true
RunAllCephCommandsInToolboxPod = "rook-ceph-tools"
args = []string{}
command, args = FinalizeCephCommandArgs(CephTool, clusterInfo, args, "/etc")
assert.Equal(t, Kubectl, command)
@@ -51,7 +51,7 @@ func TestCephArgs(t *testing.T) {
assert.Equal(t, "--", args[5])
assert.Equal(t, CephTool, args[6])
assert.Equal(t, "--connect-timeout=15", args[7])
RunAllCephCommandsInToolbox = false
RunAllCephCommandsInToolboxPod = ""
// cluster under /var/lib/rook
args = []string{"myarg"}
+3 -4
View File
@@ -31,6 +31,7 @@ import (
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/daemon/ceph/osd/kms"
"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
controllerutil "github.com/rook/rook/pkg/operator/ceph/controller"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/ceph/csi"
"github.com/rook/rook/pkg/operator/k8sutil"
@@ -51,7 +52,6 @@ import (
const (
controllerName = "ceph-cluster-controller"
enableFlexDriver = "ROOK_ENABLE_FLEX_DRIVER"
detectCephVersionTimeout = 15 * time.Minute
)
@@ -293,7 +293,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
// Remove finalizer
err = removeFinalizer(r.client, request.NamespacedName)
if err != nil {
return reconcile.Result{}, errors.Wrap(err, "failed to remove finalize")
return reconcile.Result{}, errors.Wrap(err, "failed to remove finalizer")
}
// Return and do not requeue. Successful deletion.
@@ -421,8 +421,7 @@ func (c *ClusterController) checkIfVolumesExist(cluster *cephv1.CephCluster) err
return err
}
}
flexDriverEnabled := os.Getenv(enableFlexDriver) != "false"
if !flexDriverEnabled {
if !controllerutil.FlexDriverEnabled(c.context) {
logger.Debugf("Flex driver disabled, skipping check for volume attachments for cluster %q", cluster.Namespace)
return nil
}
@@ -33,7 +33,6 @@ func GetConfigStoreName(nodeName string) string {
}
const (
StoreTypeKey = "storeType"
WalSizeMBKey = "walSizeMB"
DatabaseSizeMBKey = "databaseSizeMB"
JournalSizeMBKey = "journalSizeMB"
@@ -45,7 +44,6 @@ const (
// StoreConfig represents the configuration of an OSD on a device.
type StoreConfig struct {
StoreType string `json:"storeType,omitempty"`
WalSizeMB int `json:"walSizeMB,omitempty"`
DatabaseSizeMB int `json:"databaseSizeMB,omitempty"`
OSDsPerDevice int `json:"osdsPerDevice,omitempty"`
@@ -66,8 +64,6 @@ func ToStoreConfig(config map[string]string) StoreConfig {
storeConfig := NewStoreConfig()
for k, v := range config {
switch k {
case StoreTypeKey:
storeConfig.StoreType = v
case WalSizeMBKey:
storeConfig.WalSizeMB = convertToIntIgnoreErr(v)
case DatabaseSizeMBKey:
-3
View File
@@ -29,7 +29,6 @@ import (
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1"
kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms"
"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
opconfig "github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
@@ -325,7 +324,6 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
return nil, errors.New("empty volumes")
}
storeType := config.Bluestore
osdID := strconv.Itoa(osd.ID)
tiniEnvVar := v1.EnvVar{Name: "TINI_SUBREAPER", Value: ""}
envVars := append(c.getConfigEnvVars(osdProps, dataDir), []v1.EnvVar{
@@ -335,7 +333,6 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
envVars = append(envVars, []v1.EnvVar{
{Name: "ROOK_OSD_UUID", Value: osd.UUID},
{Name: "ROOK_OSD_ID", Value: osdID},
{Name: "ROOK_OSD_STORE_TYPE", Value: storeType},
{Name: "ROOK_CEPH_MON_HOST",
ValueFrom: &v1.EnvVarSource{
SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{
@@ -59,6 +59,18 @@ var (
OperatorCephBaseImageVersion string
)
func FlexDriverEnabled(context *clusterd.Context) bool {
// Ignore the error. In the remote chance that the configmap fails to be read, we will default to disabling the flex driver
value, _ := k8sutil.GetOperatorSetting(context.Clientset, OperatorSettingConfigMapName, "ROOK_ENABLE_FLEX_DRIVER", "false")
return value == "true"
}
func DiscoveryDaemonEnabled(context *clusterd.Context) bool {
// Ignore the error. In the remote chance that the configmap fails to be read, we will default to disabling the discovery daemon
value, _ := k8sutil.GetOperatorSetting(context.Clientset, OperatorSettingConfigMapName, "ROOK_ENABLE_DISCOVERY_DAEMON", "false")
return value == "true"
}
// CheckForCancelledOrchestration checks whether a cancellation has been requested
func CheckForCancelledOrchestration(context *clusterd.Context) error {
defer context.RequestCancelOrchestration.UnSet()
@@ -130,6 +142,7 @@ func IsReadyToReconcile(c client.Client, clustercontext *clusterd.Context, names
}
}
logger.Debugf("%q: CephCluster %q initial reconcile is not complete yet...", controllerName, namespacedName.Namespace)
return cephCluster, false, cephClusterExists, WaitForRequeueIfCephClusterNotReady
}
+1 -1
View File
@@ -312,7 +312,7 @@ func (r *ReconcileObjectStoreUser) initializeObjectStoreContext(u *cephv1.CephOb
r.objContext = objContext
r.objContext.Endpoint = store.Status.Info["endpoint"]
if store.Status.Info["secureEndpoint"] != "" {
if store.Status != nil && store.Status.Info["secureEndpoint"] != "" {
r.objContext.Endpoint = store.Status.Info["secureEndpoint"]
}
if r.objContext.Endpoint == "" {
+3 -9
View File
@@ -57,12 +57,6 @@ var provisionerConfigs = map[string]string{
}
var (
// EnableFlexDriver Whether to enable the flex driver. If true, the rook-ceph-agent daemonset will be started.
EnableFlexDriver = true
// EnableDiscoveryDaemon Whether to enable the daemon for device discovery. If true, the rook-ceph-discover daemonset will be started.
EnableDiscoveryDaemon = true
// ImmediateRetryResult Return this for a immediate retry of the reconciliation loop with the same request object.
ImmediateRetryResult = reconcile.Result{Requeue: true}
)
@@ -119,7 +113,7 @@ func (o *Operator) Run() error {
defer stopFunc()
rookDiscover := discover.New(o.context.Clientset)
if EnableDiscoveryDaemon {
if opcontroller.DiscoveryDaemonEnabled(o.context) {
if err := rookDiscover.Start(o.operatorNamespace, o.rookImage, o.securityAccount, true); err != nil {
return errors.Wrap(err, "failed to start device discovery daemonset")
}
@@ -145,7 +139,7 @@ func (o *Operator) Run() error {
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
// For Flex Driver, run volume provisioner for each of the supported configurations
if EnableFlexDriver {
if opcontroller.FlexDriverEnabled(o.context) {
for name, vendor := range provisionerConfigs {
volumeProvisioner := provisioner.New(o.context, vendor)
pc := controller.NewProvisionController(
@@ -216,7 +210,7 @@ func (o *Operator) updateDrivers() error {
return errors.Errorf("rook operator namespace is not provided. expose it via downward API in the rook operator manifest file using environment variable %s", k8sutil.PodNamespaceEnvVar)
}
if EnableFlexDriver {
if opcontroller.FlexDriverEnabled(o.context) {
rookAgent := agent.New(o.context.Clientset)
if err := rookAgent.Start(o.operatorNamespace, o.rookImage, o.securityAccount); err != nil {
return errors.Wrap(err, "error starting agent daemonset")
+13 -13
View File
@@ -62,26 +62,26 @@ func (b *BlockOperation) Create(manifest string, size int) (string, error) {
}
func (b *BlockOperation) CreateStorageClassAndPVC(csi bool, pvcNamespace, clusterNamespace, systemNamespace, poolName, storageClassName, reclaimPolicy, blockName, mode string) error {
if err := b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockPoolDef(poolName, clusterNamespace, "1")); err != nil {
func (b *BlockOperation) CreateStorageClassAndPVC(pvcNamespace, poolName, storageClassName, reclaimPolicy, blockName, mode string) error {
if err := b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockPool(poolName, "1")); err != nil {
return err
}
if err := b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockStorageClassDef(csi, poolName, storageClassName, reclaimPolicy, clusterNamespace, systemNamespace)); err != nil {
if err := b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockStorageClass(poolName, storageClassName, reclaimPolicy)); err != nil {
return err
}
return b.k8sClient.ResourceOperation("apply", b.manifests.GetPVC(blockName, pvcNamespace, storageClassName, mode, "1M"))
return b.k8sClient.ResourceOperation("apply", installer.GetPVC(blockName, pvcNamespace, storageClassName, mode, "1M"))
}
func (b *BlockOperation) CreatePVC(namespace, claimName, storageClassName, mode, size string) error {
return b.k8sClient.ResourceOperation("apply", b.manifests.GetPVC(claimName, namespace, storageClassName, mode, size))
return b.k8sClient.ResourceOperation("apply", installer.GetPVC(claimName, namespace, storageClassName, mode, size))
}
func (b *BlockOperation) CreatePod(podName, claimName, namespace, mountPoint string, readOnly bool) error {
return b.k8sClient.ResourceOperation("apply", b.manifests.GetPod(podName, claimName, namespace, mountPoint, readOnly))
return b.k8sClient.ResourceOperation("apply", installer.GetPodWithVolume(podName, claimName, namespace, mountPoint, readOnly))
}
func (b *BlockOperation) CreateStorageClass(csi bool, poolName, storageClassName, reclaimPolicy, namespace string) error {
return b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockStorageClassDef(csi, poolName, storageClassName, reclaimPolicy, namespace, installer.SystemNamespace(namespace)))
return b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockStorageClass(poolName, storageClassName, reclaimPolicy))
}
func (b *BlockOperation) DeletePVC(namespace, claimName string) error {
@@ -91,27 +91,27 @@ func (b *BlockOperation) DeletePVC(namespace, claimName string) error {
}
func (b *BlockOperation) CreatePVCRestore(namespace, claimName, snapshotName, storageClassName, mode, size string) error {
return b.k8sClient.ResourceOperation("apply", b.manifests.GetPVCRestore(claimName, snapshotName, namespace, storageClassName, mode, size))
return b.k8sClient.ResourceOperation("apply", installer.GetPVCRestore(claimName, snapshotName, namespace, storageClassName, mode, size))
}
func (b *BlockOperation) CreatePVCClone(namespace, cloneClaimName, parentClaimName, storageClassName, mode, size string) error {
return b.k8sClient.ResourceOperation("apply", b.manifests.GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, mode, size))
return b.k8sClient.ResourceOperation("apply", installer.GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, mode, size))
}
func (b *BlockOperation) CreateSnapshotClass(snapshotClassName, deletePolicy, namespace string) error {
return b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockSnapshotClass(snapshotClassName, namespace, namespace, deletePolicy))
return b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockSnapshotClass(snapshotClassName, deletePolicy))
}
func (b *BlockOperation) DeleteSnapshotClass(snapshotClassName, deletePolicy, namespace string) error {
return b.k8sClient.ResourceOperation("delete", b.manifests.GetBlockSnapshotClass(snapshotClassName, namespace, namespace, deletePolicy))
return b.k8sClient.ResourceOperation("delete", b.manifests.GetBlockSnapshotClass(snapshotClassName, deletePolicy))
}
func (b *BlockOperation) CreateSnapshot(snapshotName, claimName, snapshotClassName, namespace string) error {
return b.k8sClient.ResourceOperation("apply", b.manifests.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace))
return b.k8sClient.ResourceOperation("apply", installer.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace))
}
func (b *BlockOperation) DeleteSnapshot(snapshotName, claimName, snapshotClassName, namespace string) error {
return b.k8sClient.ResourceOperation("delete", b.manifests.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace))
return b.k8sClient.ResourceOperation("delete", installer.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace))
}
func (b *BlockOperation) DeleteStorageClass(storageClassName string) error {
+4 -4
View File
@@ -37,20 +37,20 @@ func CreateBucketOperation(k8sh *utils.K8sHelper, manifests installer.CephManife
}
func (b *BucketOperation) CreateBucketStorageClass(namespace string, storeName string, storageClassName string, reclaimPolicy string, region string) error {
return b.k8sh.ResourceOperation("create", b.manifests.GetBucketStorageClass(namespace, storeName, storageClassName, reclaimPolicy, region))
return b.k8sh.ResourceOperation("create", b.manifests.GetBucketStorageClass(storeName, storageClassName, reclaimPolicy, region))
}
func (b *BucketOperation) DeleteBucketStorageClass(namespace string, storeName string, storageClassName string, reclaimPolicy string, region string) error {
err := b.k8sh.ResourceOperation("delete", b.manifests.GetBucketStorageClass(namespace, storeName, storageClassName, reclaimPolicy, region))
err := b.k8sh.ResourceOperation("delete", b.manifests.GetBucketStorageClass(storeName, storageClassName, reclaimPolicy, region))
return err
}
func (b *BucketOperation) CreateObc(obcName string, storageClassName string, bucketName string, maxObject string, createBucket bool) error {
return b.k8sh.ResourceOperation("create", b.manifests.GetObc(obcName, storageClassName, bucketName, maxObject, createBucket))
return b.k8sh.ResourceOperation("create", b.manifests.GetOBC(obcName, storageClassName, bucketName, maxObject, createBucket))
}
func (b *BucketOperation) DeleteObc(obcName string, storageClassName string, bucketName string, maxObject string, createBucket bool) error {
return b.k8sh.ResourceOperation("delete", b.manifests.GetObc(obcName, storageClassName, bucketName, maxObject, createBucket))
return b.k8sh.ResourceOperation("delete", b.manifests.GetOBC(obcName, storageClassName, bucketName, maxObject, createBucket))
}
// CheckOBC, returns true if the obc, secret and configmap are all in the "check" state,
+2 -2
View File
@@ -42,7 +42,7 @@ func CreateClientOperation(k8sh *utils.K8sHelper, manifests installer.CephManife
// Create creates a client in Rook
func (c *ClientOperation) Create(name, namespace string, caps map[string]string) error {
logger.Infof("creating the client via CRD")
if err := c.k8sh.ResourceOperation("apply", c.manifests.GetClient(namespace, name, caps)); err != nil {
if err := c.k8sh.ResourceOperation("apply", c.manifests.GetClient(name, caps)); err != nil {
return err
}
return nil
@@ -76,7 +76,7 @@ func (c *ClientOperation) Get(clusterInfo *client.ClusterInfo, clientName string
func (c *ClientOperation) Update(clusterInfo *client.ClusterInfo, clientName string, caps map[string]string) (updatedcaps map[string]string, error error) {
context := c.k8sh.MakeContext()
logger.Infof("updating the client via CRD")
if err := c.k8sh.ResourceOperation("apply", c.manifests.GetClient(clusterInfo.Namespace, clientName, caps)); err != nil {
if err := c.k8sh.ResourceOperation("apply", c.manifests.GetClient(clientName, caps)); err != nil {
return nil, err
}
+12 -12
View File
@@ -42,7 +42,7 @@ func CreateFilesystemOperation(k8sh *utils.K8sHelper, manifests installer.CephMa
// Create creates a filesystem in Rook
func (f *FilesystemOperation) Create(name, namespace string, activeCount int) error {
logger.Infof("creating the filesystem via CRD")
if err := f.k8sh.ResourceOperation("apply", f.manifests.GetFilesystem(namespace, name, activeCount)); err != nil {
if err := f.k8sh.ResourceOperation("apply", f.manifests.GetFilesystem(name, activeCount)); err != nil {
return err
}
@@ -58,32 +58,32 @@ func (f *FilesystemOperation) Create(name, namespace string, activeCount int) er
// CreateStorageClass creates a storage class for CephFS clients
func (f *FilesystemOperation) CreateStorageClass(fsName, systemNamespace, namespace, storageClassName string) error {
return f.k8sh.ResourceOperation("apply", f.manifests.GetFileStorageClassDef(fsName, storageClassName, systemNamespace, namespace))
return f.k8sh.ResourceOperation("apply", f.manifests.GetFileStorageClass(fsName, storageClassName))
}
// CreateSnapshotClass creates a snapshot class for CephFS clients
func (f *FilesystemOperation) CreateSnapshotClass(snapshotClassName, reclaimPolicy, namespace string) error {
return f.k8sh.ResourceOperation("apply", f.manifests.GetFileStorageSnapshotClass(snapshotClassName, namespace, namespace, reclaimPolicy))
return f.k8sh.ResourceOperation("apply", f.manifests.GetFileStorageSnapshotClass(snapshotClassName, reclaimPolicy))
}
// CreatePVCRestore creates a pvc from snapshot
func (f *FilesystemOperation) CreatePVCRestore(namespace, claimName, snapshotName, storageClassName, mode, size string) error {
return f.k8sh.ResourceOperation("apply", f.manifests.GetPVCRestore(claimName, snapshotName, namespace, storageClassName, mode, size))
return f.k8sh.ResourceOperation("apply", installer.GetPVCRestore(claimName, snapshotName, namespace, storageClassName, mode, size))
}
// CreatePVCClone creates a pvc from pvc
func (f *FilesystemOperation) CreatePVCClone(namespace, cloneClaimName, parentClaimName, storageClassName, mode, size string) error {
return f.k8sh.ResourceOperation("apply", f.manifests.GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, mode, size))
return f.k8sh.ResourceOperation("apply", installer.GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, mode, size))
}
// CreateSnapshot creates a snapshot from pvc
func (f *FilesystemOperation) CreateSnapshot(snapshotName, claimName, snapshotClassName, namespace string) error {
return f.k8sh.ResourceOperation("apply", f.manifests.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace))
return f.k8sh.ResourceOperation("apply", installer.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace))
}
// DeleteSnapshot deletes the snapshot
func (f *FilesystemOperation) DeleteSnapshot(snapshotName, claimName, snapshotClassName, namespace string) error {
return f.k8sh.ResourceOperation("delete", f.manifests.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace))
return f.k8sh.ResourceOperation("delete", installer.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace))
}
func (f *FilesystemOperation) DeletePVC(namespace, claimName string) error {
@@ -104,21 +104,21 @@ func (f *FilesystemOperation) DeleteStorageClass(storageClassName string) error
}
func (f *FilesystemOperation) CreatePVC(namespace, claimName, storageClassName, mode, size string) error {
return f.k8sh.ResourceOperation("apply", f.manifests.GetPVC(claimName, namespace, storageClassName, mode, size))
return f.k8sh.ResourceOperation("apply", installer.GetPVC(claimName, namespace, storageClassName, mode, size))
}
func (f *FilesystemOperation) CreatePod(podName, claimName, namespace, mountPoint string, readOnly bool) error {
return f.k8sh.ResourceOperation("apply", f.manifests.GetPod(podName, claimName, namespace, mountPoint, readOnly))
return f.k8sh.ResourceOperation("apply", installer.GetPodWithVolume(podName, claimName, namespace, mountPoint, readOnly))
}
func (f *FilesystemOperation) DeleteSnapshotClass(snapshotClassName, deletePolicy, namespace string) error {
return f.k8sh.ResourceOperation("delete", f.manifests.GetFileStorageSnapshotClass(snapshotClassName, namespace, namespace, deletePolicy))
return f.k8sh.ResourceOperation("delete", f.manifests.GetFileStorageSnapshotClass(snapshotClassName, deletePolicy))
}
// ScaleDown scales down the number of active metadata servers of a filesystem in Rook
func (f *FilesystemOperation) ScaleDown(name, namespace string) error {
logger.Infof("scaling down the number of filesystem active metadata servers via CRD")
if err := f.k8sh.ResourceOperation("apply", f.manifests.GetFilesystem(namespace, name, 1)); err != nil {
if err := f.k8sh.ResourceOperation("apply", f.manifests.GetFilesystem(name, 1)); err != nil {
return err
}
@@ -144,7 +144,7 @@ func (f *FilesystemOperation) Delete(name, namespace string) error {
}
logger.Infof("Deleted filesystem %s in namespace %s", name, namespace)
return f.k8sh.WaitForCustomResourceDeletion(namespace, crdCheckerFunc)
return f.k8sh.WaitForCustomResourceDeletion(namespace, name, crdCheckerFunc)
}
// List lists filesystems in Rook
+1 -1
View File
@@ -42,7 +42,7 @@ func CreateNFSOperation(k8sh *utils.K8sHelper, manifests installer.CephManifests
func (n *NFSOperation) Create(namespace, name, pool string, daemonCount int) error {
logger.Infof("creating the NFS daemons via CRD")
if err := n.k8sh.ResourceOperation("apply", n.manifests.GetNFS(namespace, name, pool, daemonCount)); err != nil {
if err := n.k8sh.ResourceOperation("apply", n.manifests.GetNFS(name, pool, daemonCount)); err != nil {
return err
}
+1 -1
View File
@@ -43,7 +43,7 @@ func CreateObjectOperation(k8sh *utils.K8sHelper, manifests installer.CephManife
func (o *ObjectOperation) Create(namespace, storeName string, replicaCount int32) error {
logger.Infof("creating the object store via CRD")
if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStore(namespace, storeName, int(replicaCount), rgwPort)); err != nil {
if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStore(storeName, int(replicaCount), rgwPort)); err != nil {
return err
}
+1 -1
View File
@@ -68,7 +68,7 @@ func (o *ObjectUserOperation) UserSecretExists(namespace string, store string, u
func (o *ObjectUserOperation) Create(namespace string, userid string, displayName string, store string) error {
logger.Infof("creating the object store user via CRD")
if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStoreUser(namespace, userid, displayName, store)); err != nil {
if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStoreUser(userid, displayName, store)); err != nil {
return err
}
return nil
+2 -2
View File
@@ -50,7 +50,7 @@ func (p *PoolOperation) Update(name, namespace string, replicas int) error {
}
func (p *PoolOperation) createOrUpdatePool(name, namespace, action string, replicas int) error {
return p.k8sh.ResourceOperation(action, p.manifests.GetBlockPoolDef(name, namespace, strconv.Itoa(replicas)))
return p.k8sh.ResourceOperation(action, p.manifests.GetBlockPool(name, strconv.Itoa(replicas)))
}
func (p *PoolOperation) ListCephPools(clusterInfo *client.ClusterInfo) ([]client.CephStoragePoolSummary, error) {
@@ -147,5 +147,5 @@ func (p *PoolOperation) DeletePool(blockClient *BlockOperation, clusterInfo *cli
return err
}
return p.k8sh.WaitForCustomResourceDeletion(clusterInfo.Namespace, crdCheckerFunc)
return p.k8sh.WaitForCustomResourceDeletion(clusterInfo.Namespace, poolName, crdCheckerFunc)
}
+1 -1
View File
@@ -41,7 +41,7 @@ func CreateRBDMirrorOperation(k8sh *utils.K8sHelper, manifests installer.CephMan
func (r *RBDMirrorOperation) Create(namespace, name string, daemonCount int) error {
logger.Infof("creating the RBDMirror daemons via CRD")
if err := r.k8sh.ResourceOperation("apply", r.manifests.GetRBDMirror(namespace, name, daemonCount)); err != nil {
if err := r.k8sh.ResourceOperation("apply", r.manifests.GetRBDMirror(name, daemonCount)); err != nil {
return err
}
@@ -131,7 +131,7 @@ func (ci *CassandraInstaller) DeleteCassandraCluster(namespace string) {
_, err := ci.k8sHelper.RookClientset.CassandraV1alpha1().Clusters(namespace).Get(ctx, namespace, metav1.GetOptions{})
return err
}
err = ci.k8sHelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc)
err = ci.k8sHelper.WaitForCustomResourceDeletion(namespace, namespace, crdCheckerFunc)
assert.NoError(ci.T(), err)
// Delete Namespace
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+142
View File
@@ -0,0 +1,142 @@
/*
Copyright 2021 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package installer
import (
"fmt"
"io/ioutil"
"net/http"
"os"
"path"
"strings"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/tests/framework/utils"
)
// TestCephSettings struct for handling panic and test suite tear down
type TestCephSettings struct {
DataDirHostPath string
ClusterName string
Namespace string
OperatorNamespace string
StorageClassName string
UseHelm bool
UsePVC bool
Mons int
UseCrashPruner bool
MultipleMgrs bool
SkipOSDCreation bool
UseCSI bool
EnableDiscovery bool
EnableAdmissionController bool
IsExternal bool
SkipClusterCleanup bool
SkipCleanupPolicy bool
DirectMountToolbox bool
RookVersion string
CephVersion cephv1.CephVersionSpec
}
func (s *TestCephSettings) ApplyEnvVars() {
// skip the cleanup by default
s.SkipClusterCleanup = true
if os.Getenv("SKIP_TEST_CLEANUP") == "false" {
s.SkipClusterCleanup = false
}
s.SkipCleanupPolicy = true
if os.Getenv("SKIP_CLEANUP_POLICY") == "false" {
s.SkipCleanupPolicy = false
}
}
func (s *TestCephSettings) readManifest(filename string) string {
rootDir, err := utils.FindRookRoot()
if err != nil {
panic(err)
}
manifest := path.Join(rootDir, "cluster/examples/kubernetes/ceph", filename)
logger.Infof("Reading manifest: %s", manifest)
contents, err := ioutil.ReadFile(manifest)
if err != nil {
panic(errors.Wrapf(err, "failed to read manifest at %s", manifest))
}
return replaceNamespaces(manifest, string(contents), s.OperatorNamespace, s.Namespace)
}
func (s *TestCephSettings) readManifestFromGithub(filename string) string {
return s.readManifestFromGithubWithClusterNamespace(filename, s.Namespace)
}
func (s *TestCephSettings) readManifestFromGithubWithClusterNamespace(filename, clusterNamespace string) string {
url := fmt.Sprintf("https://raw.githubusercontent.com/rook/rook/%s/cluster/examples/kubernetes/ceph/%s", s.RookVersion, filename)
logger.Infof("Retrieving manifest: %s", url)
// #nosec G107 This is only test code and is expected to read from a url
response, err := http.Get(url)
if err != nil {
panic(errors.Wrapf(err, "failed to read manifest from %s", url))
}
defer response.Body.Close()
content, err := ioutil.ReadAll(response.Body)
if err != nil {
panic(errors.Wrapf(err, "failed to read content from %s", url))
}
return replaceNamespaces(url, string(content), s.OperatorNamespace, clusterNamespace)
}
func (s *TestCephSettings) replaceOperatorSettings(manifest string) string {
manifest = strings.ReplaceAll(manifest, `# CSI_LOG_LEVEL: "0"`, `CSI_LOG_LEVEL: "5"`)
manifest = strings.ReplaceAll(manifest, `ROOK_ENABLE_DISCOVERY_DAEMON: "false"`, fmt.Sprintf(`ROOK_ENABLE_DISCOVERY_DAEMON: "%t"`, s.EnableDiscovery))
manifest = strings.ReplaceAll(manifest, `ROOK_ENABLE_FLEX_DRIVER: "false"`, fmt.Sprintf(`ROOK_ENABLE_FLEX_DRIVER: "%t"`, !s.UseCSI))
manifest = strings.ReplaceAll(manifest, `ROOK_CSI_ENABLE_CEPHFS: "true"`, fmt.Sprintf(`ROOK_CSI_ENABLE_CEPHFS: "%t"`, s.UseCSI))
manifest = strings.ReplaceAll(manifest, `ROOK_CSI_ENABLE_RBD: "true"`, fmt.Sprintf(`ROOK_CSI_ENABLE_RBD: "%t"`, s.UseCSI))
return manifest
}
func replaceNamespaces(name, manifest, operatorNamespace, clusterNamespace string) string {
// RBAC and related namespaces
manifest = strings.ReplaceAll(manifest, "rook-ceph # namespace:operator", operatorNamespace)
manifest = strings.ReplaceAll(manifest, "rook-ceph # namespace:cluster", clusterNamespace)
manifest = strings.ReplaceAll(manifest, "rook-ceph-external # namespace:cluster", clusterNamespace)
// Double space only needed for v1.5 upgrade test
manifest = strings.ReplaceAll(manifest, "rook-ceph # namespace:operator", operatorNamespace)
// SCC namespaces for operator and Ceph daemons
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-system # serviceaccount:namespace:operator", operatorNamespace+":rook-ceph-system")
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-mgr # serviceaccount:namespace:cluster", clusterNamespace+":rook-ceph-mgr")
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-osd # serviceaccount:namespace:cluster", clusterNamespace+":rook-ceph-osd")
// SCC namespaces for CSI driver
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-rbd-plugin-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-rbd-plugin-sa")
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-rbd-provisioner-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-rbd-provisioner-sa")
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-cephfs-plugin-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-cephfs-plugin-sa")
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-cephfs-provisioner-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-cephfs-provisioner-sa")
// CSI Drivers
manifest = strings.ReplaceAll(manifest, "rook-ceph.cephfs.csi.ceph.com # driver:namespace:operator", operatorNamespace+".cephfs.csi.ceph.com")
manifest = strings.ReplaceAll(manifest, "rook-ceph.rbd.csi.ceph.com # driver:namespace:operator", operatorNamespace+".rbd.csi.ceph.com")
// Bucket storage class
manifest = strings.ReplaceAll(manifest, "rook-ceph.ceph.rook.io/bucket # driver:namespace:cluster", clusterNamespace+".ceph.rook.io/bucket")
if strings.Contains(manifest, "namespace:operator") || strings.Contains(manifest, "namespace:cluster") || strings.Contains(manifest, "driver:namespace:") || strings.Contains(manifest, "serviceaccount:namespace:") {
logger.Infof("BAD MANIFEST:\n%s", manifest)
panic(fmt.Sprintf("manifest %s still contains a namespace identifier", name))
}
return manifest
}
@@ -0,0 +1,110 @@
/*
Copyright 2021 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package installer
import (
"strconv"
)
func GetPodWithVolume(podName, claimName, namespace, mountPath string, readOnly bool) string {
return `
apiVersion: v1
kind: Pod
metadata:
name: ` + podName + `
namespace: ` + namespace + `
spec:
containers:
- name: ` + podName + `
image: busybox
command: ["/bin/sleep", "infinity"]
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: ` + mountPath + `
name: csivol
volumes:
- name: csivol
persistentVolumeClaim:
claimName: ` + claimName + `
readOnly: ` + strconv.FormatBool(readOnly) + `
restartPolicy: Never
`
}
func GetPVC(claimName, namespace, storageClassName, accessModes, size string) string {
return `apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: ` + claimName + `
namespace: ` + namespace + `
spec:
storageClassName: ` + storageClassName + `
accessModes:
- ` + accessModes + `
resources:
requests:
storage: ` + size
}
func GetPVCRestore(claimName, snapshotName, namespace, storageClassName, accessModes, size string) string {
return `apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: ` + claimName + `
namespace: ` + namespace + `
spec:
storageClassName: ` + storageClassName + `
dataSource:
name: ` + snapshotName + `
kind: VolumeSnapshot
apiGroup: snapshot.storage.k8s.io
accessModes:
- ` + accessModes + `
resources:
requests:
storage: ` + size
}
func GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, accessModes, size string) string {
return `apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: ` + cloneClaimName + `
namespace: ` + namespace + `
spec:
storageClassName: ` + storageClassName + `
dataSource:
name: ` + parentClaimName + `
kind: PersistentVolumeClaim
accessModes:
- ` + accessModes + `
resources:
requests:
storage: ` + size
}
func GetSnapshot(snapshotName, claimName, snapshotClassName, namespace string) string {
return `apiVersion: snapshot.storage.k8s.io/v1beta1
kind: VolumeSnapshot
metadata:
name: ` + snapshotName + `
namespace: ` + namespace + `
spec:
volumeSnapshotClassName: ` + snapshotClassName + `
source:
persistentVolumeClaimName: ` + claimName
}
-5
View File
@@ -45,11 +45,6 @@ var (
deleteFromStdinArgs = append(deleteArgs, "-")
)
type TestSuite interface {
Setup()
Teardown()
}
func SkipTestSuite(name string) bool {
testsToRun := testStorageProvider()
// jenkins passes "null" if the env var is not set.
+1 -1
View File
@@ -166,7 +166,7 @@ func (h *NFSInstaller) UninstallNFSServer(systemNamespace, namespace string) {
_, err := h.k8shelper.RookClientset.NfsV1alpha1().NFSServers(namespace).Get(ctx, namespace, metav1.GetOptions{})
return err
}
err = h.k8shelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc)
err = h.k8shelper.WaitForCustomResourceDeletion(namespace, namespace, crdCheckerFunc)
checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace))
err = h.k8shelper.DeleteResource("namespace", namespace)
+1 -1
View File
@@ -62,7 +62,7 @@ func (h *HelmHelper) InstallLocalRookHelmChart(namespace, chartSettings string)
return errors.Wrap(err, "failed to find rook root")
}
chartDir := path.Join(rootDir, "cluster/charts/rook-ceph/")
cmdArgs := []string{"install", HelmDeployName, chartDir, "-f", path.Join(chartDir, "values.yaml")}
cmdArgs := []string{"install", "--create-namespace", HelmDeployName, chartDir, "-f", path.Join(chartDir, "values.yaml")}
if namespace != "" {
cmdArgs = append(cmdArgs, "--namespace", namespace)
}
+21 -5
View File
@@ -347,7 +347,12 @@ func (k8sh *K8sHelper) ResourceOperationFromTemplate(action string, podDefinitio
// ResourceOperation performs a kubectl action on a pod definition
func (k8sh *K8sHelper) ResourceOperation(action string, manifest string) error {
args := []string{action, "-f", "-"}
logger.Infof("kubectl %s manifest:\n%s", action, manifest)
maxManifestCharsToPrint := 4000
if len(manifest) > maxManifestCharsToPrint {
logger.Infof("kubectl %s manifest (too long to print)", action)
} else {
logger.Infof("kubectl %s manifest:\n%s", action, manifest)
}
_, err := k8sh.KubectlWithStdin(manifest, args...)
if err == nil {
return nil
@@ -381,23 +386,23 @@ func (k8sh *K8sHelper) DeleteResource(args ...string) error {
}
// WaitForCustomResourceDeletion waits for the CRD deletion
func (k8sh *K8sHelper) WaitForCustomResourceDeletion(namespace string, checkerFunc func() error) error {
func (k8sh *K8sHelper) WaitForCustomResourceDeletion(namespace, name string, checkerFunc func() error) error {
// wait for the operator to finalize and delete the CRD
for i := 0; i < 60; i++ {
err := checkerFunc()
if err == nil {
logger.Infof("custom resource %s still exists", namespace)
logger.Infof("custom resource %q in namespace %q still exists", name, namespace)
time.Sleep(2 * time.Second)
continue
}
if kerrors.IsNotFound(err) {
logger.Infof("custom resource %s deleted", namespace)
logger.Infof("custom resource %q in namespace %s deleted", name, namespace)
return nil
}
return err
}
logger.Errorf("gave up deleting custom resource %s", namespace)
logger.Errorf("gave up deleting custom resource %q ", name)
return nil
}
@@ -931,6 +936,16 @@ func (k8sh *K8sHelper) PrintPVCs(namespace string, detailed bool) {
}
}
func (k8sh *K8sHelper) PrintResources(namespace, name string) {
args := []string{"-n", namespace, "get", name, "-o", "yaml"}
result, err := k8sh.Kubectl(args...)
if err != nil {
logger.Warningf("failed to get resource %s. %v", name, err)
} else {
logger.Infof("%s\n", result)
}
}
func (k8sh *K8sHelper) PrintStorageClasses(detailed bool) {
ctx := context.TODO()
scs, err := k8sh.Clientset.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{})
@@ -1218,6 +1233,7 @@ func (k8sh *K8sHelper) IsPodInExpectedState(podNamePattern string, namespace str
}
}
logger.Infof("waiting for pod with label app=%s in namespace %q to be in state %q...", podNamePattern, namespace, state)
time.Sleep(RetryInterval * time.Second)
}
+11 -13
View File
@@ -24,7 +24,6 @@ import (
"testing"
"time"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/tests/framework/clients"
@@ -269,8 +268,6 @@ func runBlockCSITest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.
poolName := "replicapool"
storageClassName := "rook-ceph-block"
blockName := "block-pv-claim"
systemNamespace := installer.SystemNamespace(namespace)
podNameWithPVRetained := "block-test-retained"
poolNameRetained := "replicapoolretained"
storageClassNameRetained := "rook-ceph-block-retained"
@@ -287,10 +284,10 @@ func runBlockCSITest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.
assert.Equal(s.T(), 0, len(initBlockImages), "there should not already be any images in the pool")
logger.Infof("step 1: Create block storage")
err := helper.BlockClient.CreateStorageClassAndPVC(true, defaultNamespace, namespace, systemNamespace, poolName, storageClassName, "Delete", blockName, "ReadWriteOnce")
err := helper.BlockClient.CreateStorageClassAndPVC(defaultNamespace, poolName, storageClassName, "Delete", blockName, "ReadWriteOnce")
require.NoError(s.T(), err)
require.NoError(s.T(), retryBlockImageCountCheck(helper, clusterInfo, 1), "Make sure a new block is created")
err = helper.BlockClient.CreateStorageClassAndPVC(true, defaultNamespace, namespace, systemNamespace, poolNameRetained, storageClassNameRetained, "Retain", blockNameRetained, "ReadWriteOnce")
err = helper.BlockClient.CreateStorageClassAndPVC(defaultNamespace, poolNameRetained, storageClassNameRetained, "Retain", blockNameRetained, "ReadWriteOnce")
require.NoError(s.T(), err)
require.NoError(s.T(), retryBlockImageCountCheck(helper, clusterInfo, 2), "Make sure another new block is created")
logger.Infof("Block Storage created successfully")
@@ -405,7 +402,8 @@ func restartOSDPods(k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
options := metav1.DeleteOptions{}
err := k8sh.Clientset.CoreV1().Pods(namespace).Delete(ctx, pod.Name, options)
assert.NoError(s.T(), err)
}
for _, pod := range pods.Items {
logger.Infof("Waiting for osd pod %s to be deleted", pod.Name)
deleted := k8sh.WaitUntilPodIsDeleted(pod.Name, namespace)
assert.True(s.T(), deleted)
@@ -417,20 +415,20 @@ func restartOSDPods(k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
assert.NoError(s.T(), err)
}
func runBlockCSITestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, clusterNamespace, systemNamespace string, version cephv1.CephVersionSpec) {
func runBlockCSITestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings) {
checkSkipCSITest(s.T(), k8sh)
logger.Infof("Block Storage End to End Integration Test - create storageclass,pool and pvc")
logger.Infof("Running on Rook Cluster %s", clusterNamespace)
clusterInfo := client.AdminClusterInfo(clusterNamespace)
logger.Infof("Running on Rook Cluster %s", settings.Namespace)
clusterInfo := client.AdminClusterInfo(settings.Namespace)
poolName := "rookpool"
storageClassName := "rook-ceph-block-lite"
blockName := "test-block-claim-lite"
podName := "test-pod-lite"
defer blockTestDataCleanUp(helper, k8sh, s, clusterInfo, poolName, storageClassName, blockName, podName, true)
setupBlockLite(helper, k8sh, s, clusterInfo, systemNamespace, poolName, storageClassName, blockName, podName, version)
setupBlockLite(helper, k8sh, s, clusterInfo, poolName, storageClassName, blockName, podName)
if !skipSnapshotTest(k8sh) {
blockCSISnapshotTest(helper, k8sh, s, storageClassName, clusterNamespace)
blockCSISnapshotTest(helper, k8sh, s, storageClassName, settings.Namespace)
}
if !skipCloneTest(k8sh) {
@@ -439,7 +437,7 @@ func runBlockCSITestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s su
}
func setupBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, clusterInfo *client.ClusterInfo,
systemNamespace, poolName, storageClassName, blockName, podName string, version cephv1.CephVersionSpec) {
poolName, storageClassName, blockName, podName string) {
// Check initial number of blocks
initialBlocks, err := helper.BlockClient.ListAllImages(clusterInfo)
@@ -449,7 +447,7 @@ func setupBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S
logger.Infof("step : Create Pool,StorageClass and PVC")
err = helper.BlockClient.CreateStorageClassAndPVC(true, defaultNamespace, clusterInfo.Namespace, systemNamespace, poolName, storageClassName, "Delete", blockName, "ReadWriteOnce")
err = helper.BlockClient.CreateStorageClassAndPVC(defaultNamespace, poolName, storageClassName, "Delete", blockName, "ReadWriteOnce")
require.NoError(s.T(), err)
require.True(s.T(), k8sh.WaitUntilPVCIsBound(defaultNamespace, blockName))
+15 -60
View File
@@ -25,7 +25,6 @@ import (
"testing"
"github.com/coreos/pkg/capnslog"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
@@ -87,36 +86,15 @@ func checkIfRookClusterIsHealthy(s suite.Suite, testClient *clients.TestClient,
require.Nil(s.T(), err)
}
func HandlePanics(r interface{}, op installer.TestSuite, t func() *testing.T) {
func HandlePanics(r interface{}, uninstaller func(), t func() *testing.T) {
if r != nil {
logger.Infof("unexpected panic occurred during test %s, --> %v", t().Name(), r)
t().Fail()
op.Teardown()
uninstaller()
t().FailNow()
}
}
// TestCluster struct for handling panic and test suite tear down
type TestCluster struct {
installer *installer.CephInstaller
kh *utils.K8sHelper
helper *clients.TestClient
T func() *testing.T
clusterName string
namespace string
storeType string
storageClassName string
useHelm bool
usePVC bool
mons int
rbdMirrorWorkers int
rookCephCleanup bool
skipOSDCreation bool
minimalMatrixK8sVersion string
rookVersion string
cephVersion cephv1.CephVersionSpec
}
func checkIfShouldRunForMinimalTestMatrix(t func() *testing.T, k8sh *utils.K8sHelper, version string) {
testArgs := os.Getenv("TEST_ARGUMENTS")
if !strings.Contains(testArgs, "min-test-matrix") {
@@ -140,50 +118,27 @@ func checkIfShouldRunForMinimalTestMatrix(t func() *testing.T, k8sh *utils.K8sHe
logger.Infof("Running test suite since kube version is %q", kubeVersion)
}
// StartTestCluster creates new instance of TestCluster struct
func StartTestCluster(t func() *testing.T, cluster *TestCluster) (*TestCluster, *utils.K8sHelper) {
kh, err := utils.CreateK8sHelper(t)
// StartTestCluster creates new instance of TestCephSettings struct
func StartTestCluster(t func() *testing.T, settings *installer.TestCephSettings, minimalMatrixK8sVersion string) (*installer.CephInstaller, *utils.K8sHelper) {
k8shelper, err := utils.CreateK8sHelper(t)
require.NoError(t(), err)
checkIfShouldRunForMinimalTestMatrix(t, kh, cluster.minimalMatrixK8sVersion)
checkIfShouldRunForMinimalTestMatrix(t, k8shelper, minimalMatrixK8sVersion)
cluster.installer = installer.NewCephInstaller(t, kh.Clientset, cluster.useHelm, cluster.clusterName, cluster.rookVersion, cluster.cephVersion, cluster.rookCephCleanup)
cluster.kh = kh
cluster.helper = nil
cluster.T = t
if cluster.rookVersion != installer.VersionMaster {
// make sure we have the images from a previous release locally so the test doesn't hit a timeout
assert.NoError(t(), kh.GetDockerImage("rook/ceph:"+cluster.rookVersion))
}
assert.NoError(t(), kh.GetDockerImage(cluster.cephVersion.Image))
cluster.Setup()
return cluster, kh
}
// Setup is a wrapper for setting up rook
func (op *TestCluster) Setup() {
// Turn on DEBUG logging
capnslog.SetGlobalLogLevel(capnslog.DEBUG)
isRookInstalled, err := op.installer.InstallRook(op.namespace, op.storeType, op.usePVC, op.storageClassName,
cephv1.MonSpec{Count: op.mons, AllowMultiplePerNode: true}, false /* startWithAllNodes */, op.rbdMirrorWorkers, op.skipOSDCreation, op.rookVersion)
installer := installer.NewCephInstaller(t, k8shelper.Clientset, settings)
isRookInstalled, err := installer.InstallRook()
if !isRookInstalled || err != nil {
logger.Errorf("Rook was not installed successfully: %v", err)
if !op.installer.T().Failed() {
op.installer.GatherAllRookLogs(op.installer.T().Name(), op.namespace, installer.SystemNamespace(op.namespace))
if !installer.T().Failed() {
installer.GatherAllRookLogs(t().Name(), settings.Namespace, settings.OperatorNamespace)
}
op.T().Fail()
op.Teardown()
op.T().FailNow()
t().Fail()
installer.UninstallRook()
t().FailNow()
}
}
// SetInstallData updates the installer helper based on the version of Rook desired
func (op *TestCluster) SetInstallData(version string) {}
// Teardown is a wrapper for tearDown after Suite
func (op *TestCluster) Teardown() {
op.installer.UninstallRook(op.namespace)
return installer, k8shelper
}
+60 -62
View File
@@ -239,83 +239,83 @@ func fileSystemCSISnapshotTest(helper *clients.TestClient, k8sh *utils.K8sHelper
// Smoke Test for File System Storage - Test check the following operations on Filesystem Storage in order
// Create,Mount,Write,Read,Unmount and Delete.
func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName string, useCSI bool, preserveFilesystemOnDelete bool) {
if useCSI {
func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string, preserveFilesystemOnDelete bool) {
if settings.UseCSI {
checkSkipCSITest(s.T(), k8sh)
}
defer fileTestDataCleanUp(helper, k8sh, s, filePodName, namespace, filesystemName)
logger.Infof("Running on Rook Cluster %s", namespace)
defer fileTestDataCleanUp(helper, k8sh, s, filePodName, settings.Namespace, filesystemName)
logger.Infof("Running on Rook Cluster %s", settings.Namespace)
logger.Infof("File Storage End To End Integration Test - create, mount, write to, read from, and unmount")
activeCount := 2
createFilesystem(helper, k8sh, s, namespace, filesystemName, activeCount)
createFilesystem(helper, k8sh, s, settings, filesystemName, activeCount)
if preserveFilesystemOnDelete {
_, err := k8sh.Kubectl("-n", namespace, "patch", "CephFilesystem", filesystemName, "--type=merge", "-p", `{"spec": {"preserveFilesystemOnDelete": true}}`)
_, err := k8sh.Kubectl("-n", settings.Namespace, "patch", "CephFilesystem", filesystemName, "--type=merge", "-p", `{"spec": {"preserveFilesystemOnDelete": true}}`)
assert.NoError(s.T(), err)
}
// Create a test pod where CephFS is consumed without user creds
storageClassName := "cephfs-storageclass"
err := helper.FSClient.CreateStorageClass(filesystemName, installer.SystemNamespace(namespace), namespace, storageClassName)
err := helper.FSClient.CreateStorageClass(filesystemName, settings.OperatorNamespace, settings.Namespace, storageClassName)
assert.NoError(s.T(), err)
createFilesystemConsumerPod(helper, k8sh, s, namespace, filesystemName, storageClassName, useCSI)
createFilesystemConsumerPod(helper, k8sh, s, settings, filesystemName, storageClassName)
// Test reading and writing to the first pod
err = writeAndReadToFilesystem(helper, k8sh, s, namespace, filePodName, "test_file")
err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, filePodName, "test_file")
assert.NoError(s.T(), err)
// TODO: Also mount with user credentials with the CSI driver
if !useCSI {
if !settings.UseCSI {
// Create a test pod where CephFS is consumed with a mountUser and mountSecret specified.
createFilesystemMountCephCredentials(helper, k8sh, s, namespace, filesystemName)
createFilesystemMountUserConsumerPod(helper, k8sh, s, namespace, filesystemName, storageClassName)
createFilesystemMountCephCredentials(helper, k8sh, s, settings, filesystemName)
createFilesystemMountUserConsumerPod(helper, k8sh, s, settings, filesystemName, storageClassName)
// Test reading and writing to the second pod
err = writeAndReadToFilesystem(helper, k8sh, s, namespace, fileMountUserPodName, "canttouchthis")
err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, fileMountUserPodName, "canttouchthis")
assert.Error(s.T(), err, "we should not be able to write to file canttouchthis on CephFS `/`")
err = writeAndReadToFilesystem(helper, k8sh, s, namespace, fileMountUserPodName, "foo/test_file")
err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, fileMountUserPodName, "foo/test_file")
assert.NoError(s.T(), err, "we should be able to write to the `/foo` directory on CephFS")
cleanupFilesystemConsumer(helper, k8sh, s, namespace, fileMountUserPodName)
cleanupFilesystemConsumer(helper, k8sh, s, settings.Namespace, fileMountUserPodName)
assert.NoError(s.T(), err)
}
// Start the NFS daemons
testNFSDaemons(helper, k8sh, s, namespace, filesystemName)
testNFSDaemons(helper, k8sh, s, settings, filesystemName)
// Cleanup the filesystem and its clients
cleanupFilesystemConsumer(helper, k8sh, s, namespace, filePodName)
cleanupFilesystemConsumer(helper, k8sh, s, settings.Namespace, filePodName)
assert.NoError(s.T(), err)
downscaleMetadataServers(helper, k8sh, s, namespace, filesystemName)
cleanupFilesystem(helper, k8sh, s, namespace, filesystemName)
downscaleMetadataServers(helper, k8sh, s, settings.Namespace, filesystemName)
cleanupFilesystem(helper, k8sh, s, settings.Namespace, filesystemName)
err = helper.FSClient.DeleteStorageClass(storageClassName)
assertNoErrorUnlessNotFound(s, err)
if preserveFilesystemOnDelete {
fses, err := helper.FSClient.List(namespace)
fses, err := helper.FSClient.List(settings.Namespace)
assert.NoError(s.T(), err)
assert.Len(s.T(), fses, 1)
assert.Equal(s.T(), fses[0].Name, filesystemName)
err = helper.FSClient.Delete(filesystemName, namespace)
err = helper.FSClient.Delete(filesystemName, settings.Namespace)
assert.NoError(s.T(), err)
}
}
func testNFSDaemons(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) {
func testNFSDaemons(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) {
name := "my-nfs"
err := helper.NFSClient.Create(namespace, name, filesystemName+"-data0", 2)
err := helper.NFSClient.Create(settings.Namespace, name, filesystemName+"-data0", 2)
require.Nil(s.T(), err)
err = helper.NFSClient.Delete(namespace, name)
err = helper.NFSClient.Delete(settings.Namespace, name)
assert.Nil(s.T(), err)
}
func createFilesystemConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName, storageClassName string, useCSI bool) {
err := createPodWithFilesystem(k8sh, s, filePodName, namespace, filesystemName, storageClassName, false, useCSI)
func createFilesystemConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName, storageClassName string) {
err := createPodWithFilesystem(k8sh, s, settings, filePodName, filesystemName, storageClassName, false)
require.NoError(s.T(), err)
filePodRunning := k8sh.IsPodRunning(filePodName, namespace)
filePodRunning := k8sh.IsPodRunning(filePodName, settings.Namespace)
require.True(s.T(), filePodRunning, "make sure file-test pod is in running state")
logger.Infof("File system mounted successfully")
}
@@ -358,36 +358,36 @@ func cleanupFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suit
}
// Test File System Creation on Rook that was installed on a custom namespace i.e. Namespace != "rook" and delete it again
func runFileE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) {
func runFileE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) {
checkSkipCSITest(s.T(), k8sh)
logger.Infof("File Storage End to End Integration Test - create Filesystem and make sure mds pod is running")
logger.Infof("Running on Rook Cluster %s", namespace)
logger.Infof("Running on Rook Cluster %s", settings.Namespace)
activeCount := 1
createFilesystem(helper, k8sh, s, namespace, filesystemName, activeCount)
createFilesystem(helper, k8sh, s, settings, filesystemName, activeCount)
// Create a test pod where CephFS is consumed without user creds
storageClassName := "cephfs-storageclass"
err := helper.FSClient.CreateStorageClass(filesystemName, namespace, namespace, storageClassName)
err := helper.FSClient.CreateStorageClass(filesystemName, settings.OperatorNamespace, settings.Namespace, storageClassName)
assert.NoError(s.T(), err)
assert.NoError(s.T(), err)
if !skipSnapshotTest(k8sh) {
fileSystemCSISnapshotTest(helper, k8sh, s, storageClassName, namespace)
fileSystemCSISnapshotTest(helper, k8sh, s, storageClassName, settings.Namespace)
}
if !skipCloneTest(k8sh) {
fileSystemCSICloneTest(helper, k8sh, s, storageClassName, namespace)
fileSystemCSICloneTest(helper, k8sh, s, storageClassName, settings.Namespace)
}
cleanupFilesystem(helper, k8sh, s, namespace, filesystemName)
cleanupFilesystem(helper, k8sh, s, settings.Namespace, filesystemName)
err = helper.FSClient.DeleteStorageClass(storageClassName)
assertNoErrorUnlessNotFound(s, err)
}
func createFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName string, activeCount int) {
func createFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string, activeCount int) {
logger.Infof("Create file System")
fscErr := helper.FSClient.Create(filesystemName, namespace, activeCount)
fscErr := helper.FSClient.Create(filesystemName, settings.Namespace, activeCount)
require.Nil(s.T(), fscErr)
logger.Infof("File system %s created", filesystemName)
filesystemList, _ := helper.FSClient.List(namespace)
filesystemList, _ := helper.FSClient.List(settings.Namespace)
require.Equal(s.T(), 1, len(filesystemList), "There should be one shared file system present")
}
@@ -399,13 +399,12 @@ func fileTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s su
assert.NoError(s.T(), err)
}
func createPodWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podName, namespace, filesystemName, storageClassName string, mountUser, useCSI bool) error {
func createPodWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, podName, filesystemName, storageClassName string, mountUser bool) error {
var testPodManifest string
if useCSI {
testPodManifest = getFilesystemCSITestPod(podName, namespace, storageClassName)
if settings.UseCSI {
testPodManifest = getFilesystemCSITestPod(settings, podName, storageClassName)
} else {
driverName := installer.SystemNamespace(namespace)
testPodManifest = getFilesystemFlexTestPod(podName, namespace, filesystemName, driverName, mountUser)
testPodManifest = getFilesystemFlexTestPod(settings, podName, filesystemName, mountUser)
}
if err := k8sh.ResourceOperation("create", testPodManifest); err != nil {
return fmt.Errorf("failed to create pod -- %s. %+v", testPodManifest, err)
@@ -413,7 +412,7 @@ func createPodWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podName, name
return nil
}
func getFilesystemFlexTestPod(podName, namespace, filesystemName, driverName string, mountUser bool) string {
func getFilesystemFlexTestPod(settings *installer.TestCephSettings, podName, filesystemName string, mountUser bool) string {
mountUserInsert := ""
if mountUser {
mountUserInsert = `
@@ -425,7 +424,7 @@ func getFilesystemFlexTestPod(podName, namespace, filesystemName, driverName str
kind: Pod
metadata:
name: ` + podName + `
namespace: ` + namespace + `
namespace: ` + settings.Namespace + `
spec:
containers:
- name: ` + podName + `
@@ -444,23 +443,23 @@ spec:
volumes:
- name: ` + filesystemName + `
flexVolume:
driver: ceph.rook.io/` + driverName + `
driver: ceph.rook.io/` + settings.OperatorNamespace + `
fsType: ceph
options:
fsName: ` + filesystemName + `
clusterNamespace: ` + namespace + mountUserInsert + `
clusterNamespace: ` + settings.Namespace + mountUserInsert + `
restartPolicy: Always
`
}
func getFilesystemCSITestPod(podName, namespace, storageClassName string) string {
func getFilesystemCSITestPod(settings *installer.TestCephSettings, podName, storageClassName string) string {
claimName := podName
return `
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: ` + claimName + `
namespace: ` + namespace + `
namespace: ` + settings.Namespace + `
spec:
accessModes:
- ReadWriteOnce
@@ -473,7 +472,7 @@ apiVersion: v1
kind: Pod
metadata:
name: ` + podName + `
namespace: ` + namespace + `
namespace: ` + settings.Namespace + `
spec:
containers:
- name: ` + podName + `
@@ -496,20 +495,20 @@ spec:
`
}
func createFilesystemMountCephCredentials(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) {
func createFilesystemMountCephCredentials(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) {
ctx := context.TODO()
// Create agent binding for access to Secrets
err := k8sh.ResourceOperation("apply", getFilesystemAgentMountSecretsBinding(namespace))
err := k8sh.ResourceOperation("apply", getFilesystemAgentMountSecretsBinding(settings.Namespace))
require.Nil(s.T(), err)
// Mount CephFS in toolbox and create /foo directory on it
logger.Info("Creating /foo directory on CephFS")
_, err = k8sh.Exec(namespace, "rook-ceph-tools", "mkdir", []string{"-p", utils.TestMountPath})
_, err = k8sh.Exec(settings.Namespace, client.RunAllCephCommandsInToolboxPod, "mkdir", []string{"-p", utils.TestMountPath})
require.Nil(s.T(), err)
_, err = k8sh.ExecWithRetry(10, namespace, "rook-ceph-tools", "bash", []string{"-c", fmt.Sprintf("mount -t ceph -o mds_namespace=%s,name=admin,secret=$(grep key /etc/ceph/keyring | awk '{print $3}') $(grep mon_host /etc/ceph/ceph.conf | awk '{print $3}'):/ %s", filesystemName, utils.TestMountPath)})
_, err = k8sh.ExecWithRetry(10, settings.Namespace, client.RunAllCephCommandsInToolboxPod, "bash", []string{"-c", fmt.Sprintf("mount -t ceph -o mds_namespace=%s,name=admin,secret=$(grep key /etc/ceph/keyring | awk '{print $3}') $(grep mon_host /etc/ceph/ceph.conf | awk '{print $3}'):/ %s", filesystemName, utils.TestMountPath)})
require.Nil(s.T(), err)
_, err = k8sh.Exec(namespace, "rook-ceph-tools", "mkdir", []string{"-p", fmt.Sprintf("%s/foo", utils.TestMountPath)})
_, err = k8sh.Exec(settings.Namespace, client.RunAllCephCommandsInToolboxPod, "mkdir", []string{"-p", fmt.Sprintf("%s/foo", utils.TestMountPath)})
require.Nil(s.T(), err)
_, err = k8sh.Exec(namespace, "rook-ceph-tools", "umount", []string{utils.TestMountPath})
_, err = k8sh.Exec(settings.Namespace, client.RunAllCephCommandsInToolboxPod, "umount", []string{utils.TestMountPath})
require.Nil(s.T(), err)
logger.Info("Created /foo directory on CephFS")
@@ -523,15 +522,15 @@ func createFilesystemMountCephCredentials(helper *clients.TestClient, k8sh *util
),
}
logger.Infof("ceph credentials command args: %s", commandArgs[1])
result, err := k8sh.Exec(namespace, "rook-ceph-tools", "bash", commandArgs)
result, err := k8sh.Exec(settings.Namespace, client.RunAllCephCommandsInToolboxPod, "bash", commandArgs)
logger.Infof("Ceph filesystem credentials output: %s", result)
logger.Info("Created Ceph credentials")
require.Nil(s.T(), err)
// Save Ceph credentials to Kubernetes
_, err = k8sh.Clientset.CoreV1().Secrets(namespace).Create(ctx, &v1.Secret{
_, err = k8sh.Clientset.CoreV1().Secrets(settings.Namespace).Create(ctx, &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: fileMountSecret,
Namespace: namespace,
Namespace: settings.Namespace,
},
Data: map[string][]byte{
"mykey": []byte(result),
@@ -541,12 +540,11 @@ func createFilesystemMountCephCredentials(helper *clients.TestClient, k8sh *util
logger.Info("Created Ceph credentials Secret in Kubernetes")
}
func createFilesystemMountUserConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName, storageClassName string) {
func createFilesystemMountUserConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName, storageClassName string) {
// TODO: Mount with user credentials for the CSI driver
useCSI := false
mtfsErr := createPodWithFilesystem(k8sh, s, fileMountUserPodName, namespace, filesystemName, storageClassName, true, useCSI)
mtfsErr := createPodWithFilesystem(k8sh, s, settings, fileMountUserPodName, filesystemName, storageClassName, true)
require.Nil(s.T(), mtfsErr)
filePodRunning := k8sh.IsPodRunning(fileMountUserPodName, namespace)
filePodRunning := k8sh.IsPodRunning(fileMountUserPodName, settings.Namespace)
require.True(s.T(), filePodRunning, "make sure file-mountuser-test pod is in running state")
logger.Infof("File system mounted successfully")
}
+29 -29
View File
@@ -24,9 +24,9 @@ import (
"github.com/rook/rook/pkg/daemon/ceph/client"
rgw "github.com/rook/rook/pkg/operator/ceph/object"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -59,11 +59,11 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
logger.Infof("Step 0 : Create Object Store User")
cosuErr := helper.ObjectUserClient.Create(namespace, userid, userdisplayname, storeName)
require.Nil(s.T(), cosuErr)
assert.Nil(s.T(), cosuErr)
logger.Infof("Step 1 : Create Object Store")
cobsErr := helper.ObjectClient.Create(namespace, storeName, 3)
require.Nil(s.T(), cobsErr)
assert.Nil(s.T(), cobsErr)
// check that ObjectStore is created
logger.Infof("Check that RGW pods are Running")
@@ -86,7 +86,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
assert.True(s.T(), helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid))
userInfo, err := helper.ObjectUserClient.GetUser(namespace, storeName, userid)
require.NoError(s.T(), err)
assert.NoError(s.T(), err)
assert.Equal(s.T(), userid, userInfo.UserID)
assert.Equal(s.T(), userdisplayname, *userInfo.DisplayName)
logger.Infof("Done creating object store user")
@@ -107,11 +107,11 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
assert.NotEmpty(s.T(), objectStore.Status.Info["endpoint"])
break
}
require.NotEqual(s.T(), i, 4)
assert.NotEqual(s.T(), i, 4)
logger.Infof("Step 2 : Test Deleting User")
dosuErr := helper.ObjectUserClient.Delete(namespace, userid)
require.Nil(s.T(), dosuErr)
assert.Nil(s.T(), dosuErr)
logger.Infof("Object store user deleted successfully")
logger.Infof("Checking to see if the user secret has been deleted")
for i = 0; i < 4 && helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid) == true; i++ {
@@ -119,7 +119,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
time.Sleep(5 * time.Second)
}
assert.False(s.T(), helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid))
require.NotEqual(s.T(), i, 4)
assert.NotEqual(s.T(), i, 4)
logger.Infof("Check that MGRs are not in a crashloop")
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mgr", namespace, 1, "Running"))
@@ -129,14 +129,14 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
logger.Infof("Step 3 : Create Object Bucket Claim with reclaim policy delete")
bucketStorageClassName := "rook-smoke-delete-bucket"
cobErr := helper.BucketClient.CreateBucketStorageClass(namespace, storeName, bucketStorageClassName, "Delete", region)
require.Nil(s.T(), cobErr)
assert.Nil(s.T(), cobErr)
cobcErr := helper.BucketClient.CreateObc(obcName, bucketStorageClassName, bucketname, maxObject, true)
require.Nil(s.T(), cobcErr)
assert.Nil(s.T(), cobcErr)
created := utils.Retry(12, 2*time.Second, "OBC is created", func() bool {
return helper.BucketClient.CheckOBC(obcName, "bound")
})
require.True(s.T(), created)
assert.True(s.T(), created)
logger.Infof("Check if bucket was created")
context := k8sh.MakeContext()
@@ -153,7 +153,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
time.Sleep(5 * time.Second)
}
assert.NotEqual(s.T(), i, 4)
require.Equal(s.T(), bucketname, bkt.Name)
assert.Equal(s.T(), bucketname, bkt.Name)
logger.Infof("OBC, Secret and ConfigMap created")
logger.Infof("Step 4 : Create s3 client")
@@ -161,37 +161,37 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true)
require.Nil(s.T(), err)
assert.Nil(s.T(), err)
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
logger.Infof("Step 5 : Put Object on bucket")
_, poErr := s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey1, contentType)
require.Nil(s.T(), poErr)
assert.Nil(s.T(), poErr)
logger.Infof("Step 6 : Get Object from bucket")
read, err := s3client.GetObjectInBucket(bucketname, ObjectKey1)
require.Nil(s.T(), err)
require.Equal(s.T(), ObjBody, read)
assert.Nil(s.T(), err)
assert.Equal(s.T(), ObjBody, read)
logger.Infof("Object Created and Retrieved on bucket successfully")
logger.Infof("Step 7 : Testing Quota for the OBC")
logger.Infof("Adding one more object to the bucket")
_, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey2, contentType)
require.Nil(s.T(), poErr)
assert.Nil(s.T(), poErr)
logger.Infof("Testing the max object limit")
_, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey3, contentType)
require.Error(s.T(), poErr)
assert.Error(s.T(), poErr)
logger.Infof("Step 8 : Delete objects on bucket")
_, delobjErr := s3client.DeleteObjectInBucket(bucketname, ObjectKey1)
require.Nil(s.T(), delobjErr)
assert.Nil(s.T(), delobjErr)
_, delobjErr = s3client.DeleteObjectInBucket(bucketname, ObjectKey2)
require.Nil(s.T(), delobjErr)
assert.Nil(s.T(), delobjErr)
logger.Infof("Objects deleted on bucket successfully")
logger.Infof("Step 9 : Delete Object Bucket Claim")
dobcErr := helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketname, maxObject, true)
require.Nil(s.T(), dobcErr)
assert.Nil(s.T(), dobcErr)
logger.Infof("Checking to see if the obc, secret and cm have all been deleted")
for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "deleted"); i++ {
logger.Infof("(%d) obc deleted check, sleeping for 5 seconds ...", i)
@@ -220,27 +220,27 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
}
// Test Object StoreCreation on Rook that was installed via helm
func runObjectE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, name string, replicaSize int, deleteStore bool) {
func runObjectE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, name string, replicaSize int, deleteStore bool) {
logger.Infof("Object Storage End To End Integration Test - Create Object Store and check if rgw service is Running")
logger.Infof("Running on Rook Cluster %s", namespace)
logger.Infof("Running on Rook Cluster %s", settings.Namespace)
logger.Infof("Step 1 : Create Object Store")
err := helper.ObjectClient.Create(namespace, name, int32(replicaSize))
require.Nil(s.T(), err)
err := helper.ObjectClient.Create(settings.Namespace, name, int32(replicaSize))
assert.Nil(s.T(), err)
logger.Infof("Step 2 : check rook-ceph-rgw service status and count")
require.True(s.T(), k8sh.IsPodInExpectedState("rook-ceph-rgw", namespace, "Running"),
assert.True(s.T(), k8sh.IsPodInExpectedState("rook-ceph-rgw", settings.Namespace, "Running"),
"Make sure rook-ceph-rgw is in running state")
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-rgw", namespace, replicaSize, "Running"),
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-rgw", settings.Namespace, replicaSize, "Running"),
"Make sure all rook-ceph-rgw pods are in Running state")
require.True(s.T(), k8sh.IsServiceUp("rook-ceph-rgw-"+name, namespace))
assert.True(s.T(), k8sh.IsServiceUp("rook-ceph-rgw-"+name, settings.Namespace))
if deleteStore {
logger.Infof("Delete Object Store")
err = helper.ObjectClient.Delete(namespace, name)
require.Nil(s.T(), err)
err = helper.ObjectClient.Delete(settings.Namespace, name)
assert.Nil(s.T(), err)
logger.Infof("Done deleting object store")
}
}
+33 -37
View File
@@ -61,7 +61,7 @@ func TestCephFlexSuite(t *testing.T) {
s := new(CephFlexDriverSuite)
defer func(s *CephFlexDriverSuite) {
HandlePanics(recover(), s.op, s.T)
HandlePanics(recover(), s.TearDownSuite, s.T)
}(s)
suite.Run(t, s)
}
@@ -72,47 +72,45 @@ type CephFlexDriverSuite struct {
clusterInfo *client.ClusterInfo
bc *clients.BlockOperation
kh *utils.K8sHelper
namespace string
installer *installer.CephInstaller
settings *installer.TestCephSettings
pvcNameRWO string
pvcNameRWX string
op *TestCluster
}
func (s *CephFlexDriverSuite) SetupSuite() {
s.namespace = "flex-ns"
namespace := "flex-ns"
s.pvcNameRWO = "block-persistent-rwo"
s.pvcNameRWX = "block-persistent-rwx"
flexTestCluster := TestCluster{
namespace: s.namespace,
storeType: "bluestore",
storageClassName: "",
useHelm: false,
usePVC: false,
mons: 1,
rbdMirrorWorkers: 1,
rookCephCleanup: true,
skipOSDCreation: false,
minimalMatrixK8sVersion: flexDriverMinimalTestVersion,
rookVersion: installer.VersionMaster,
cephVersion: installer.OctopusVersion,
s.settings = &installer.TestCephSettings{
ClusterName: "flex-test",
Namespace: namespace,
OperatorNamespace: installer.SystemNamespace(namespace),
StorageClassName: "",
UseHelm: false,
UsePVC: false,
Mons: 1,
SkipOSDCreation: false,
UseCSI: false,
DirectMountToolbox: true,
RookVersion: installer.VersionMaster,
CephVersion: installer.OctopusVersion,
}
s.clusterInfo = client.AdminClusterInfo(s.namespace)
s.op, s.kh = StartTestCluster(s.T, &flexTestCluster)
s.testClient = clients.CreateTestClient(s.kh, s.op.installer.Manifests)
s.settings.ApplyEnvVars()
s.clusterInfo = client.AdminClusterInfo(namespace)
s.installer, s.kh = StartTestCluster(s.T, s.settings, flexDriverMinimalTestVersion)
s.testClient = clients.CreateTestClient(s.kh, s.installer.Manifests)
s.bc = s.testClient.BlockClient
}
func (s *CephFlexDriverSuite) AfterTest(suiteName, testName string) {
s.op.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(s.namespace))
s.installer.CollectOperatorLog(suiteName, testName)
}
func (s *CephFlexDriverSuite) TestFileSystem() {
useCSI := false
preserveFilesystemOnDelete := false
runFileE2ETest(s.testClient, s.kh, s.Suite, s.namespace, "smoke-test-fs", useCSI, preserveFilesystemOnDelete)
runFileE2ETest(s.testClient, s.kh, s.Suite, s.settings, "flex-fs", preserveFilesystemOnDelete)
}
func (s *CephFlexDriverSuite) TestBlockStorageMountUnMountForStatefulSets() {
@@ -127,9 +125,9 @@ func (s *CephFlexDriverSuite) TestBlockStorageMountUnMountForStatefulSets() {
logger.Infof("Test case when block persistent volumes are scaled up and down along with StatefulSet")
logger.Info("Step 1: Create pool and storageClass")
err := s.testClient.PoolClient.Create(poolName, s.namespace, 1)
err := s.testClient.PoolClient.Create(poolName, s.settings.Namespace, 1)
assert.Nil(s.T(), err)
err = s.testClient.BlockClient.CreateStorageClass(false, poolName, storageClassName, reclaimPolicy, s.namespace)
err = s.testClient.BlockClient.CreateStorageClass(false, poolName, storageClassName, reclaimPolicy, s.settings.Namespace)
assert.Nil(s.T(), err)
logger.Info("Step 2 : Deploy statefulSet with 1X replication")
service, statefulset := getBlockStatefulSetAndServiceDefinition(defaultNamespace, statefulSetName, statefulPodsName, storageClassName)
@@ -190,11 +188,9 @@ func (s *CephFlexDriverSuite) setupPVCs() {
logger.Infof("creating the test PVCs")
poolNameRWO := "block-pool-rwo"
storageClassNameRWO := "rook-ceph-block-rwo"
systemNamespace := installer.SystemNamespace(s.namespace)
// Create PVCs
useCSI := false
err := s.testClient.BlockClient.CreateStorageClassAndPVC(useCSI, defaultNamespace, s.namespace, systemNamespace, poolNameRWO, storageClassNameRWO, "Delete", s.pvcNameRWO, "ReadWriteOnce")
err := s.testClient.BlockClient.CreateStorageClassAndPVC(defaultNamespace, poolNameRWO, storageClassNameRWO, "Delete", s.pvcNameRWO, "ReadWriteOnce")
require.Nil(s.T(), err)
require.True(s.T(), s.kh.WaitUntilPVCIsBound(defaultNamespace, s.pvcNameRWO), "Make sure PVC is Bound")
@@ -207,13 +203,13 @@ func (s *CephFlexDriverSuite) setupPVCs() {
require.Nil(s.T(), err)
crdName, err := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWO)
require.Nil(s.T(), err)
s.kh.IsVolumeResourcePresent(systemNamespace, crdName)
s.kh.IsVolumeResourcePresent(s.settings.OperatorNamespace, crdName)
err = s.bc.CreateClientPod(getFlexBlockPodDefinition("setup-block-rwx", s.pvcNameRWX, false))
require.Nil(s.T(), err)
crdName, err = s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWX)
require.Nil(s.T(), err)
s.kh.IsVolumeResourcePresent(systemNamespace, crdName)
s.kh.IsVolumeResourcePresent(s.settings.OperatorNamespace, crdName)
require.True(s.T(), s.kh.IsPodRunning("setup-block-rwo", defaultNamespace), "make sure setup-block-rwo pod is in running state")
require.True(s.T(), s.kh.IsPodRunning("setup-block-rwx", defaultNamespace), "make sure setup-block-rwx pod is in running state")
@@ -239,9 +235,9 @@ func (s *CephFlexDriverSuite) TearDownSuite() {
"setup-block-rwo", "setup-block-rwx", "rwo-block-rw-one", "rwo-block-rw-two", "rwo-block-ro-one",
"rwo-block-ro-two", "rwx-block-rw-one", "rwx-block-rw-two", "rwx-block-ro-one", "rwx-block-ro-two")
assert.NoError(s.T(), err)
err = s.testClient.BlockClient.DeletePVC(s.namespace, s.pvcNameRWO)
err = s.testClient.BlockClient.DeletePVC(s.settings.Namespace, s.pvcNameRWO)
assertNoErrorUnlessNotFound(s.Suite, err)
err = s.testClient.BlockClient.DeletePVC(s.namespace, s.pvcNameRWX)
err = s.testClient.BlockClient.DeletePVC(s.settings.Namespace, s.pvcNameRWX)
assertNoErrorUnlessNotFound(s.Suite, err)
err = s.testClient.BlockClient.DeleteStorageClass("rook-ceph-block-rwo")
assert.NoError(s.T(), err)
@@ -251,7 +247,7 @@ func (s *CephFlexDriverSuite) TearDownSuite() {
assert.NoError(s.T(), err)
err = s.testClient.PoolClient.DeletePool(s.testClient.BlockClient, s.clusterInfo, "block-pool-rwx")
assert.NoError(s.T(), err)
s.op.Teardown()
s.installer.UninstallRook()
}
func (s *CephFlexDriverSuite) TestBlockStorageMountUnMountForDifferentAccessModes() {
@@ -267,12 +263,12 @@ func (s *CephFlexDriverSuite) TestBlockStorageMountUnMountForDifferentAccessMode
require.Nil(s.T(), err)
crdName, err := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWO)
assert.Nil(s.T(), err)
assert.True(s.T(), s.kh.IsVolumeResourcePresent(installer.SystemNamespace(s.namespace), crdName), fmt.Sprintf("make sure Volume %s is created", crdName))
assert.True(s.T(), s.kh.IsVolumeResourcePresent(s.settings.OperatorNamespace, crdName), fmt.Sprintf("make sure Volume %s is created", crdName))
assert.True(s.T(), s.kh.IsPodRunning("rwo-block-rw-one", defaultNamespace), "make sure block-rw-one pod is in running state")
crdName, err = s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWX)
assert.Nil(s.T(), err)
assert.True(s.T(), s.kh.IsVolumeResourcePresent(installer.SystemNamespace(s.namespace), crdName), fmt.Sprintf("make sure Volume %s is created", crdName))
assert.True(s.T(), s.kh.IsVolumeResourcePresent(s.settings.OperatorNamespace, crdName), fmt.Sprintf("make sure Volume %s is created", crdName))
assert.True(s.T(), s.kh.IsPodRunning("rwx-block-rw-one", defaultNamespace), "make sure rwx-block-rw-one pod is in running state")
logger.Infof("Step 2: Check if previously persisted data is readable from ReadWriteOnce and ReadWriteMany PVC")
+34 -34
View File
@@ -47,7 +47,7 @@ func TestCephHelmSuite(t *testing.T) {
s := new(HelmSuite)
defer func(s *HelmSuite) {
HandlePanics(recover(), s.op, s.T)
HandlePanics(recover(), s.TearDownSuite, s.T)
}(s)
suite.Run(t, s)
}
@@ -55,56 +55,56 @@ func TestCephHelmSuite(t *testing.T) {
type HelmSuite struct {
suite.Suite
helper *clients.TestClient
kh *utils.K8sHelper
op *TestCluster
namespace string
installer *installer.CephInstaller
settings *installer.TestCephSettings
k8shelper *utils.K8sHelper
}
func (hs *HelmSuite) SetupSuite() {
hs.namespace = "helm-ns"
helmTestCluster := TestCluster{
namespace: hs.namespace,
storeType: "bluestore",
storageClassName: "",
useHelm: true,
usePVC: false,
mons: 1,
rbdMirrorWorkers: 1,
rookCephCleanup: true,
skipOSDCreation: false,
minimalMatrixK8sVersion: helmMinimalTestVersion,
rookVersion: installer.VersionMaster,
cephVersion: installer.NautilusVersion,
func (h *HelmSuite) SetupSuite() {
namespace := "helm-ns"
h.settings = &installer.TestCephSettings{
Namespace: namespace,
OperatorNamespace: namespace,
StorageClassName: "",
UseHelm: true,
UsePVC: false,
Mons: 1,
UseCSI: true,
SkipOSDCreation: false,
EnableAdmissionController: true,
EnableDiscovery: true,
RookVersion: installer.VersionMaster,
CephVersion: installer.OctopusVersion,
}
hs.op, hs.kh = StartTestCluster(hs.T, &helmTestCluster)
hs.helper = clients.CreateTestClient(hs.kh, hs.op.installer.Manifests)
h.settings.ApplyEnvVars()
h.installer, h.k8shelper = StartTestCluster(h.T, h.settings, helmMinimalTestVersion)
h.helper = clients.CreateTestClient(h.k8shelper, h.installer.Manifests)
}
func (hs *HelmSuite) TearDownSuite() {
hs.op.Teardown()
func (h *HelmSuite) TearDownSuite() {
h.installer.UninstallRook()
}
func (hs *HelmSuite) AfterTest(suiteName, testName string) {
hs.op.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(hs.namespace))
func (h *HelmSuite) AfterTest(suiteName, testName string) {
h.installer.CollectOperatorLog(suiteName, testName)
}
// Test to make sure all rook components are installed and Running
func (hs *HelmSuite) TestARookInstallViaHelm() {
checkIfRookClusterIsInstalled(hs.Suite, hs.kh, hs.namespace, hs.namespace, 1)
func (h *HelmSuite) TestARookInstallViaHelm() {
checkIfRookClusterIsInstalled(h.Suite, h.k8shelper, h.settings.Namespace, h.settings.Namespace, 1)
}
// Test BlockCreation on Rook that was installed via Helm
func (hs *HelmSuite) TestBlockStoreOnRookInstalledViaHelm() {
runBlockCSITestLite(hs.helper, hs.kh, hs.Suite, hs.namespace, hs.namespace, hs.op.installer.CephVersion)
func (h *HelmSuite) TestBlockStoreOnRookInstalledViaHelm() {
runBlockCSITestLite(h.helper, h.k8shelper, h.Suite, h.settings)
}
// Test File System Creation on Rook that was installed via helm
func (hs *HelmSuite) TestFileStoreOnRookInstalledViaHelm() {
runFileE2ETestLite(hs.helper, hs.kh, hs.Suite, hs.namespace, "testfs")
func (h *HelmSuite) TestFileStoreOnRookInstalledViaHelm() {
runFileE2ETestLite(h.helper, h.k8shelper, h.Suite, h.settings, "testfs")
}
// Test Object StoreCreation on Rook that was installed via helm
func (hs *HelmSuite) TestObjectStoreOnRookInstalledViaHelm() {
runObjectE2ETestLite(hs.helper, hs.kh, hs.Suite, hs.namespace, "default", 3, true)
func (h *HelmSuite) TestObjectStoreOnRookInstalledViaHelm() {
runObjectE2ETestLite(h.helper, h.k8shelper, h.Suite, h.settings, "default", 3, true)
}
+60 -61
View File
@@ -55,15 +55,16 @@ func TestCephMgrSuite(t *testing.T) {
s := new(CephMgrSuite)
defer func(s *CephMgrSuite) {
HandlePanics(recover(), s.cluster, s.T)
HandlePanics(recover(), s.TearDownSuite, s.T)
}(s)
suite.Run(t, s)
}
type CephMgrSuite struct {
suite.Suite
cluster *TestCluster
settings *installer.TestCephSettings
k8sh *utils.K8sHelper
installer *installer.CephInstaller
namespace string
}
@@ -87,46 +88,44 @@ type service struct {
Status serviceStatus
}
func (suite *CephMgrSuite) SetupSuite() {
suite.namespace = "mgr-ns"
func (s *CephMgrSuite) SetupSuite() {
s.namespace = "mgr-ns"
mgrTestCluster := TestCluster{
clusterName: suite.namespace,
namespace: suite.namespace,
storeType: "bluestore",
storageClassName: "",
useHelm: false,
usePVC: false,
mons: 1,
rbdMirrorWorkers: 0,
rookCephCleanup: true,
skipOSDCreation: true,
minimalMatrixK8sVersion: cephMasterSuiteMinimalTestVersion,
rookVersion: installer.VersionMaster,
cephVersion: installer.MasterVersion,
s.settings = &installer.TestCephSettings{
ClusterName: s.namespace,
OperatorNamespace: installer.SystemNamespace(s.namespace),
Namespace: s.namespace,
StorageClassName: "",
UseHelm: false,
UsePVC: false,
Mons: 1,
UseCSI: true,
SkipOSDCreation: true,
RookVersion: installer.VersionMaster,
CephVersion: installer.MasterVersion,
}
suite.cluster, suite.k8sh = StartTestCluster(suite.T, &mgrTestCluster)
suite.waitForOrchestrationModule()
s.settings.ApplyEnvVars()
s.installer, s.k8sh = StartTestCluster(s.T, s.settings, cephMasterSuiteMinimalTestVersion)
s.waitForOrchestrationModule()
}
func (suite *CephMgrSuite) AfterTest(suiteName, testName string) {
suite.cluster.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(suite.namespace))
func (s *CephMgrSuite) AfterTest(suiteName, testName string) {
s.installer.CollectOperatorLog(suiteName, testName)
}
func (suite *CephMgrSuite) TearDownSuite() {
suite.cluster.Teardown()
func (s *CephMgrSuite) TearDownSuite() {
s.installer.UninstallRook()
}
func (suite *CephMgrSuite) execute(command []string) (error, string) {
func (s *CephMgrSuite) execute(command []string) (error, string) {
orchCommand := append([]string{"orch"}, command...)
return suite.cluster.installer.Execute("ceph", orchCommand, suite.namespace)
return s.installer.Execute("ceph", orchCommand, s.namespace)
}
func (suite *CephMgrSuite) waitForOrchestrationModule() {
func (s *CephMgrSuite) waitForOrchestrationModule() {
var err error
for timeout := 0; timeout < 30; timeout++ {
err, output := suite.execute([]string{"status"})
err, output := s.execute([]string{"status"})
logger.Infof("%s", output)
if err == nil {
logger.Info("Rook Toolbox ready to execute commands")
@@ -135,30 +134,30 @@ func (suite *CephMgrSuite) waitForOrchestrationModule() {
time.Sleep(2 * time.Second)
}
logger.Error("Giving up waiting for Rook Toolbox to be ready")
assert.Nil(suite.T(), err)
assert.Nil(s.T(), err)
}
func (suite *CephMgrSuite) TestDeviceLs() {
func (s *CephMgrSuite) TestDeviceLs() {
logger.Info("Testing .... <ceph orch device ls>")
err, device_list := suite.execute([]string{"device", "ls"})
assert.Nil(suite.T(), err)
err, device_list := s.execute([]string{"device", "ls"})
assert.Nil(s.T(), err)
logger.Infof("output = %s", device_list)
}
func (suite *CephMgrSuite) TestStatus() {
func (s *CephMgrSuite) TestStatus() {
logger.Info("Testing .... <ceph orch status>")
err, status := suite.execute([]string{"status"})
assert.Nil(suite.T(), err)
err, status := s.execute([]string{"status"})
assert.Nil(s.T(), err)
logger.Infof("output = %s", status)
assert.Equal(suite.T(), status, "Backend: rook\nAvailable: True")
assert.Equal(s.T(), status, "Backend: rook\nAvailable: True")
}
func (suite *CephMgrSuite) TestHostLs() {
func (s *CephMgrSuite) TestHostLs() {
logger.Info("Testing .... <ceph orch host ls>")
// Get the orchestrator hosts
err, output := suite.execute([]string{"host", "ls", "json"})
assert.Nil(suite.T(), err)
err, output := s.execute([]string{"host", "ls", "json"})
assert.Nil(s.T(), err)
logger.Infof("output = %s", output)
hosts := []byte(output)
@@ -166,7 +165,7 @@ func (suite *CephMgrSuite) TestHostLs() {
err = json.Unmarshal(hosts, &hostsList)
if err != nil {
assert.Nil(suite.T(), err)
assert.Nil(s.T(), err)
}
var hostOutput []string
@@ -176,8 +175,8 @@ func (suite *CephMgrSuite) TestHostLs() {
sort.Strings(hostOutput)
// get the k8s nodes
nodes, err := k8sutil.GetNodeHostNames(suite.k8sh.Clientset)
assert.Nil(suite.T(), err)
nodes, err := k8sutil.GetNodeHostNames(s.k8sh.Clientset)
assert.Nil(s.T(), err)
k8sNodes := make([]string, 0, len(nodes))
for k := range nodes {
@@ -186,21 +185,21 @@ func (suite *CephMgrSuite) TestHostLs() {
sort.Strings(k8sNodes)
// nodes and hosts must be the same
assert.Equal(suite.T(), hostOutput, k8sNodes)
assert.Equal(s.T(), hostOutput, k8sNodes)
}
func (suite *CephMgrSuite) TestCreateOSD() {
func (s *CephMgrSuite) TestCreateOSD() {
logger.Info("Testing .... <ceph orch create OSD>")
// Get the first available device
err, deviceList := suite.execute([]string{"device", "ls", "--format", "json"})
assert.Nil(suite.T(), err)
err, deviceList := s.execute([]string{"device", "ls", "--format", "json"})
assert.Nil(s.T(), err)
logger.Infof("output = %s", deviceList)
inventory := make([]map[string]interface{}, 0)
err = json.Unmarshal([]byte(deviceList), &inventory)
assert.Nil(suite.T(), err)
assert.Nil(s.T(), err)
selectedNode := ""
selectedDevice := ""
@@ -216,40 +215,40 @@ func (suite *CephMgrSuite) TestCreateOSD() {
break
}
}
assert.NotEqual(suite.T(), "", selectedDevice, "No devices available to create test OSD")
assert.NotEqual(suite.T(), "", selectedNode, "No nodes available to create test OSD")
assert.NotEqual(s.T(), "", selectedDevice, "No devices available to create test OSD")
assert.NotEqual(s.T(), "", selectedNode, "No nodes available to create test OSD")
if selectedDevice == "" || selectedNode == "" {
return
}
// Create the OSD
err, output := suite.execute([]string{"daemon", "add", "osd", fmt.Sprintf("%s:%s", selectedNode, selectedDevice)})
err, output := s.execute([]string{"daemon", "add", "osd", fmt.Sprintf("%s:%s", selectedNode, selectedDevice)})
assert.Nil(suite.T(), err)
assert.Nil(s.T(), err)
logger.Infof("output = %s", output)
err = suite.k8sh.WaitForPodCount("app=rook-ceph-osd", suite.namespace, 1)
assert.Nil(suite.T(), err)
err = s.k8sh.WaitForPodCount("app=rook-ceph-osd", s.namespace, 1)
assert.Nil(s.T(), err)
}
func (suite *CephMgrSuite) TestServiceLs() {
func (s *CephMgrSuite) TestServiceLs() {
logger.Info("Testing .... <ceph orch ls>")
err, output := suite.execute([]string{"ls", "--format", "json"})
assert.Nil(suite.T(), err)
err, output := s.execute([]string{"ls", "--format", "json"})
assert.Nil(s.T(), err)
logger.Infof("output = %s", output)
services := []byte(output)
var servicesList []service
err = json.Unmarshal(services, &servicesList)
assert.Nil(suite.T(), err)
assert.Nil(s.T(), err)
for _, svc := range servicesList {
labelFilter := fmt.Sprintf("app=rook-ceph-%s", svc.ServiceName)
k8sPods, err := k8sutil.PodsRunningWithLabel(suite.k8sh.Clientset, suite.namespace, labelFilter)
k8sPods, err := k8sutil.PodsRunningWithLabel(s.k8sh.Clientset, s.namespace, labelFilter)
logger.Infof("Service: %+v", svc)
logger.Infof("k8s pods for svc %q: %d", svc.ServiceName, k8sPods)
assert.Nil(suite.T(), err)
assert.Equal(suite.T(), svc.Status.Running, k8sPods, fmt.Sprintf("Wrong number of pods for kind of service <%s>", svc.ServiceName))
assert.Nil(s.T(), err)
assert.Equal(s.T(), svc.Status.Running, k8sPods, fmt.Sprintf("Wrong number of pods for kind of service <%s>", svc.ServiceName))
}
}
+102 -154
View File
@@ -17,16 +17,14 @@ limitations under the License.
package integration
import (
"fmt"
"path/filepath"
"testing"
"time"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
@@ -58,180 +56,130 @@ func TestCephMultiClusterDeploySuite(t *testing.T) {
s := new(MultiClusterDeploySuite)
defer func(s *MultiClusterDeploySuite) {
HandlePanics(recover(), s.op, s.T)
HandlePanics(recover(), s.TearDownSuite, s.T)
}(s)
suite.Run(t, s)
}
type MultiClusterDeploySuite struct {
suite.Suite
testClient *clients.TestClient
k8sh *utils.K8sHelper
namespace1 string
namespace2 string
op *MCTestOperations
poolName string
testClient *clients.TestClient
k8sh *utils.K8sHelper
settings *installer.TestCephSettings
externalManifests installer.CephManifests
installer *installer.CephInstaller
coreToolbox string
externalToolbox string
poolName string
}
// Deploy Multiple Rook clusters
func (mrc *MultiClusterDeploySuite) SetupSuite() {
mrc.poolName = "multi-cluster-pool1"
mrc.namespace1 = "mrc-n1"
mrc.namespace2 = "mrc-n2"
mrc.op, mrc.k8sh = NewMCTestOperations(mrc.T, mrc.namespace1, mrc.namespace2)
mrc.testClient = clients.CreateTestClient(mrc.k8sh, mrc.op.installer.Manifests)
mrc.createPools()
}
func (mrc *MultiClusterDeploySuite) AfterTest(suiteName, testName string) {
mrc.op.installer.CollectOperatorLog(suiteName, testName, mrc.op.systemNamespace)
}
func (mrc *MultiClusterDeploySuite) createPools() {
// create a test pool in each cluster so that we get some PGs
logger.Infof("Creating pool %s", mrc.poolName)
err := mrc.testClient.PoolClient.Create(mrc.poolName, mrc.namespace1, 1)
require.Nil(mrc.T(), err)
}
func (mrc *MultiClusterDeploySuite) deletePools() {
// create a test pool in each cluster so that we get some PGs
logger.Infof("Deleting pool %s", mrc.poolName)
clusterInfo := client.AdminClusterInfo(mrc.namespace1)
if err := mrc.testClient.PoolClient.DeletePool(mrc.testClient.BlockClient, clusterInfo, mrc.poolName); err != nil {
logger.Errorf("failed to delete pool %q. %v", mrc.poolName, err)
} else {
logger.Infof("deleted pool %q", mrc.poolName)
func (s *MultiClusterDeploySuite) SetupSuite() {
s.poolName = "multi-cluster-pool1"
coreNamespace := "multi-core"
s.settings = &installer.TestCephSettings{
ClusterName: "multi-cluster",
Namespace: coreNamespace,
OperatorNamespace: installer.SystemNamespace(coreNamespace),
StorageClassName: "manual",
UsePVC: installer.UsePVC(),
Mons: 1,
UseCSI: true,
MultipleMgrs: true,
EnableAdmissionController: true,
RookVersion: installer.VersionMaster,
CephVersion: installer.NautilusVersion,
}
}
func (mrc *MultiClusterDeploySuite) TearDownSuite() {
mrc.deletePools()
mrc.op.Teardown()
}
// Test to make sure all rook components are installed and Running
func (mrc *MultiClusterDeploySuite) TestInstallingMultipleRookClusters() {
// Check if Rook cluster 1 is deployed successfully
checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace1, 1)
checkIfRookClusterIsHealthy(mrc.Suite, mrc.testClient, mrc.namespace1)
// Check if Rook external cluster is deployed successfully
// Checking health status is enough to validate the connection
checkIfRookClusterIsHealthy(mrc.Suite, mrc.testClient, mrc.namespace2)
}
// MCTestOperations struct for handling panic and test suite tear down
type MCTestOperations struct {
installer *installer.CephInstaller
kh *utils.K8sHelper
T func() *testing.T
namespace1 string
namespace2 string
systemNamespace string
storageClassName string
testOverPVC bool
}
// NewMCTestOperations creates new instance of TestCluster struct
func NewMCTestOperations(t func() *testing.T, namespace1 string, namespace2 string) (*MCTestOperations, *utils.K8sHelper) {
kh, err := utils.CreateK8sHelper(t)
require.NoError(t(), err)
checkIfShouldRunForMinimalTestMatrix(t, kh, multiClusterMinimalTestVersion)
cleanupHost := false
i := installer.NewCephInstaller(t, kh.Clientset, false, "", installer.VersionMaster, installer.NautilusVersion, cleanupHost)
op := &MCTestOperations{i, kh, t, namespace1, namespace2, installer.SystemNamespace(namespace1), "", false}
if kh.VersionAtLeast("v1.13.0") {
op.testOverPVC = true
op.storageClassName = "manual"
s.settings.ApplyEnvVars()
externalSettings := &installer.TestCephSettings{
IsExternal: true,
ClusterName: "test-external",
Namespace: "multi-external",
OperatorNamespace: s.settings.OperatorNamespace,
RookVersion: s.settings.RookVersion,
UseCSI: true,
}
op.Setup()
return op, kh
}
externalSettings.ApplyEnvVars()
s.externalManifests = installer.NewCephManifests(externalSettings)
// Setup is wrapper for setting up multiple rook clusters.
func (o MCTestOperations) Setup() {
var err error
if o.testOverPVC {
root, err := utils.FindRookRoot()
require.NoError(o.T(), err, "failed to get rook root")
cmdArgs := utils.CommandArgs{Command: filepath.Join(root, localPathPVCmd),
CmdArgs: []string{installer.TestScratchDevice()}}
cmdOut := utils.ExecuteCommand(cmdArgs)
require.NoError(o.T(), cmdOut.Err)
k8sh, err := utils.CreateK8sHelper(s.T)
assert.NoError(s.T(), err)
if !k8sh.VersionAtLeast("v1.16.0") {
s.T().Skip("requires at least k8s 1.16, no need to run on older versions")
}
err = o.installer.CreateCephOperator(installer.SystemNamespace(o.namespace1))
require.NoError(o.T(), err)
require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-operator", o.systemNamespace, "Running"),
"Make sure rook-operator is in running state")
require.True(o.T(), o.kh.IsPodInExpectedState("rook-discover", o.systemNamespace, "Running"),
"Make sure rook-discover is in running state")
time.Sleep(10 * time.Second)
// start the two clusters in parallel
logger.Infof("starting two clusters, one traditional and one external")
err = o.startCluster(o.namespace1, "bluestore")
require.NoError(o.T(), err)
// Wait for the monitors to be up so that we can fetch the configmap and secrets
require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-mon", o.namespace1, "Running"),
"Make sure rook-ceph-mon is in running state")
// Start the core storage cluster
s.setupMultiClusterCore()
s.createPools()
// Start the external cluster that will connect to the core cluster
// create an external cluster
err = o.startExternalCluster(o.namespace2)
require.NoError(o.T(), err)
s.startExternalCluster()
logger.Infof("finished starting clusters")
}
// Teardown is a wrapper for tearDown after suite
func (o MCTestOperations) Teardown() {
o.installer.UninstallRookFromMultipleNS(installer.SystemNamespace(o.namespace1), o.namespace1, o.namespace2)
func (s *MultiClusterDeploySuite) AfterTest(suiteName, testName string) {
s.installer.CollectOperatorLog(suiteName, testName)
}
func (o MCTestOperations) startCluster(namespace, store string) error {
logger.Infof("starting cluster %s", namespace)
multipleManagers := true
err := o.installer.CreateRookCluster(namespace, o.systemNamespace, store, o.testOverPVC, o.storageClassName,
cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, multipleManagers, false, false, installer.NautilusVersion)
func (s *MultiClusterDeploySuite) createPools() {
// create a test pool in each cluster so that we get some PGs
logger.Infof("Creating pool %s", s.poolName)
err := s.testClient.PoolClient.Create(s.poolName, s.settings.Namespace, 1)
require.Nil(s.T(), err)
}
func (s *MultiClusterDeploySuite) deletePools() {
// create a test pool in each cluster so that we get some PGs
clusterInfo := client.AdminClusterInfo(s.settings.Namespace)
if err := s.testClient.PoolClient.DeletePool(s.testClient.BlockClient, clusterInfo, s.poolName); err != nil {
logger.Errorf("failed to delete pool %q. %v", s.poolName, err)
} else {
logger.Infof("deleted pool %q", s.poolName)
}
}
func (s *MultiClusterDeploySuite) TearDownSuite() {
s.deletePools()
s.installer.UninstallRookFromMultipleNS(s.installer.Manifests, s.externalManifests)
}
// Test to make sure all rook components are installed and Running
func (s *MultiClusterDeploySuite) TestInstallingMultipleRookClusters() {
// Check if Rook cluster 1 is deployed successfully
client.RunAllCephCommandsInToolboxPod = s.coreToolbox
checkIfRookClusterIsInstalled(s.Suite, s.k8sh, s.settings.OperatorNamespace, s.settings.Namespace, 1)
checkIfRookClusterIsHealthy(s.Suite, s.testClient, s.settings.Namespace)
// Check if Rook external cluster is deployed successfully
// Checking health status is enough to validate the connection
client.RunAllCephCommandsInToolboxPod = s.externalToolbox
checkIfRookClusterIsHealthy(s.Suite, s.testClient, s.externalManifests.Settings().Namespace)
}
// Setup is wrapper for setting up multiple rook clusters.
func (s *MultiClusterDeploySuite) setupMultiClusterCore() {
root, err := utils.FindRookRoot()
require.NoError(s.T(), err, "failed to get rook root")
cmdArgs := utils.CommandArgs{Command: filepath.Join(root, localPathPVCmd),
CmdArgs: []string{installer.TestScratchDevice()}}
cmdOut := utils.ExecuteCommand(cmdArgs)
require.NoError(s.T(), cmdOut.Err)
s.installer, s.k8sh = StartTestCluster(s.T, s.settings, multiClusterMinimalTestVersion)
s.testClient = clients.CreateTestClient(s.k8sh, s.installer.Manifests)
s.coreToolbox = client.RunAllCephCommandsInToolboxPod
}
func (s *MultiClusterDeploySuite) startExternalCluster() {
err := s.installer.CreateRookExternalCluster(s.externalManifests)
if err != nil {
o.T().Fail()
o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace)
return fmt.Errorf("failed to create cluster %s. %+v", namespace, err)
s.T().Fail()
s.installer.GatherAllRookLogs(s.T().Name(), s.externalManifests.Settings().Namespace)
require.NoError(s.T(), err)
}
if err := o.installer.CreateRookToolbox(namespace); err != nil {
o.T().Fail()
o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace)
return fmt.Errorf("failed to create toolbox for %s. %+v", namespace, err)
}
logger.Infof("succeeded starting cluster %s", namespace)
return nil
}
func (o MCTestOperations) startExternalCluster(namespace string) error {
logger.Infof("starting external cluster %q", namespace)
err := o.installer.CreateRookExternalCluster(namespace, o.namespace1)
if err != nil {
o.T().Fail()
o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace)
return fmt.Errorf("failed to create external cluster %s. %+v", namespace, err)
}
logger.Infof("running toolbox on namespace %q", namespace)
if err := o.installer.CreateRookToolbox(namespace); err != nil {
o.T().Fail()
o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace)
return fmt.Errorf("failed to create toolbox for %s. %+v", namespace, err)
}
logger.Infof("succeeded starting external cluster %s", namespace)
return nil
s.externalToolbox = client.RunAllCephCommandsInToolboxPod
logger.Infof("succeeded starting external cluster %s", s.externalManifests.Settings().Namespace)
}
+98 -93
View File
@@ -72,7 +72,7 @@ func TestCephSmokeSuite(t *testing.T) {
s := new(SmokeSuite)
defer func(s *SmokeSuite) {
HandlePanics(recover(), s.op, s.T)
HandlePanics(recover(), s.TearDownSuite, s.T)
}(s)
suite.Run(t, s)
}
@@ -80,83 +80,85 @@ func TestCephSmokeSuite(t *testing.T) {
type SmokeSuite struct {
suite.Suite
helper *clients.TestClient
op *TestCluster
settings *installer.TestCephSettings
installer *installer.CephInstaller
k8sh *utils.K8sHelper
namespace string
}
func (suite *SmokeSuite) SetupSuite() {
suite.namespace = "smoke-ns"
smokeTestCluster := TestCluster{
namespace: suite.namespace,
storeType: "bluestore",
storageClassName: installer.StorageClassName(),
useHelm: false,
usePVC: installer.UsePVC(),
mons: 3,
rbdMirrorWorkers: 1,
rookCephCleanup: true,
skipOSDCreation: false,
minimalMatrixK8sVersion: smokeSuiteMinimalTestVersion,
rookVersion: installer.VersionMaster,
cephVersion: installer.OctopusVersion,
func (s *SmokeSuite) SetupSuite() {
namespace := "smoke-ns"
s.settings = &installer.TestCephSettings{
ClusterName: "smoke-cluster",
Namespace: namespace,
OperatorNamespace: installer.SystemNamespace(namespace),
StorageClassName: installer.StorageClassName(),
UseHelm: false,
UsePVC: installer.UsePVC(),
Mons: 3,
SkipOSDCreation: false,
UseCSI: true,
EnableAdmissionController: true,
UseCrashPruner: true,
RookVersion: installer.VersionMaster,
CephVersion: installer.OctopusVersion,
}
s.settings.ApplyEnvVars()
suite.op, suite.k8sh = StartTestCluster(suite.T, &smokeTestCluster)
suite.helper = clients.CreateTestClient(suite.k8sh, suite.op.installer.Manifests)
s.installer, s.k8sh = StartTestCluster(s.T, s.settings, smokeSuiteMinimalTestVersion)
s.helper = clients.CreateTestClient(s.k8sh, s.installer.Manifests)
}
func (suite *SmokeSuite) AfterTest(suiteName, testName string) {
suite.op.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(suite.namespace))
func (s *SmokeSuite) AfterTest(suiteName, testName string) {
s.installer.CollectOperatorLog(suiteName, testName)
}
func (suite *SmokeSuite) TearDownSuite() {
suite.op.Teardown()
func (s *SmokeSuite) TearDownSuite() {
s.installer.UninstallRook()
}
func (suite *SmokeSuite) TestBlockStorage_SmokeTest() {
runBlockCSITest(suite.helper, suite.k8sh, suite.Suite, suite.namespace)
func (s *SmokeSuite) TestBlockStorage_SmokeTest() {
runBlockCSITest(s.helper, s.k8sh, s.Suite, s.settings.Namespace)
}
func (suite *SmokeSuite) TestFileStorage_SmokeTest() {
useCSI := true
func (s *SmokeSuite) TestFileStorage_SmokeTest() {
preserveFilesystemOnDelete := true
runFileE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace, "smoke-test-fs", useCSI, preserveFilesystemOnDelete)
runFileE2ETest(s.helper, s.k8sh, s.Suite, s.settings, "smoke-test-fs", preserveFilesystemOnDelete)
}
func (suite *SmokeSuite) TestObjectStorage_SmokeTest() {
if !utils.IsPlatformOpenShift() {
runObjectE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace)
func (s *SmokeSuite) TestObjectStorage_SmokeTest() {
if utils.IsPlatformOpenShift() {
s.T().Skip("object store tests skipped on openshift")
}
runObjectE2ETest(s.helper, s.k8sh, s.Suite, s.settings.Namespace)
}
// Test to make sure all rook components are installed and Running
func (suite *SmokeSuite) TestARookClusterInstallation_SmokeTest() {
checkIfRookClusterIsInstalled(suite.Suite, suite.k8sh, installer.SystemNamespace(suite.namespace), suite.namespace, 3)
func (s *SmokeSuite) TestARookClusterInstallation_SmokeTest() {
checkIfRookClusterIsInstalled(s.Suite, s.k8sh, s.settings.OperatorNamespace, s.settings.Namespace, 3)
}
// Smoke Test for Mon failover - Test check the following operations for the Mon failover in order
// Delete mon pod, Wait for new mon pod
func (suite *SmokeSuite) TestMonFailover() {
func (s *SmokeSuite) TestMonFailover() {
ctx := context.TODO()
logger.Infof("Mon Failover Smoke Test")
deployments, err := suite.getNonCanaryMonDeployments()
require.Nil(suite.T(), err)
require.Equal(suite.T(), 3, len(deployments))
deployments, err := s.getNonCanaryMonDeployments()
require.NoError(s.T(), err)
require.Equal(s.T(), 3, len(deployments))
monToKill := deployments[0].Name
logger.Infof("Killing mon %s", monToKill)
propagation := metav1.DeletePropagationForeground
delOptions := &metav1.DeleteOptions{PropagationPolicy: &propagation}
err = suite.k8sh.Clientset.AppsV1().Deployments(suite.namespace).Delete(ctx, monToKill, *delOptions)
require.Nil(suite.T(), err)
err = s.k8sh.Clientset.AppsV1().Deployments(s.settings.Namespace).Delete(ctx, monToKill, *delOptions)
require.NoError(s.T(), err)
// Wait for the health check to start a new monitor
originalMonDeleted := false
for i := 0; i < 30; i++ {
deployments, err := suite.getNonCanaryMonDeployments()
require.Nil(suite.T(), err)
deployments, err := s.getNonCanaryMonDeployments()
require.NoError(s.T(), err)
// Make sure the old mon is not still alive
foundOldMon := false
@@ -188,31 +190,31 @@ func (suite *SmokeSuite) TestMonFailover() {
return
}
assert.Equal(suite.T(), 2, len(deployments))
assert.Equal(s.T(), 2, len(deployments))
}
time.Sleep(5 * time.Second)
}
require.Fail(suite.T(), "giving up waiting for a new monitor")
require.Fail(s.T(), "giving up waiting for a new monitor")
}
// Smoke Test for pool Resizing
func (suite *SmokeSuite) TestPoolResize() {
func (s *SmokeSuite) TestPoolResize() {
ctx := context.TODO()
logger.Infof("Pool Resize Smoke Test")
poolName := "testpool"
err := suite.helper.PoolClient.Create(poolName, suite.namespace, 1)
require.Nil(suite.T(), err)
err := s.helper.PoolClient.Create(poolName, s.settings.Namespace, 1)
require.NoError(s.T(), err)
poolFound := false
clusterInfo := client.AdminClusterInfo(suite.namespace)
clusterInfo := client.AdminClusterInfo(s.settings.Namespace)
// Wait for pool to appear
for i := 0; i < 10; i++ {
pools, err := suite.helper.PoolClient.ListCephPools(clusterInfo)
require.Nil(suite.T(), err)
pools, err := s.helper.PoolClient.ListCephPools(clusterInfo)
require.NoError(s.T(), err)
for _, p := range pools {
if p.Name != poolName {
continue
@@ -227,24 +229,24 @@ func (suite *SmokeSuite) TestPoolResize() {
time.Sleep(2 * time.Second)
}
require.Equal(suite.T(), true, poolFound, "pool not found")
require.Equal(s.T(), true, poolFound, "pool not found")
err = suite.helper.PoolClient.Update(poolName, suite.namespace, 2)
require.Nil(suite.T(), err)
err = s.helper.PoolClient.Update(poolName, s.settings.Namespace, 2)
require.NoError(s.T(), err)
poolResized := false
// Wait for pool resize to happen
for i := 0; i < 10; i++ {
details, err := suite.helper.PoolClient.GetCephPoolDetails(clusterInfo, poolName)
require.Nil(suite.T(), err)
details, err := s.helper.PoolClient.GetCephPoolDetails(clusterInfo, poolName)
require.NoError(s.T(), err)
if details.Size > 1 {
logger.Infof("pool %s size was updated", poolName)
require.Equal(suite.T(), 2, int(details.Size))
require.Equal(s.T(), 2, int(details.Size))
poolResized = true
// resize the pool back to 1 to avoid hangs around not having enough OSDs to satisfy rbd
err = suite.helper.PoolClient.Update(poolName, suite.namespace, 1)
require.Nil(suite.T(), err)
err = s.helper.PoolClient.Update(poolName, s.settings.Namespace, 1)
require.NoError(s.T(), err)
} else if poolResized && details.Size == 1 {
logger.Infof("pool resized back to 1")
break
@@ -255,45 +257,45 @@ func (suite *SmokeSuite) TestPoolResize() {
time.Sleep(2 * time.Second)
}
require.Equal(suite.T(), true, poolResized, fmt.Sprintf("pool %s not found", poolName))
require.Equal(s.T(), true, poolResized, fmt.Sprintf("pool %s not found", poolName))
// Verify the Kubernetes Secret has been created (bootstrap peer token)
pool, err := suite.k8sh.RookClientset.CephV1().CephBlockPools(suite.namespace).Get(ctx, poolName, metav1.GetOptions{})
assert.NoError(suite.T(), err)
pool, err := s.k8sh.RookClientset.CephV1().CephBlockPools(s.settings.Namespace).Get(ctx, poolName, metav1.GetOptions{})
assert.NoError(s.T(), err)
if pool.Spec.Mirroring.Enabled {
secretName := pool.Status.Info[oppool.RBDMirrorBootstrapPeerSecretName]
assert.NotEmpty(suite.T(), secretName)
assert.NotEmpty(s.T(), secretName)
// now fetch the secret which contains the bootstrap peer token
s, err := suite.k8sh.Clientset.CoreV1().Secrets(suite.namespace).Get(ctx, secretName, metav1.GetOptions{})
require.Nil(suite.T(), err)
assert.NotEmpty(suite.T(), s.Data["token"])
secret, err := s.k8sh.Clientset.CoreV1().Secrets(s.settings.Namespace).Get(ctx, secretName, metav1.GetOptions{})
require.NoError(s.T(), err)
assert.NotEmpty(s.T(), secret.Data["token"])
// Once we have a scenario with another Ceph cluster - needs to be added in the MultiCluster suite
// We would need to add a bootstrap peer token following the below procedure
// bootstrapSecretName := "bootstrap-peer-token"
// token := "eyJmc2lkIjoiYzZiMDg3ZjItNzgyOS00ZGJiLWJjZmMtNTNkYzM0ZTBiMzVkIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBV1lsWmZVQ1Q2RGhBQVBtVnAwbGtubDA5YVZWS3lyRVV1NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMTExLjEwOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTA6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjEyOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTI6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjExOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTE6Njc4OV0ifQ=="
// s = oppool.GenerateBootstrapPeerSecret(bootstrapSecretName, suite.namespace, string(pool.GetUID()), []byte(token))
// s, err = suite.k8sh.Clientset.CoreV1().Secrets(suite.namespace).Create(s)
// require.Nil(suite.T(), err, err.Error())
// s = oppool.GenerateBootstrapPeerSecret(bootstrapSecretName, s.settings.Namespace, string(pool.GetUID()), []byte(token))
// s, err = s.k8sh.Clientset.CoreV1().Secrets(s.settings.Namespace).Create(s)
// require.Nil(s.T(), err, err.Error())
// // update the ceph block pool cr
// pool.Spec.Mirrored.PeersSecretNames = append(pool.Spec.Mirrored.PeersSecretNames, bootstrapSecretName)
// _, err = suite.k8sh.RookClientset.CephV1().CephBlockPools(suite.namespace).Update(pool)
// require.Nil(suite.T(), err, err.Error())
// _, err = s.k8sh.RookClientset.CephV1().CephBlockPools(s.settings.Namespace).Update(pool)
// require.Nil(s.T(), err, err.Error())
// mirrorInfo, err := client.PrintPoolMirroringInfo(suite.k8sh.MakeContext(), clusterInfo, poolName)
// require.Nil(suite.T(), err, err.Error())
// assert.Equal(suite.T(), "image", mirrorInfo.Mode)
// assert.Equal(suite.T(), 1, len(mirrorInfo.Peers))
// mirrorInfo, err := client.PrintPoolMirroringInfo(s.k8sh.MakeContext(), clusterInfo, poolName)
// require.Nil(s.T(), err, err.Error())
// assert.Equal(s.T(), "image", mirrorInfo.Mode)
// assert.Equal(s.T(), 1, len(mirrorInfo.Peers))
}
// clean up the pool
err = suite.helper.PoolClient.DeletePool(suite.helper.BlockClient, clusterInfo, poolName)
assert.NoError(suite.T(), err)
err = s.helper.PoolClient.DeletePool(s.helper.BlockClient, clusterInfo, poolName)
assert.NoError(s.T(), err)
}
// Smoke Test for Client CRD
func (suite *SmokeSuite) TestCreateClient() {
func (s *SmokeSuite) TestCreateClient() {
logger.Infof("Create Client Smoke Test")
clientName := "client1"
@@ -302,14 +304,14 @@ func (suite *SmokeSuite) TestCreateClient() {
"mgr": "allow rwx",
"osd": "allow rwx",
}
clusterInfo := client.AdminClusterInfo(suite.namespace)
err := suite.helper.UserClient.Create(clientName, suite.namespace, caps)
require.Nil(suite.T(), err)
clusterInfo := client.AdminClusterInfo(s.settings.Namespace)
err := s.helper.UserClient.Create(clientName, s.settings.Namespace, caps)
require.NoError(s.T(), err)
clientFound := false
for i := 0; i < 30; i++ {
clients, _ := suite.helper.UserClient.Get(clusterInfo, "client."+clientName)
clients, _ := s.helper.UserClient.Get(clusterInfo, "client."+clientName)
if clients != "" {
clientFound = true
}
@@ -322,7 +324,7 @@ func (suite *SmokeSuite) TestCreateClient() {
time.Sleep(2 * time.Second)
}
require.Equal(suite.T(), true, clientFound, "client not found")
assert.Equal(s.T(), true, clientFound, "client not found")
logger.Infof("Update Client Smoke Test")
newcaps := map[string]string{
@@ -330,30 +332,33 @@ func (suite *SmokeSuite) TestCreateClient() {
"mgr": "allow rw",
"osd": "allow *",
}
caps, _ = suite.helper.UserClient.Update(clusterInfo, clientName, newcaps)
caps, _ = s.helper.UserClient.Update(clusterInfo, clientName, newcaps)
require.Equal(suite.T(), "allow r", caps["mon"], "wrong caps")
require.Equal(suite.T(), "allow rw", caps["mgr"], "wrong caps")
require.Equal(suite.T(), "allow *", caps["osd"], "wrong caps")
assert.Equal(s.T(), "allow r", caps["mon"], "wrong caps")
assert.Equal(s.T(), "allow rw", caps["mgr"], "wrong caps")
assert.Equal(s.T(), "allow *", caps["osd"], "wrong caps")
err = s.helper.UserClient.Delete(clientName, s.settings.Namespace)
require.NoError(s.T(), err)
}
// Smoke Test for RBD Mirror CRD
func (suite *SmokeSuite) TestCreateRBDMirrorClient() {
func (s *SmokeSuite) TestCreateRBDMirrorClient() {
logger.Infof("Create rbd-mirror Smoke Test")
rbdMirrorName := "my-rbd-mirror"
err := suite.helper.RBDMirrorClient.Create(suite.namespace, rbdMirrorName, 1)
require.Nil(suite.T(), err)
err := s.helper.RBDMirrorClient.Create(s.settings.Namespace, rbdMirrorName, 1)
require.NoError(s.T(), err)
err = suite.helper.RBDMirrorClient.Delete(suite.namespace, rbdMirrorName)
require.Nil(suite.T(), err)
err = s.helper.RBDMirrorClient.Delete(s.settings.Namespace, rbdMirrorName)
require.NoError(s.T(), err)
}
func (suite *SmokeSuite) getNonCanaryMonDeployments() ([]appsv1.Deployment, error) {
func (s *SmokeSuite) getNonCanaryMonDeployments() ([]appsv1.Deployment, error) {
ctx := context.TODO()
opts := metav1.ListOptions{LabelSelector: "app=rook-ceph-mon"}
deployments, err := suite.k8sh.Clientset.AppsV1().Deployments(suite.namespace).List(ctx, opts)
deployments, err := s.k8sh.Clientset.AppsV1().Deployments(s.settings.Namespace).List(ctx, opts)
if err != nil {
return nil, err
}
+36 -45
View File
@@ -58,7 +58,7 @@ func TestCephUpgradeSuite(t *testing.T) {
s := new(UpgradeSuite)
defer func(s *UpgradeSuite) {
HandlePanics(recover(), s.op, s.T)
HandlePanics(recover(), s.TearDownSuite, s.T)
}(s)
suite.Run(t, s)
}
@@ -66,40 +66,37 @@ func TestCephUpgradeSuite(t *testing.T) {
type UpgradeSuite struct {
suite.Suite
helper *clients.TestClient
op *TestCluster
k8sh *utils.K8sHelper
settings *installer.TestCephSettings
installer *installer.CephInstaller
namespace string
}
func (s *UpgradeSuite) SetupSuite() {
s.namespace = "upgrade-ns"
upgradeTestCluster := TestCluster{
clusterName: s.namespace,
namespace: s.namespace,
storeType: "",
storageClassName: "",
useHelm: false,
usePVC: false,
mons: 1,
rbdMirrorWorkers: 0,
rookCephCleanup: false,
skipOSDCreation: false,
minimalMatrixK8sVersion: upgradeMinimalTestVersion,
rookVersion: installer.Version1_5,
cephVersion: installer.NautilusVersion,
s.settings = &installer.TestCephSettings{
ClusterName: s.namespace,
Namespace: s.namespace,
OperatorNamespace: installer.SystemNamespace(s.namespace),
StorageClassName: "",
UseHelm: false,
UsePVC: false,
Mons: 1,
UseCSI: true,
SkipOSDCreation: false,
RookVersion: installer.Version1_5,
CephVersion: installer.NautilusVersion,
}
s.op, s.k8sh = StartTestCluster(s.T, &upgradeTestCluster)
s.helper = clients.CreateTestClient(s.k8sh, s.op.installer.Manifests)
s.installer, s.k8sh = StartTestCluster(s.T, s.settings, upgradeMinimalTestVersion)
s.helper = clients.CreateTestClient(s.k8sh, s.installer.Manifests)
}
func (s *UpgradeSuite) TearDownSuite() {
s.op.Teardown()
s.installer.UninstallRook()
}
func (s *UpgradeSuite) TestUpgradeToMaster() {
systemNamespace := installer.SystemNamespace(s.namespace)
//
// Create block, object, and file storage before the upgrade
//
@@ -108,7 +105,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
blockName := "block-claim-upgrade"
logger.Infof("Initializing block before the upgrade")
clusterInfo := client.AdminClusterInfo(s.namespace)
setupBlockLite(s.helper, s.k8sh, s.Suite, clusterInfo, systemNamespace, poolName, storageClassName, blockName, rbdPodName, s.op.installer.CephVersion)
setupBlockLite(s.helper, s.k8sh, s.Suite, clusterInfo, poolName, storageClassName, blockName, rbdPodName)
createPodWithBlock(s.helper, s.k8sh, s.Suite, s.namespace, storageClassName, rbdPodName, blockName)
@@ -121,13 +118,12 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
logger.Infof("Initializing file before the upgrade")
filesystemName := "upgrade-test-fs"
activeCount := 1
createFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, activeCount)
createFilesystem(s.helper, s.k8sh, s.Suite, s.settings, filesystemName, activeCount)
// Start the file test client
fsStorageClass := "file-upgrade"
assert.NoError(s.T(), s.helper.FSClient.CreateStorageClass(filesystemName, systemNamespace, s.namespace, fsStorageClass))
useCSI := true
createFilesystemConsumerPod(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, fsStorageClass, useCSI)
assert.NoError(s.T(), s.helper.FSClient.CreateStorageClass(filesystemName, s.settings.OperatorNamespace, s.namespace, fsStorageClass))
createFilesystemConsumerPod(s.helper, s.k8sh, s.Suite, s.settings, filesystemName, fsStorageClass)
defer func() {
cleanupFilesystemConsumer(s.helper, s.k8sh, s.Suite, s.namespace, filePodName)
cleanupFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName)
@@ -135,7 +131,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
logger.Infof("Initializing object before the upgrade")
objectStoreName := "upgraded-object"
runObjectE2ETestLite(s.helper, s.k8sh, s.Suite, s.namespace, objectStoreName, 1, false)
runObjectE2ETestLite(s.helper, s.k8sh, s.Suite, s.settings, objectStoreName, 1, false)
logger.Info("Initializing object bucket claim before the upgrade")
bucketStorageClassName := "rook-smoke-delete-bucket"
@@ -147,6 +143,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
defer func() {
_ = s.helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketPrefix, maxObject, false)
_ = s.helper.BucketClient.DeleteBucketStorageClass(s.namespace, objectStoreName, bucketStorageClassName, "Delete", region)
objectStoreCleanUp(s.Suite, s.helper, s.k8sh, s.settings.Namespace, objectStoreName)
}()
created := utils.Retry(12, 2*time.Second, "OBC is created", func() bool {
@@ -178,7 +175,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
// Upgrade Rook from v1.5 to master
//
logger.Infof("*** UPGRADING ROOK FROM %s to master ***", installer.Version1_5)
s.gatherLogs(systemNamespace, "_before_master_upgrade")
s.gatherLogs(s.settings.OperatorNamespace, "_before_master_upgrade")
s.upgradeToMaster()
s.verifyOperatorImage(installer.VersionMaster)
@@ -199,7 +196,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
// Upgrade from nautilus to octopus
//
logger.Infof("*** UPGRADING CEPH FROM Nautilus TO Octopus ***")
s.gatherLogs(systemNamespace, "_before_octopus_upgrade")
s.gatherLogs(s.settings.OperatorNamespace, "_before_octopus_upgrade")
s.upgradeCephVersion(installer.OctopusVersion.Image, numOSDs)
// Verify reading and writing to the test clients
newFile = "post-octopus-upgrade-file"
@@ -213,7 +210,7 @@ func (s *UpgradeSuite) gatherLogs(systemNamespace, testSuffix string) {
s.k8sh.PrintPodDescribe(s.namespace)
}
n := strings.Replace(s.T().Name(), "/", "_", -1) + testSuffix
s.op.installer.GatherAllRookLogs(n, systemNamespace, s.namespace)
s.installer.GatherAllRookLogs(n, systemNamespace, s.namespace)
}
func (s *UpgradeSuite) upgradeCephVersion(newCephImage string, numOSDs int) {
@@ -241,7 +238,7 @@ func (s *UpgradeSuite) verifyRookUpgrade(numOSDs int) {
// Get some info about the currently deployed mons to determine later if they are all updated
monDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-mon")
require.NoError(s.T(), err)
require.Equal(s.T(), s.op.mons, len(monDepList.Items), monDepList.Items)
require.Equal(s.T(), s.settings.Mons, len(monDepList.Items), monDepList.Items)
// Get some info about the currently deployed mgr to determine later if it is updated
mgrDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-mgr")
@@ -263,7 +260,7 @@ func (s *UpgradeSuite) verifyRookUpgrade(numOSDs int) {
func (s *UpgradeSuite) waitForUpgradedDaemons(previousVersion, versionLabel string, numOSDs int, waitForMDS bool) {
// wait for the mon(s) to be updated
monsNotOldVersion := fmt.Sprintf("app=rook-ceph-mon,%s!=%s", versionLabel, previousVersion)
err := s.k8sh.WaitForDeploymentCount(monsNotOldVersion, s.namespace, s.op.mons)
err := s.k8sh.WaitForDeploymentCount(monsNotOldVersion, s.namespace, s.settings.Mons)
require.NoError(s.T(), err, "mon(s) didn't update")
err = s.k8sh.WaitForLabeledDeploymentsToBeReady(monsNotOldVersion, s.namespace)
require.NoError(s.T(), err)
@@ -336,19 +333,13 @@ func (s *UpgradeSuite) verifyFilesAfterUpgrade(fsName, newFileToWrite, messageFo
// UpgradeToMaster performs the steps necessary to upgrade a Rook v1.4 cluster to master. It does not
// verify the upgrade but merely starts the upgrade process.
func (s *UpgradeSuite) upgradeToMaster() {
v1ExtensionsSupported := false
if s.k8sh.VersionAtLeast("v1.16.0") {
v1ExtensionsSupported = true
}
require.NoError(s.T(), s.k8sh.ResourceOperation("apply", upgradeManifestToMaster(s.namespace, v1ExtensionsSupported)))
// Apply the CRDs for the latest master
s.settings.RookVersion = installer.VersionMaster
m := installer.NewCephManifests(s.settings)
require.NoError(s.T(), s.k8sh.ResourceOperation("apply", m.GetCRDs(s.k8sh)))
require.NoError(s.T(), s.k8sh.ResourceOperation("apply", m.GetCommon()))
require.NoError(s.T(),
s.k8sh.SetDeploymentVersion(installer.SystemNamespace(s.namespace), operatorContainer, operatorContainer, installer.VersionMaster))
}
func upgradeManifestToMaster(namespace string, useV1Extensions bool) string {
// Always apply CRDs for the latest master
m := installer.NewCephManifests(installer.VersionMaster)
manifests := m.GetRookCRDs(useV1Extensions)
return manifests
s.k8sh.SetDeploymentVersion(s.settings.OperatorNamespace, operatorContainer, operatorContainer, installer.VersionMaster))
}
+42 -42
View File
@@ -45,7 +45,7 @@ func TestNfsSuite(t *testing.T) {
s := new(NfsSuite)
defer func(s *NfsSuite) {
HandlePanics(recover(), s, s.T)
HandlePanics(recover(), s.Teardown, s.T)
}(s)
suite.Run(t, s)
}
@@ -60,87 +60,87 @@ type NfsSuite struct {
instanceCount int
}
func (suite *NfsSuite) SetupSuite() {
suite.Setup()
func (s *NfsSuite) SetupSuite() {
s.Setup()
}
func (suite *NfsSuite) TearDownSuite() {
suite.Teardown()
func (s *NfsSuite) TearDownSuite() {
s.Teardown()
}
func (suite *NfsSuite) Setup() {
suite.namespace = "rook-nfs"
suite.systemNamespace = installer.SystemNamespace(suite.namespace)
suite.instanceCount = 1
func (s *NfsSuite) Setup() {
s.namespace = "rook-nfs"
s.systemNamespace = installer.SystemNamespace(s.namespace)
s.instanceCount = 1
k8shelper, err := utils.CreateK8sHelper(suite.T)
k8shelper, err := utils.CreateK8sHelper(s.T)
v := version.MustParseSemantic(k8shelper.GetK8sServerVersion())
if !v.AtLeast(version.MustParseSemantic("1.14.0")) {
logger.Info("Skipping NFS tests when not at least K8s v1.14")
suite.T().Skip()
s.T().Skip()
}
require.NoError(suite.T(), err)
suite.k8shelper = k8shelper
require.NoError(s.T(), err)
s.k8shelper = k8shelper
k8sversion := suite.k8shelper.GetK8sServerVersion()
k8sversion := s.k8shelper.GetK8sServerVersion()
logger.Infof("Installing nfs server on k8s %s", k8sversion)
suite.installer = installer.NewNFSInstaller(suite.k8shelper, suite.T)
s.installer = installer.NewNFSInstaller(s.k8shelper, s.T)
suite.rwClient = clients.CreateReadWriteOperation(suite.k8shelper)
s.rwClient = clients.CreateReadWriteOperation(s.k8shelper)
err = suite.installer.InstallNFSServer(suite.systemNamespace, suite.namespace, suite.instanceCount)
err = s.installer.InstallNFSServer(s.systemNamespace, s.namespace, s.instanceCount)
if err != nil {
logger.Errorf("nfs server installation failed: %+v", err)
suite.T().Fail()
suite.Teardown()
suite.T().FailNow()
s.T().Fail()
s.Teardown()
s.T().FailNow()
}
}
func (suite *NfsSuite) Teardown() {
suite.installer.GatherAllNFSServerLogs(suite.systemNamespace, suite.namespace, suite.T().Name())
suite.installer.UninstallNFSServer(suite.systemNamespace, suite.namespace)
func (s *NfsSuite) Teardown() {
s.installer.GatherAllNFSServerLogs(s.systemNamespace, s.namespace, s.T().Name())
s.installer.UninstallNFSServer(s.systemNamespace, s.namespace)
}
func (suite *NfsSuite) TestNfsServerInstallation() {
logger.Infof("Verifying that nfs server pod %s is running", suite.namespace)
func (s *NfsSuite) TestNfsServerInstallation() {
logger.Infof("Verifying that nfs server pod %s is running", s.namespace)
// verify nfs server operator is running OK
assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-nfs-operator", suite.systemNamespace, 1, "Running"),
assert.True(s.T(), s.k8shelper.CheckPodCountAndState("rook-nfs-operator", s.systemNamespace, 1, "Running"),
"1 rook-nfs-operator must be in Running state")
// verify nfs server instances are running OK
assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState(suite.namespace, suite.namespace, suite.instanceCount, "Running"),
fmt.Sprintf("%d rook-nfs pods must be in Running state", suite.instanceCount))
assert.True(s.T(), s.k8shelper.CheckPodCountAndState(s.namespace, s.namespace, s.instanceCount, "Running"),
fmt.Sprintf("%d rook-nfs pods must be in Running state", s.instanceCount))
// verify bigger export is running OK
assert.True(suite.T(), true, suite.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim-bigger"))
assert.True(s.T(), true, s.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim-bigger"))
podList, err := suite.rwClient.CreateWriteClient("nfs-pv-claim-bigger")
require.NoError(suite.T(), err)
assert.True(suite.T(), true, suite.checkReadData(podList))
err = suite.rwClient.Delete()
assert.NoError(suite.T(), err)
podList, err := s.rwClient.CreateWriteClient("nfs-pv-claim-bigger")
require.NoError(s.T(), err)
assert.True(s.T(), true, s.checkReadData(podList))
err = s.rwClient.Delete()
assert.NoError(s.T(), err)
// verify another smaller export is running OK
assert.True(suite.T(), true, suite.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim"))
assert.True(s.T(), true, s.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim"))
defer suite.rwClient.Delete() //nolint, delete a nfs consuming pod in rook
podList, err = suite.rwClient.CreateWriteClient("nfs-pv-claim")
require.NoError(suite.T(), err)
assert.True(suite.T(), true, suite.checkReadData(podList))
defer s.rwClient.Delete() //nolint, delete a nfs consuming pod in rook
podList, err = s.rwClient.CreateWriteClient("nfs-pv-claim")
require.NoError(s.T(), err)
assert.True(s.T(), true, s.checkReadData(podList))
}
func (suite *NfsSuite) checkReadData(podList []string) bool {
func (s *NfsSuite) checkReadData(podList []string) bool {
var result string
var err error
// the following for loop retries to read data from the first pod in the pod list
for i := 0; i < utils.RetryLoop; i++ {
// the nfs volume is mounted on "/mnt" and the data(hostname of the pod) is written in "/mnt/data" of the pod
// results stores the hostname of either one of the pod which is same as the pod name, which is read from "/mnt/data"
result, err = suite.rwClient.Read(podList[0])
result, err = s.rwClient.Read(podList[0])
logger.Infof("nfs volume read exited, err: %+v. result: %s", err, result)
if err == nil {
break
@@ -148,7 +148,7 @@ func (suite *NfsSuite) checkReadData(podList []string) bool {
logger.Warning("nfs volume read failed, will try again")
time.Sleep(utils.RetryInterval * time.Second)
}
require.NoError(suite.T(), err)
require.NoError(s.T(), err)
// the value of result must be same as the name of pod.
if result == podList[0] || result == podList[1] {
return true
+1 -1
View File
@@ -17,7 +17,7 @@ trap cleanup SIGINT ERR
SERVER_VERSION=$(kubectl version --short | awk -F "." '/Server Version/ {print $2}')
MINIMUM_VERSION=16
if [ ${SERVER_VERSION} -le ${MINIMUM_VERSION} ]; then
if [ ${SERVER_VERSION} -lt ${MINIMUM_VERSION} ]; then
echo "required minimum kubernetes version 1.$MINIMUM_VERSION.0"
exit
fi