Merge pull request #7390 from travisn/refactor-tests
ceph: Refactor integration tests for simplification
This commit is contained in:
55 files changed
+1426
-4014
No files matched your search
@@ -54,4 +54,4 @@ jobs:
|
||||
|
||||
- name: setup tmate session for debugging
|
||||
if: failure() && contains(github.event.pull_request.labels.*.name, 'debug-flex-suite')
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
tests/scripts/minikube.sh helm
|
||||
tests/scripts/helm.sh up
|
||||
export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1)
|
||||
go test -v -timeout 1800s -run CephHelmSuite github.com/rook/rook/tests/integration
|
||||
SKIP_TEST_CLEANUP=false SKIP_CLEANUP_POLICY=false go test -v -timeout 1800s -run CephHelmSuite github.com/rook/rook/tests/integration
|
||||
|
||||
- name: Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
@@ -56,4 +56,4 @@ jobs:
|
||||
|
||||
- name: setup tmate session for debugging
|
||||
if: failure() && contains(github.event.pull_request.labels.*.name, 'debug-helm-suite')
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
@@ -55,4 +55,4 @@ jobs:
|
||||
|
||||
- name: setup tmate session for debugging
|
||||
if: failure() && contains(github.event.pull_request.labels.*.name, 'debug-multi-cluster-suite')
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
- name: TestCephSmokeSuite
|
||||
run: |
|
||||
export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1)
|
||||
go test -v -timeout 1800s -run CephSmokeSuite github.com/rook/rook/tests/integration
|
||||
SKIP_CLEANUP_POLICY=false go test -v -timeout 1800s -run CephSmokeSuite github.com/rook/rook/tests/integration
|
||||
|
||||
- name: Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
@@ -54,4 +54,4 @@ jobs:
|
||||
|
||||
- name: setup tmate session for debugging
|
||||
if: failure() && contains(github.event.pull_request.labels.*.name, 'debug-smoke-suite')
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
@@ -54,4 +54,4 @@ jobs:
|
||||
|
||||
- name: setup tmate session for debugging
|
||||
if: failure() && contains(github.event.pull_request.labels.*.name, 'debug-upgrade-suite')
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
@@ -459,7 +459,6 @@ The following are the settings for Storage Class Device Sets which can be config
|
||||
The following storage selection settings are specific to Ceph and do not apply to other backends. All variables are key-value pairs represented as strings.
|
||||
|
||||
* `metadataDevice`: Name of a device to use for the metadata of OSDs on each node. Performance can be improved by using a low latency device (such as SSD or NVMe) as the metadata device, while other spinning platter (HDD) devices on a node are used to store data. Provisioning will fail if the user specifies a `metadataDevice` but that device is not used as a metadata device by Ceph. Notably, `ceph-volume` will not use a device of the same device class (HDD, SSD, NVMe) as OSD devices for metadata, resulting in this failure.
|
||||
* `storeType`: `bluestore`, the underlying storage format to use for each OSD. The default is set dynamically to `bluestore` for devices and is the only supported format at this point.
|
||||
* `databaseSizeMB`: The size in MB of a bluestore database. Include quotes around the size.
|
||||
* `walSizeMB`: The size in MB of a bluestore write ahead log (WAL). Include quotes around the size.
|
||||
* `deviceClass`: The [CRUSH device class](https://ceph.io/community/new-luminous-crush-device-classes/) to use for this selection of storage devices. (By default, if a device's class has not already been set, OSDs will automatically set a device's class to either `hdd`, `ssd`, or `nvme` based on the hardware properties exposed by the Linux kernel.) These storage classes can then be used to select the devices backing a storage pool by specifying them as the value of [the pool spec's `deviceClass` field](ceph-pool-crd.md#spec).
|
||||
@@ -764,7 +763,6 @@ spec:
|
||||
- name: "sdc1" # One specific partition. Should not have a file system on it.
|
||||
- name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # both device name and explicit udev links are supported
|
||||
config: # configuration can be specified at the node level which overrides the cluster level config
|
||||
storeType: bluestore
|
||||
- name: "172.17.4.301"
|
||||
deviceFilter: "^sd."
|
||||
```
|
||||
|
||||
@@ -12,7 +12,7 @@ Rook allows creation and customization of object stores through the custom resou
|
||||
|
||||
### Erasure Coded
|
||||
|
||||
Erasure coded pools require the OSDs to use `bluestore` for the configured [`storeType`](ceph-cluster-crd.md#osd-configuration-settings). Additionally, erasure coded pools can only be used with `dataPools`. The `metadataPool` must use a replicated pool.
|
||||
Erasure coded pools can only be used with `dataPools`. The `metadataPool` must use a replicated pool.
|
||||
|
||||
> **NOTE**: This sample requires *at least 3 bluestore OSDs*, with each OSD located on a *different node*.
|
||||
|
||||
|
||||
Vendored
+2
@@ -232,6 +232,8 @@ def RunIntegrationTest(k, v) {
|
||||
sh '''#!/bin/bash
|
||||
set -o pipefail
|
||||
export KUBECONFIG=$HOME/admin.conf \
|
||||
SKIP_TEST_CLEANUP=false \
|
||||
SKIP_CLEANUP_POLICY=false \
|
||||
TEST_ENV_NAME='''+"${k}"+''' \
|
||||
TEST_BASE_DIR="WORKING_DIR" \
|
||||
TEST_LOG_COLLECTION_LEVEL='''+"${env.getLogs}"+''' \
|
||||
|
||||
@@ -68,16 +68,16 @@ EOF
|
||||
build_helm_resources() {
|
||||
# Add helm annotations to all CRDS
|
||||
yq w -i -d'*' "$HELM_CRDS_FILE_PATH" "metadata.annotations[helm.sh/resource-policy]" keep
|
||||
|
||||
|
||||
# add header
|
||||
sed -i '1s/^/{{- if .Values.crds.enabled }}\n/' "$HELM_CRDS_FILE_PATH"
|
||||
sed -i '1s/^/{{- if semverCompare ">=1.16.0" .Capabilities.KubeVersion.GitVersion }}\n/' "$HELM_CRDS_FILE_PATH"
|
||||
|
||||
sed -i '1s/^/{{- if .Values.crds.enabled }}\n/' "$HELM_CRDS_FILE_PATH"
|
||||
|
||||
echo "Generating helm resources.yaml"
|
||||
{
|
||||
# add else
|
||||
echo "{{- else }}"
|
||||
|
||||
|
||||
# add footer
|
||||
cat "$CRDS_BEFORE_1_16_FILE_PATH"
|
||||
# DO NOT REMOVE the empty line, it is necessary
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{{- if semverCompare ">=1.16.0" .Capabilities.KubeVersion.GitVersion }}
|
||||
{{- if .Values.crds.enabled }}
|
||||
{{- if semverCompare ">=1.16.0" .Capabilities.KubeVersion.GitVersion }}
|
||||
apiVersion: apiextensions.k8s.io/v1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
|
||||
@@ -218,7 +218,6 @@ spec:
|
||||
# osdsPerDevice: "5"
|
||||
# - name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # devices can be specified using full udev paths
|
||||
# config: # configuration can be specified at the node level which overrides the cluster level config
|
||||
# storeType: filestore
|
||||
# - name: "172.17.4.301"
|
||||
# deviceFilter: "^sd."
|
||||
# The section for configuring management of daemon disruptions during upgrade or fencing.
|
||||
|
||||
@@ -350,6 +350,13 @@ data:
|
||||
# Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used
|
||||
ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true"
|
||||
|
||||
# Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release
|
||||
# in favor of the CSI driver.
|
||||
ROOK_ENABLE_FLEX_DRIVER: "false"
|
||||
# Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster.
|
||||
# This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs.
|
||||
ROOK_ENABLE_DISCOVERY_DAEMON: "false"
|
||||
|
||||
# (Optional) Admission controller NodeAffinity.
|
||||
# ADMISSION_CONTROLLER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
|
||||
# (Optional) Admission controller tolerations list. Put here list of taints you want to tolerate in YAML format.
|
||||
@@ -475,16 +482,6 @@ spec:
|
||||
- name: DISCOVER_DAEMON_UDEV_BLACKLIST
|
||||
value: "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+"
|
||||
|
||||
# Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release
|
||||
# in favor of the CSI driver.
|
||||
- name: ROOK_ENABLE_FLEX_DRIVER
|
||||
value: "false"
|
||||
|
||||
# Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster.
|
||||
# This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs.
|
||||
- name: ROOK_ENABLE_DISCOVERY_DAEMON
|
||||
value: "false"
|
||||
|
||||
# Whether to start machineDisruptionBudget and machineLabel controller to watch for the osd pods and MDBs.
|
||||
- name: ROOK_ENABLE_MACHINE_DISRUPTION_BUDGET
|
||||
value: "false"
|
||||
|
||||
@@ -274,6 +274,13 @@ data:
|
||||
# Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used
|
||||
ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true"
|
||||
|
||||
# Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release
|
||||
# in favor of the CSI driver.
|
||||
ROOK_ENABLE_FLEX_DRIVER: "false"
|
||||
# Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster.
|
||||
# This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs.
|
||||
ROOK_ENABLE_DISCOVERY_DAEMON: "false"
|
||||
|
||||
# (Optional) Admission controller NodeAffinity.
|
||||
# ADMISSION_CONTROLLER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
|
||||
# (Optional) Admission controller tolerations list. Put here list of taints you want to tolerate in YAML format.
|
||||
@@ -424,16 +431,6 @@ spec:
|
||||
- name: DISCOVER_DAEMON_UDEV_BLACKLIST
|
||||
value: "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+"
|
||||
|
||||
# Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release
|
||||
# in favor of the CSI driver.
|
||||
- name: ROOK_ENABLE_FLEX_DRIVER
|
||||
value: "false"
|
||||
|
||||
# Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster.
|
||||
# This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs.
|
||||
- name: ROOK_ENABLE_DISCOVERY_DAEMON
|
||||
value: "false"
|
||||
|
||||
# Time to wait until the node controller will move Rook pods to other
|
||||
# nodes after detecting an unreachable node.
|
||||
# Pods affected by this setting are:
|
||||
|
||||
@@ -47,9 +47,6 @@ func init() {
|
||||
operatorCmd.Flags().DurationVar(&mon.HealthCheckInterval, "mon-healthcheck-interval", mon.HealthCheckInterval, "mon health check interval (duration)")
|
||||
operatorCmd.Flags().DurationVar(&mon.MonOutTimeout, "mon-out-timeout", mon.MonOutTimeout, "mon out timeout (duration)")
|
||||
|
||||
operatorCmd.Flags().BoolVar(&operator.EnableFlexDriver, "enable-flex-driver", true, "enable the rook flex driver")
|
||||
operatorCmd.Flags().BoolVar(&operator.EnableDiscoveryDaemon, "enable-discovery-daemon", false, "enable the rook discovery daemon")
|
||||
|
||||
// csi deployment templates
|
||||
operatorCmd.Flags().StringVar(&csi.RBDPluginTemplatePath, "csi-rbd-plugin-template-path", csi.DefaultRBDPluginTemplatePath, "path to ceph-csi rbd plugin template")
|
||||
|
||||
|
||||
@@ -115,7 +115,6 @@ func addOSDConfigFlags(command *cobra.Command) {
|
||||
// OSD store config flags
|
||||
command.Flags().IntVar(&cfg.storeConfig.WalSizeMB, "osd-wal-size", osdcfg.WalDefaultSizeMB, "default size (MB) for OSD write ahead log (WAL) (bluestore)")
|
||||
command.Flags().IntVar(&cfg.storeConfig.DatabaseSizeMB, "osd-database-size", 0, "default size (MB) for OSD database (bluestore)")
|
||||
command.Flags().StringVar(&cfg.storeConfig.StoreType, "osd-store", "", "type of backing OSD store to use (bluestore or filestore)")
|
||||
command.Flags().IntVar(&cfg.storeConfig.OSDsPerDevice, "osds-per-device", 1, "the number of OSDs per device")
|
||||
command.Flags().BoolVar(&cfg.storeConfig.EncryptedDevice, "encrypted-device", false, "whether to encrypt the OSD with dmcrypt")
|
||||
command.Flags().StringVar(&cfg.storeConfig.DeviceClass, "osd-crush-device-class", "", "The device class for all OSDs configured on this node")
|
||||
|
||||
@@ -25,9 +25,10 @@ import (
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
)
|
||||
|
||||
// RunAllCephCommandsInToolbox - when running the e2e tests, all ceph commands need to be run in the toolbox.
|
||||
// RunAllCephCommandsInToolboxPod - when running the e2e tests, all ceph commands need to be run in the toolbox.
|
||||
// Everywhere else, the ceph tools are assumed to be in the container where we can shell out.
|
||||
var RunAllCephCommandsInToolbox = false
|
||||
// This is the name of the pod.
|
||||
var RunAllCephCommandsInToolboxPod string
|
||||
|
||||
const (
|
||||
// AdminUsername is the name of the admin user
|
||||
@@ -66,8 +67,8 @@ func FinalizeCephCommandArgs(command string, clusterInfo *ClusterInfo, args []st
|
||||
}
|
||||
|
||||
// If the command should be run inside the toolbox pod, include the kubectl args to call the toolbox
|
||||
if RunAllCephCommandsInToolbox {
|
||||
toolArgs := []string{"exec", "-i", "rook-ceph-tools", "-n", clusterInfo.Namespace, "--", command}
|
||||
if RunAllCephCommandsInToolboxPod != "" {
|
||||
toolArgs := []string{"exec", "-i", RunAllCephCommandsInToolboxPod, "-n", clusterInfo.Namespace, "--", command}
|
||||
return Kubectl, append(toolArgs, args...)
|
||||
}
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
)
|
||||
|
||||
func TestFinalizeCephCommandArgs(t *testing.T) {
|
||||
RunAllCephCommandsInToolbox = false
|
||||
RunAllCephCommandsInToolboxPod = ""
|
||||
configDir := "/var/lib/rook/rook-ceph"
|
||||
expectedCommand := "ceph"
|
||||
args := []string{"quorum_status"}
|
||||
@@ -43,7 +43,7 @@ func TestFinalizeCephCommandArgs(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFinalizeRadosGWAdminCommandArgs(t *testing.T) {
|
||||
RunAllCephCommandsInToolbox = false
|
||||
RunAllCephCommandsInToolboxPod = ""
|
||||
configDir := "/var/lib/rook/rook-ceph"
|
||||
expectedCommand := "radosgw-admin"
|
||||
args := []string{
|
||||
@@ -73,7 +73,7 @@ func TestFinalizeRadosGWAdminCommandArgs(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFinalizeCephCommandArgsToolBox(t *testing.T) {
|
||||
RunAllCephCommandsInToolbox = true
|
||||
RunAllCephCommandsInToolboxPod = "rook-ceph-tools"
|
||||
configDir := "/var/lib/rook/rook-ceph"
|
||||
expectedCommand := "ceph"
|
||||
args := []string{"health"}
|
||||
@@ -93,5 +93,5 @@ func TestFinalizeCephCommandArgsToolBox(t *testing.T) {
|
||||
cmd, args := FinalizeCephCommandArgs(expectedCommand, clusterInfo, args, configDir)
|
||||
assert.Exactly(t, "kubectl", cmd)
|
||||
assert.Exactly(t, expectedArgs, args)
|
||||
RunAllCephCommandsInToolbox = false
|
||||
RunAllCephCommandsInToolboxPod = ""
|
||||
}
|
||||
@@ -38,7 +38,7 @@ func TestCephArgs(t *testing.T) {
|
||||
assert.Equal(t, "--name=client.admin", args[3])
|
||||
assert.Equal(t, "--keyring=/etc/a/client.admin.keyring", args[4])
|
||||
|
||||
RunAllCephCommandsInToolbox = true
|
||||
RunAllCephCommandsInToolboxPod = "rook-ceph-tools"
|
||||
args = []string{}
|
||||
command, args = FinalizeCephCommandArgs(CephTool, clusterInfo, args, "/etc")
|
||||
assert.Equal(t, Kubectl, command)
|
||||
@@ -51,7 +51,7 @@ func TestCephArgs(t *testing.T) {
|
||||
assert.Equal(t, "--", args[5])
|
||||
assert.Equal(t, CephTool, args[6])
|
||||
assert.Equal(t, "--connect-timeout=15", args[7])
|
||||
RunAllCephCommandsInToolbox = false
|
||||
RunAllCephCommandsInToolboxPod = ""
|
||||
|
||||
// cluster under /var/lib/rook
|
||||
args = []string{"myarg"}
|
||||
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/osd/kms"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
|
||||
controllerutil "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/ceph/csi"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
@@ -51,7 +52,6 @@ import (
|
||||
|
||||
const (
|
||||
controllerName = "ceph-cluster-controller"
|
||||
enableFlexDriver = "ROOK_ENABLE_FLEX_DRIVER"
|
||||
detectCephVersionTimeout = 15 * time.Minute
|
||||
)
|
||||
|
||||
@@ -293,7 +293,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
|
||||
// Remove finalizer
|
||||
err = removeFinalizer(r.client, request.NamespacedName)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, errors.Wrap(err, "failed to remove finalize")
|
||||
return reconcile.Result{}, errors.Wrap(err, "failed to remove finalizer")
|
||||
}
|
||||
|
||||
// Return and do not requeue. Successful deletion.
|
||||
@@ -421,8 +421,7 @@ func (c *ClusterController) checkIfVolumesExist(cluster *cephv1.CephCluster) err
|
||||
return err
|
||||
}
|
||||
}
|
||||
flexDriverEnabled := os.Getenv(enableFlexDriver) != "false"
|
||||
if !flexDriverEnabled {
|
||||
if !controllerutil.FlexDriverEnabled(c.context) {
|
||||
logger.Debugf("Flex driver disabled, skipping check for volume attachments for cluster %q", cluster.Namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -33,7 +33,6 @@ func GetConfigStoreName(nodeName string) string {
|
||||
}
|
||||
|
||||
const (
|
||||
StoreTypeKey = "storeType"
|
||||
WalSizeMBKey = "walSizeMB"
|
||||
DatabaseSizeMBKey = "databaseSizeMB"
|
||||
JournalSizeMBKey = "journalSizeMB"
|
||||
@@ -45,7 +44,6 @@ const (
|
||||
|
||||
// StoreConfig represents the configuration of an OSD on a device.
|
||||
type StoreConfig struct {
|
||||
StoreType string `json:"storeType,omitempty"`
|
||||
WalSizeMB int `json:"walSizeMB,omitempty"`
|
||||
DatabaseSizeMB int `json:"databaseSizeMB,omitempty"`
|
||||
OSDsPerDevice int `json:"osdsPerDevice,omitempty"`
|
||||
@@ -66,8 +64,6 @@ func ToStoreConfig(config map[string]string) StoreConfig {
|
||||
storeConfig := NewStoreConfig()
|
||||
for k, v := range config {
|
||||
switch k {
|
||||
case StoreTypeKey:
|
||||
storeConfig.StoreType = v
|
||||
case WalSizeMBKey:
|
||||
storeConfig.WalSizeMB = convertToIntIgnoreErr(v)
|
||||
case DatabaseSizeMBKey:
|
||||
|
||||
@@ -29,7 +29,6 @@ import (
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1"
|
||||
kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
|
||||
opconfig "github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
@@ -325,7 +324,6 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
|
||||
return nil, errors.New("empty volumes")
|
||||
}
|
||||
|
||||
storeType := config.Bluestore
|
||||
osdID := strconv.Itoa(osd.ID)
|
||||
tiniEnvVar := v1.EnvVar{Name: "TINI_SUBREAPER", Value: ""}
|
||||
envVars := append(c.getConfigEnvVars(osdProps, dataDir), []v1.EnvVar{
|
||||
@@ -335,7 +333,6 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
|
||||
envVars = append(envVars, []v1.EnvVar{
|
||||
{Name: "ROOK_OSD_UUID", Value: osd.UUID},
|
||||
{Name: "ROOK_OSD_ID", Value: osdID},
|
||||
{Name: "ROOK_OSD_STORE_TYPE", Value: storeType},
|
||||
{Name: "ROOK_CEPH_MON_HOST",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{
|
||||
|
||||
@@ -59,6 +59,18 @@ var (
|
||||
OperatorCephBaseImageVersion string
|
||||
)
|
||||
|
||||
func FlexDriverEnabled(context *clusterd.Context) bool {
|
||||
// Ignore the error. In the remote chance that the configmap fails to be read, we will default to disabling the flex driver
|
||||
value, _ := k8sutil.GetOperatorSetting(context.Clientset, OperatorSettingConfigMapName, "ROOK_ENABLE_FLEX_DRIVER", "false")
|
||||
return value == "true"
|
||||
}
|
||||
|
||||
func DiscoveryDaemonEnabled(context *clusterd.Context) bool {
|
||||
// Ignore the error. In the remote chance that the configmap fails to be read, we will default to disabling the discovery daemon
|
||||
value, _ := k8sutil.GetOperatorSetting(context.Clientset, OperatorSettingConfigMapName, "ROOK_ENABLE_DISCOVERY_DAEMON", "false")
|
||||
return value == "true"
|
||||
}
|
||||
|
||||
// CheckForCancelledOrchestration checks whether a cancellation has been requested
|
||||
func CheckForCancelledOrchestration(context *clusterd.Context) error {
|
||||
defer context.RequestCancelOrchestration.UnSet()
|
||||
@@ -130,6 +142,7 @@ func IsReadyToReconcile(c client.Client, clustercontext *clusterd.Context, names
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debugf("%q: CephCluster %q initial reconcile is not complete yet...", controllerName, namespacedName.Namespace)
|
||||
return cephCluster, false, cephClusterExists, WaitForRequeueIfCephClusterNotReady
|
||||
}
|
||||
|
||||
|
||||
@@ -312,7 +312,7 @@ func (r *ReconcileObjectStoreUser) initializeObjectStoreContext(u *cephv1.CephOb
|
||||
|
||||
r.objContext = objContext
|
||||
r.objContext.Endpoint = store.Status.Info["endpoint"]
|
||||
if store.Status.Info["secureEndpoint"] != "" {
|
||||
if store.Status != nil && store.Status.Info["secureEndpoint"] != "" {
|
||||
r.objContext.Endpoint = store.Status.Info["secureEndpoint"]
|
||||
}
|
||||
if r.objContext.Endpoint == "" {
|
||||
|
||||
@@ -57,12 +57,6 @@ var provisionerConfigs = map[string]string{
|
||||
}
|
||||
|
||||
var (
|
||||
// EnableFlexDriver Whether to enable the flex driver. If true, the rook-ceph-agent daemonset will be started.
|
||||
EnableFlexDriver = true
|
||||
|
||||
// EnableDiscoveryDaemon Whether to enable the daemon for device discovery. If true, the rook-ceph-discover daemonset will be started.
|
||||
EnableDiscoveryDaemon = true
|
||||
|
||||
// ImmediateRetryResult Return this for a immediate retry of the reconciliation loop with the same request object.
|
||||
ImmediateRetryResult = reconcile.Result{Requeue: true}
|
||||
)
|
||||
@@ -119,7 +113,7 @@ func (o *Operator) Run() error {
|
||||
defer stopFunc()
|
||||
|
||||
rookDiscover := discover.New(o.context.Clientset)
|
||||
if EnableDiscoveryDaemon {
|
||||
if opcontroller.DiscoveryDaemonEnabled(o.context) {
|
||||
if err := rookDiscover.Start(o.operatorNamespace, o.rookImage, o.securityAccount, true); err != nil {
|
||||
return errors.Wrap(err, "failed to start device discovery daemonset")
|
||||
}
|
||||
@@ -145,7 +139,7 @@ func (o *Operator) Run() error {
|
||||
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
// For Flex Driver, run volume provisioner for each of the supported configurations
|
||||
if EnableFlexDriver {
|
||||
if opcontroller.FlexDriverEnabled(o.context) {
|
||||
for name, vendor := range provisionerConfigs {
|
||||
volumeProvisioner := provisioner.New(o.context, vendor)
|
||||
pc := controller.NewProvisionController(
|
||||
@@ -216,7 +210,7 @@ func (o *Operator) updateDrivers() error {
|
||||
return errors.Errorf("rook operator namespace is not provided. expose it via downward API in the rook operator manifest file using environment variable %s", k8sutil.PodNamespaceEnvVar)
|
||||
}
|
||||
|
||||
if EnableFlexDriver {
|
||||
if opcontroller.FlexDriverEnabled(o.context) {
|
||||
rookAgent := agent.New(o.context.Clientset)
|
||||
if err := rookAgent.Start(o.operatorNamespace, o.rookImage, o.securityAccount); err != nil {
|
||||
return errors.Wrap(err, "error starting agent daemonset")
|
||||
|
||||
@@ -62,26 +62,26 @@ func (b *BlockOperation) Create(manifest string, size int) (string, error) {
|
||||
|
||||
}
|
||||
|
||||
func (b *BlockOperation) CreateStorageClassAndPVC(csi bool, pvcNamespace, clusterNamespace, systemNamespace, poolName, storageClassName, reclaimPolicy, blockName, mode string) error {
|
||||
if err := b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockPoolDef(poolName, clusterNamespace, "1")); err != nil {
|
||||
func (b *BlockOperation) CreateStorageClassAndPVC(pvcNamespace, poolName, storageClassName, reclaimPolicy, blockName, mode string) error {
|
||||
if err := b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockPool(poolName, "1")); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockStorageClassDef(csi, poolName, storageClassName, reclaimPolicy, clusterNamespace, systemNamespace)); err != nil {
|
||||
if err := b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockStorageClass(poolName, storageClassName, reclaimPolicy)); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.k8sClient.ResourceOperation("apply", b.manifests.GetPVC(blockName, pvcNamespace, storageClassName, mode, "1M"))
|
||||
return b.k8sClient.ResourceOperation("apply", installer.GetPVC(blockName, pvcNamespace, storageClassName, mode, "1M"))
|
||||
}
|
||||
|
||||
func (b *BlockOperation) CreatePVC(namespace, claimName, storageClassName, mode, size string) error {
|
||||
return b.k8sClient.ResourceOperation("apply", b.manifests.GetPVC(claimName, namespace, storageClassName, mode, size))
|
||||
return b.k8sClient.ResourceOperation("apply", installer.GetPVC(claimName, namespace, storageClassName, mode, size))
|
||||
}
|
||||
|
||||
func (b *BlockOperation) CreatePod(podName, claimName, namespace, mountPoint string, readOnly bool) error {
|
||||
return b.k8sClient.ResourceOperation("apply", b.manifests.GetPod(podName, claimName, namespace, mountPoint, readOnly))
|
||||
return b.k8sClient.ResourceOperation("apply", installer.GetPodWithVolume(podName, claimName, namespace, mountPoint, readOnly))
|
||||
}
|
||||
|
||||
func (b *BlockOperation) CreateStorageClass(csi bool, poolName, storageClassName, reclaimPolicy, namespace string) error {
|
||||
return b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockStorageClassDef(csi, poolName, storageClassName, reclaimPolicy, namespace, installer.SystemNamespace(namespace)))
|
||||
return b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockStorageClass(poolName, storageClassName, reclaimPolicy))
|
||||
}
|
||||
|
||||
func (b *BlockOperation) DeletePVC(namespace, claimName string) error {
|
||||
@@ -91,27 +91,27 @@ func (b *BlockOperation) DeletePVC(namespace, claimName string) error {
|
||||
}
|
||||
|
||||
func (b *BlockOperation) CreatePVCRestore(namespace, claimName, snapshotName, storageClassName, mode, size string) error {
|
||||
return b.k8sClient.ResourceOperation("apply", b.manifests.GetPVCRestore(claimName, snapshotName, namespace, storageClassName, mode, size))
|
||||
return b.k8sClient.ResourceOperation("apply", installer.GetPVCRestore(claimName, snapshotName, namespace, storageClassName, mode, size))
|
||||
}
|
||||
|
||||
func (b *BlockOperation) CreatePVCClone(namespace, cloneClaimName, parentClaimName, storageClassName, mode, size string) error {
|
||||
return b.k8sClient.ResourceOperation("apply", b.manifests.GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, mode, size))
|
||||
return b.k8sClient.ResourceOperation("apply", installer.GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, mode, size))
|
||||
}
|
||||
|
||||
func (b *BlockOperation) CreateSnapshotClass(snapshotClassName, deletePolicy, namespace string) error {
|
||||
return b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockSnapshotClass(snapshotClassName, namespace, namespace, deletePolicy))
|
||||
return b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockSnapshotClass(snapshotClassName, deletePolicy))
|
||||
}
|
||||
|
||||
func (b *BlockOperation) DeleteSnapshotClass(snapshotClassName, deletePolicy, namespace string) error {
|
||||
return b.k8sClient.ResourceOperation("delete", b.manifests.GetBlockSnapshotClass(snapshotClassName, namespace, namespace, deletePolicy))
|
||||
return b.k8sClient.ResourceOperation("delete", b.manifests.GetBlockSnapshotClass(snapshotClassName, deletePolicy))
|
||||
}
|
||||
|
||||
func (b *BlockOperation) CreateSnapshot(snapshotName, claimName, snapshotClassName, namespace string) error {
|
||||
return b.k8sClient.ResourceOperation("apply", b.manifests.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace))
|
||||
return b.k8sClient.ResourceOperation("apply", installer.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace))
|
||||
}
|
||||
|
||||
func (b *BlockOperation) DeleteSnapshot(snapshotName, claimName, snapshotClassName, namespace string) error {
|
||||
return b.k8sClient.ResourceOperation("delete", b.manifests.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace))
|
||||
return b.k8sClient.ResourceOperation("delete", installer.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace))
|
||||
}
|
||||
|
||||
func (b *BlockOperation) DeleteStorageClass(storageClassName string) error {
|
||||
|
||||
@@ -37,20 +37,20 @@ func CreateBucketOperation(k8sh *utils.K8sHelper, manifests installer.CephManife
|
||||
}
|
||||
|
||||
func (b *BucketOperation) CreateBucketStorageClass(namespace string, storeName string, storageClassName string, reclaimPolicy string, region string) error {
|
||||
return b.k8sh.ResourceOperation("create", b.manifests.GetBucketStorageClass(namespace, storeName, storageClassName, reclaimPolicy, region))
|
||||
return b.k8sh.ResourceOperation("create", b.manifests.GetBucketStorageClass(storeName, storageClassName, reclaimPolicy, region))
|
||||
}
|
||||
|
||||
func (b *BucketOperation) DeleteBucketStorageClass(namespace string, storeName string, storageClassName string, reclaimPolicy string, region string) error {
|
||||
err := b.k8sh.ResourceOperation("delete", b.manifests.GetBucketStorageClass(namespace, storeName, storageClassName, reclaimPolicy, region))
|
||||
err := b.k8sh.ResourceOperation("delete", b.manifests.GetBucketStorageClass(storeName, storageClassName, reclaimPolicy, region))
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *BucketOperation) CreateObc(obcName string, storageClassName string, bucketName string, maxObject string, createBucket bool) error {
|
||||
return b.k8sh.ResourceOperation("create", b.manifests.GetObc(obcName, storageClassName, bucketName, maxObject, createBucket))
|
||||
return b.k8sh.ResourceOperation("create", b.manifests.GetOBC(obcName, storageClassName, bucketName, maxObject, createBucket))
|
||||
}
|
||||
|
||||
func (b *BucketOperation) DeleteObc(obcName string, storageClassName string, bucketName string, maxObject string, createBucket bool) error {
|
||||
return b.k8sh.ResourceOperation("delete", b.manifests.GetObc(obcName, storageClassName, bucketName, maxObject, createBucket))
|
||||
return b.k8sh.ResourceOperation("delete", b.manifests.GetOBC(obcName, storageClassName, bucketName, maxObject, createBucket))
|
||||
}
|
||||
|
||||
// CheckOBC, returns true if the obc, secret and configmap are all in the "check" state,
|
||||
|
||||
@@ -42,7 +42,7 @@ func CreateClientOperation(k8sh *utils.K8sHelper, manifests installer.CephManife
|
||||
// Create creates a client in Rook
|
||||
func (c *ClientOperation) Create(name, namespace string, caps map[string]string) error {
|
||||
logger.Infof("creating the client via CRD")
|
||||
if err := c.k8sh.ResourceOperation("apply", c.manifests.GetClient(namespace, name, caps)); err != nil {
|
||||
if err := c.k8sh.ResourceOperation("apply", c.manifests.GetClient(name, caps)); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
@@ -76,7 +76,7 @@ func (c *ClientOperation) Get(clusterInfo *client.ClusterInfo, clientName string
|
||||
func (c *ClientOperation) Update(clusterInfo *client.ClusterInfo, clientName string, caps map[string]string) (updatedcaps map[string]string, error error) {
|
||||
context := c.k8sh.MakeContext()
|
||||
logger.Infof("updating the client via CRD")
|
||||
if err := c.k8sh.ResourceOperation("apply", c.manifests.GetClient(clusterInfo.Namespace, clientName, caps)); err != nil {
|
||||
if err := c.k8sh.ResourceOperation("apply", c.manifests.GetClient(clientName, caps)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ func CreateFilesystemOperation(k8sh *utils.K8sHelper, manifests installer.CephMa
|
||||
// Create creates a filesystem in Rook
|
||||
func (f *FilesystemOperation) Create(name, namespace string, activeCount int) error {
|
||||
logger.Infof("creating the filesystem via CRD")
|
||||
if err := f.k8sh.ResourceOperation("apply", f.manifests.GetFilesystem(namespace, name, activeCount)); err != nil {
|
||||
if err := f.k8sh.ResourceOperation("apply", f.manifests.GetFilesystem(name, activeCount)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -58,32 +58,32 @@ func (f *FilesystemOperation) Create(name, namespace string, activeCount int) er
|
||||
|
||||
// CreateStorageClass creates a storage class for CephFS clients
|
||||
func (f *FilesystemOperation) CreateStorageClass(fsName, systemNamespace, namespace, storageClassName string) error {
|
||||
return f.k8sh.ResourceOperation("apply", f.manifests.GetFileStorageClassDef(fsName, storageClassName, systemNamespace, namespace))
|
||||
return f.k8sh.ResourceOperation("apply", f.manifests.GetFileStorageClass(fsName, storageClassName))
|
||||
}
|
||||
|
||||
// CreateSnapshotClass creates a snapshot class for CephFS clients
|
||||
func (f *FilesystemOperation) CreateSnapshotClass(snapshotClassName, reclaimPolicy, namespace string) error {
|
||||
return f.k8sh.ResourceOperation("apply", f.manifests.GetFileStorageSnapshotClass(snapshotClassName, namespace, namespace, reclaimPolicy))
|
||||
return f.k8sh.ResourceOperation("apply", f.manifests.GetFileStorageSnapshotClass(snapshotClassName, reclaimPolicy))
|
||||
}
|
||||
|
||||
// CreatePVCRestore creates a pvc from snapshot
|
||||
func (f *FilesystemOperation) CreatePVCRestore(namespace, claimName, snapshotName, storageClassName, mode, size string) error {
|
||||
return f.k8sh.ResourceOperation("apply", f.manifests.GetPVCRestore(claimName, snapshotName, namespace, storageClassName, mode, size))
|
||||
return f.k8sh.ResourceOperation("apply", installer.GetPVCRestore(claimName, snapshotName, namespace, storageClassName, mode, size))
|
||||
}
|
||||
|
||||
// CreatePVCClone creates a pvc from pvc
|
||||
func (f *FilesystemOperation) CreatePVCClone(namespace, cloneClaimName, parentClaimName, storageClassName, mode, size string) error {
|
||||
return f.k8sh.ResourceOperation("apply", f.manifests.GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, mode, size))
|
||||
return f.k8sh.ResourceOperation("apply", installer.GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, mode, size))
|
||||
}
|
||||
|
||||
// CreateSnapshot creates a snapshot from pvc
|
||||
func (f *FilesystemOperation) CreateSnapshot(snapshotName, claimName, snapshotClassName, namespace string) error {
|
||||
return f.k8sh.ResourceOperation("apply", f.manifests.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace))
|
||||
return f.k8sh.ResourceOperation("apply", installer.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace))
|
||||
}
|
||||
|
||||
// DeleteSnapshot deletes the snapshot
|
||||
func (f *FilesystemOperation) DeleteSnapshot(snapshotName, claimName, snapshotClassName, namespace string) error {
|
||||
return f.k8sh.ResourceOperation("delete", f.manifests.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace))
|
||||
return f.k8sh.ResourceOperation("delete", installer.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace))
|
||||
}
|
||||
|
||||
func (f *FilesystemOperation) DeletePVC(namespace, claimName string) error {
|
||||
@@ -104,21 +104,21 @@ func (f *FilesystemOperation) DeleteStorageClass(storageClassName string) error
|
||||
}
|
||||
|
||||
func (f *FilesystemOperation) CreatePVC(namespace, claimName, storageClassName, mode, size string) error {
|
||||
return f.k8sh.ResourceOperation("apply", f.manifests.GetPVC(claimName, namespace, storageClassName, mode, size))
|
||||
return f.k8sh.ResourceOperation("apply", installer.GetPVC(claimName, namespace, storageClassName, mode, size))
|
||||
}
|
||||
|
||||
func (f *FilesystemOperation) CreatePod(podName, claimName, namespace, mountPoint string, readOnly bool) error {
|
||||
return f.k8sh.ResourceOperation("apply", f.manifests.GetPod(podName, claimName, namespace, mountPoint, readOnly))
|
||||
return f.k8sh.ResourceOperation("apply", installer.GetPodWithVolume(podName, claimName, namespace, mountPoint, readOnly))
|
||||
}
|
||||
|
||||
func (f *FilesystemOperation) DeleteSnapshotClass(snapshotClassName, deletePolicy, namespace string) error {
|
||||
return f.k8sh.ResourceOperation("delete", f.manifests.GetFileStorageSnapshotClass(snapshotClassName, namespace, namespace, deletePolicy))
|
||||
return f.k8sh.ResourceOperation("delete", f.manifests.GetFileStorageSnapshotClass(snapshotClassName, deletePolicy))
|
||||
}
|
||||
|
||||
// ScaleDown scales down the number of active metadata servers of a filesystem in Rook
|
||||
func (f *FilesystemOperation) ScaleDown(name, namespace string) error {
|
||||
logger.Infof("scaling down the number of filesystem active metadata servers via CRD")
|
||||
if err := f.k8sh.ResourceOperation("apply", f.manifests.GetFilesystem(namespace, name, 1)); err != nil {
|
||||
if err := f.k8sh.ResourceOperation("apply", f.manifests.GetFilesystem(name, 1)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -144,7 +144,7 @@ func (f *FilesystemOperation) Delete(name, namespace string) error {
|
||||
}
|
||||
|
||||
logger.Infof("Deleted filesystem %s in namespace %s", name, namespace)
|
||||
return f.k8sh.WaitForCustomResourceDeletion(namespace, crdCheckerFunc)
|
||||
return f.k8sh.WaitForCustomResourceDeletion(namespace, name, crdCheckerFunc)
|
||||
}
|
||||
|
||||
// List lists filesystems in Rook
|
||||
|
||||
@@ -42,7 +42,7 @@ func CreateNFSOperation(k8sh *utils.K8sHelper, manifests installer.CephManifests
|
||||
func (n *NFSOperation) Create(namespace, name, pool string, daemonCount int) error {
|
||||
|
||||
logger.Infof("creating the NFS daemons via CRD")
|
||||
if err := n.k8sh.ResourceOperation("apply", n.manifests.GetNFS(namespace, name, pool, daemonCount)); err != nil {
|
||||
if err := n.k8sh.ResourceOperation("apply", n.manifests.GetNFS(name, pool, daemonCount)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ func CreateObjectOperation(k8sh *utils.K8sHelper, manifests installer.CephManife
|
||||
func (o *ObjectOperation) Create(namespace, storeName string, replicaCount int32) error {
|
||||
|
||||
logger.Infof("creating the object store via CRD")
|
||||
if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStore(namespace, storeName, int(replicaCount), rgwPort)); err != nil {
|
||||
if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStore(storeName, int(replicaCount), rgwPort)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -68,7 +68,7 @@ func (o *ObjectUserOperation) UserSecretExists(namespace string, store string, u
|
||||
func (o *ObjectUserOperation) Create(namespace string, userid string, displayName string, store string) error {
|
||||
|
||||
logger.Infof("creating the object store user via CRD")
|
||||
if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStoreUser(namespace, userid, displayName, store)); err != nil {
|
||||
if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStoreUser(userid, displayName, store)); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -50,7 +50,7 @@ func (p *PoolOperation) Update(name, namespace string, replicas int) error {
|
||||
}
|
||||
|
||||
func (p *PoolOperation) createOrUpdatePool(name, namespace, action string, replicas int) error {
|
||||
return p.k8sh.ResourceOperation(action, p.manifests.GetBlockPoolDef(name, namespace, strconv.Itoa(replicas)))
|
||||
return p.k8sh.ResourceOperation(action, p.manifests.GetBlockPool(name, strconv.Itoa(replicas)))
|
||||
}
|
||||
|
||||
func (p *PoolOperation) ListCephPools(clusterInfo *client.ClusterInfo) ([]client.CephStoragePoolSummary, error) {
|
||||
@@ -147,5 +147,5 @@ func (p *PoolOperation) DeletePool(blockClient *BlockOperation, clusterInfo *cli
|
||||
return err
|
||||
}
|
||||
|
||||
return p.k8sh.WaitForCustomResourceDeletion(clusterInfo.Namespace, crdCheckerFunc)
|
||||
return p.k8sh.WaitForCustomResourceDeletion(clusterInfo.Namespace, poolName, crdCheckerFunc)
|
||||
}
|
||||
@@ -41,7 +41,7 @@ func CreateRBDMirrorOperation(k8sh *utils.K8sHelper, manifests installer.CephMan
|
||||
func (r *RBDMirrorOperation) Create(namespace, name string, daemonCount int) error {
|
||||
|
||||
logger.Infof("creating the RBDMirror daemons via CRD")
|
||||
if err := r.k8sh.ResourceOperation("apply", r.manifests.GetRBDMirror(namespace, name, daemonCount)); err != nil {
|
||||
if err := r.k8sh.ResourceOperation("apply", r.manifests.GetRBDMirror(name, daemonCount)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -131,7 +131,7 @@ func (ci *CassandraInstaller) DeleteCassandraCluster(namespace string) {
|
||||
_, err := ci.k8sHelper.RookClientset.CassandraV1alpha1().Clusters(namespace).Get(ctx, namespace, metav1.GetOptions{})
|
||||
return err
|
||||
}
|
||||
err = ci.k8sHelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc)
|
||||
err = ci.k8sHelper.WaitForCustomResourceDeletion(namespace, namespace, crdCheckerFunc)
|
||||
assert.NoError(ci.T(), err)
|
||||
|
||||
// Delete Namespace
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,142 @@
|
||||
/*
|
||||
Copyright 2021 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package installer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
)
|
||||
|
||||
// TestCephSettings struct for handling panic and test suite tear down
|
||||
type TestCephSettings struct {
|
||||
DataDirHostPath string
|
||||
ClusterName string
|
||||
Namespace string
|
||||
OperatorNamespace string
|
||||
StorageClassName string
|
||||
UseHelm bool
|
||||
UsePVC bool
|
||||
Mons int
|
||||
UseCrashPruner bool
|
||||
MultipleMgrs bool
|
||||
SkipOSDCreation bool
|
||||
UseCSI bool
|
||||
EnableDiscovery bool
|
||||
EnableAdmissionController bool
|
||||
IsExternal bool
|
||||
SkipClusterCleanup bool
|
||||
SkipCleanupPolicy bool
|
||||
DirectMountToolbox bool
|
||||
RookVersion string
|
||||
CephVersion cephv1.CephVersionSpec
|
||||
}
|
||||
|
||||
func (s *TestCephSettings) ApplyEnvVars() {
|
||||
// skip the cleanup by default
|
||||
s.SkipClusterCleanup = true
|
||||
if os.Getenv("SKIP_TEST_CLEANUP") == "false" {
|
||||
s.SkipClusterCleanup = false
|
||||
}
|
||||
s.SkipCleanupPolicy = true
|
||||
if os.Getenv("SKIP_CLEANUP_POLICY") == "false" {
|
||||
s.SkipCleanupPolicy = false
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestCephSettings) readManifest(filename string) string {
|
||||
rootDir, err := utils.FindRookRoot()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
manifest := path.Join(rootDir, "cluster/examples/kubernetes/ceph", filename)
|
||||
logger.Infof("Reading manifest: %s", manifest)
|
||||
contents, err := ioutil.ReadFile(manifest)
|
||||
if err != nil {
|
||||
panic(errors.Wrapf(err, "failed to read manifest at %s", manifest))
|
||||
}
|
||||
return replaceNamespaces(manifest, string(contents), s.OperatorNamespace, s.Namespace)
|
||||
}
|
||||
|
||||
func (s *TestCephSettings) readManifestFromGithub(filename string) string {
|
||||
return s.readManifestFromGithubWithClusterNamespace(filename, s.Namespace)
|
||||
}
|
||||
|
||||
func (s *TestCephSettings) readManifestFromGithubWithClusterNamespace(filename, clusterNamespace string) string {
|
||||
url := fmt.Sprintf("https://raw.githubusercontent.com/rook/rook/%s/cluster/examples/kubernetes/ceph/%s", s.RookVersion, filename)
|
||||
logger.Infof("Retrieving manifest: %s", url)
|
||||
// #nosec G107 This is only test code and is expected to read from a url
|
||||
response, err := http.Get(url)
|
||||
if err != nil {
|
||||
panic(errors.Wrapf(err, "failed to read manifest from %s", url))
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
content, err := ioutil.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
panic(errors.Wrapf(err, "failed to read content from %s", url))
|
||||
}
|
||||
return replaceNamespaces(url, string(content), s.OperatorNamespace, clusterNamespace)
|
||||
}
|
||||
|
||||
func (s *TestCephSettings) replaceOperatorSettings(manifest string) string {
|
||||
manifest = strings.ReplaceAll(manifest, `# CSI_LOG_LEVEL: "0"`, `CSI_LOG_LEVEL: "5"`)
|
||||
manifest = strings.ReplaceAll(manifest, `ROOK_ENABLE_DISCOVERY_DAEMON: "false"`, fmt.Sprintf(`ROOK_ENABLE_DISCOVERY_DAEMON: "%t"`, s.EnableDiscovery))
|
||||
manifest = strings.ReplaceAll(manifest, `ROOK_ENABLE_FLEX_DRIVER: "false"`, fmt.Sprintf(`ROOK_ENABLE_FLEX_DRIVER: "%t"`, !s.UseCSI))
|
||||
manifest = strings.ReplaceAll(manifest, `ROOK_CSI_ENABLE_CEPHFS: "true"`, fmt.Sprintf(`ROOK_CSI_ENABLE_CEPHFS: "%t"`, s.UseCSI))
|
||||
manifest = strings.ReplaceAll(manifest, `ROOK_CSI_ENABLE_RBD: "true"`, fmt.Sprintf(`ROOK_CSI_ENABLE_RBD: "%t"`, s.UseCSI))
|
||||
return manifest
|
||||
}
|
||||
|
||||
func replaceNamespaces(name, manifest, operatorNamespace, clusterNamespace string) string {
|
||||
// RBAC and related namespaces
|
||||
manifest = strings.ReplaceAll(manifest, "rook-ceph # namespace:operator", operatorNamespace)
|
||||
manifest = strings.ReplaceAll(manifest, "rook-ceph # namespace:cluster", clusterNamespace)
|
||||
manifest = strings.ReplaceAll(manifest, "rook-ceph-external # namespace:cluster", clusterNamespace)
|
||||
// Double space only needed for v1.5 upgrade test
|
||||
manifest = strings.ReplaceAll(manifest, "rook-ceph # namespace:operator", operatorNamespace)
|
||||
|
||||
// SCC namespaces for operator and Ceph daemons
|
||||
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-system # serviceaccount:namespace:operator", operatorNamespace+":rook-ceph-system")
|
||||
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-mgr # serviceaccount:namespace:cluster", clusterNamespace+":rook-ceph-mgr")
|
||||
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-osd # serviceaccount:namespace:cluster", clusterNamespace+":rook-ceph-osd")
|
||||
|
||||
// SCC namespaces for CSI driver
|
||||
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-rbd-plugin-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-rbd-plugin-sa")
|
||||
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-rbd-provisioner-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-rbd-provisioner-sa")
|
||||
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-cephfs-plugin-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-cephfs-plugin-sa")
|
||||
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-cephfs-provisioner-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-cephfs-provisioner-sa")
|
||||
|
||||
// CSI Drivers
|
||||
manifest = strings.ReplaceAll(manifest, "rook-ceph.cephfs.csi.ceph.com # driver:namespace:operator", operatorNamespace+".cephfs.csi.ceph.com")
|
||||
manifest = strings.ReplaceAll(manifest, "rook-ceph.rbd.csi.ceph.com # driver:namespace:operator", operatorNamespace+".rbd.csi.ceph.com")
|
||||
|
||||
// Bucket storage class
|
||||
manifest = strings.ReplaceAll(manifest, "rook-ceph.ceph.rook.io/bucket # driver:namespace:cluster", clusterNamespace+".ceph.rook.io/bucket")
|
||||
if strings.Contains(manifest, "namespace:operator") || strings.Contains(manifest, "namespace:cluster") || strings.Contains(manifest, "driver:namespace:") || strings.Contains(manifest, "serviceaccount:namespace:") {
|
||||
logger.Infof("BAD MANIFEST:\n%s", manifest)
|
||||
panic(fmt.Sprintf("manifest %s still contains a namespace identifier", name))
|
||||
}
|
||||
return manifest
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
/*
|
||||
Copyright 2021 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package installer
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func GetPodWithVolume(podName, claimName, namespace, mountPath string, readOnly bool) string {
|
||||
return `
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: ` + podName + `
|
||||
namespace: ` + namespace + `
|
||||
spec:
|
||||
containers:
|
||||
- name: ` + podName + `
|
||||
image: busybox
|
||||
command: ["/bin/sleep", "infinity"]
|
||||
imagePullPolicy: IfNotPresent
|
||||
volumeMounts:
|
||||
- mountPath: ` + mountPath + `
|
||||
name: csivol
|
||||
volumes:
|
||||
- name: csivol
|
||||
persistentVolumeClaim:
|
||||
claimName: ` + claimName + `
|
||||
readOnly: ` + strconv.FormatBool(readOnly) + `
|
||||
restartPolicy: Never
|
||||
`
|
||||
}
|
||||
|
||||
func GetPVC(claimName, namespace, storageClassName, accessModes, size string) string {
|
||||
return `apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: ` + claimName + `
|
||||
namespace: ` + namespace + `
|
||||
spec:
|
||||
storageClassName: ` + storageClassName + `
|
||||
accessModes:
|
||||
- ` + accessModes + `
|
||||
resources:
|
||||
requests:
|
||||
storage: ` + size
|
||||
}
|
||||
|
||||
func GetPVCRestore(claimName, snapshotName, namespace, storageClassName, accessModes, size string) string {
|
||||
return `apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: ` + claimName + `
|
||||
namespace: ` + namespace + `
|
||||
spec:
|
||||
storageClassName: ` + storageClassName + `
|
||||
dataSource:
|
||||
name: ` + snapshotName + `
|
||||
kind: VolumeSnapshot
|
||||
apiGroup: snapshot.storage.k8s.io
|
||||
accessModes:
|
||||
- ` + accessModes + `
|
||||
resources:
|
||||
requests:
|
||||
storage: ` + size
|
||||
}
|
||||
|
||||
func GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, accessModes, size string) string {
|
||||
return `apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: ` + cloneClaimName + `
|
||||
namespace: ` + namespace + `
|
||||
spec:
|
||||
storageClassName: ` + storageClassName + `
|
||||
dataSource:
|
||||
name: ` + parentClaimName + `
|
||||
kind: PersistentVolumeClaim
|
||||
accessModes:
|
||||
- ` + accessModes + `
|
||||
resources:
|
||||
requests:
|
||||
storage: ` + size
|
||||
}
|
||||
|
||||
func GetSnapshot(snapshotName, claimName, snapshotClassName, namespace string) string {
|
||||
return `apiVersion: snapshot.storage.k8s.io/v1beta1
|
||||
kind: VolumeSnapshot
|
||||
metadata:
|
||||
name: ` + snapshotName + `
|
||||
namespace: ` + namespace + `
|
||||
spec:
|
||||
volumeSnapshotClassName: ` + snapshotClassName + `
|
||||
source:
|
||||
persistentVolumeClaimName: ` + claimName
|
||||
}
|
||||
@@ -45,11 +45,6 @@ var (
|
||||
deleteFromStdinArgs = append(deleteArgs, "-")
|
||||
)
|
||||
|
||||
type TestSuite interface {
|
||||
Setup()
|
||||
Teardown()
|
||||
}
|
||||
|
||||
func SkipTestSuite(name string) bool {
|
||||
testsToRun := testStorageProvider()
|
||||
// jenkins passes "null" if the env var is not set.
|
||||
|
||||
@@ -166,7 +166,7 @@ func (h *NFSInstaller) UninstallNFSServer(systemNamespace, namespace string) {
|
||||
_, err := h.k8shelper.RookClientset.NfsV1alpha1().NFSServers(namespace).Get(ctx, namespace, metav1.GetOptions{})
|
||||
return err
|
||||
}
|
||||
err = h.k8shelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc)
|
||||
err = h.k8shelper.WaitForCustomResourceDeletion(namespace, namespace, crdCheckerFunc)
|
||||
checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace))
|
||||
|
||||
err = h.k8shelper.DeleteResource("namespace", namespace)
|
||||
|
||||
@@ -62,7 +62,7 @@ func (h *HelmHelper) InstallLocalRookHelmChart(namespace, chartSettings string)
|
||||
return errors.Wrap(err, "failed to find rook root")
|
||||
}
|
||||
chartDir := path.Join(rootDir, "cluster/charts/rook-ceph/")
|
||||
cmdArgs := []string{"install", HelmDeployName, chartDir, "-f", path.Join(chartDir, "values.yaml")}
|
||||
cmdArgs := []string{"install", "--create-namespace", HelmDeployName, chartDir, "-f", path.Join(chartDir, "values.yaml")}
|
||||
if namespace != "" {
|
||||
cmdArgs = append(cmdArgs, "--namespace", namespace)
|
||||
}
|
||||
|
||||
@@ -347,7 +347,12 @@ func (k8sh *K8sHelper) ResourceOperationFromTemplate(action string, podDefinitio
|
||||
// ResourceOperation performs a kubectl action on a pod definition
|
||||
func (k8sh *K8sHelper) ResourceOperation(action string, manifest string) error {
|
||||
args := []string{action, "-f", "-"}
|
||||
logger.Infof("kubectl %s manifest:\n%s", action, manifest)
|
||||
maxManifestCharsToPrint := 4000
|
||||
if len(manifest) > maxManifestCharsToPrint {
|
||||
logger.Infof("kubectl %s manifest (too long to print)", action)
|
||||
} else {
|
||||
logger.Infof("kubectl %s manifest:\n%s", action, manifest)
|
||||
}
|
||||
_, err := k8sh.KubectlWithStdin(manifest, args...)
|
||||
if err == nil {
|
||||
return nil
|
||||
@@ -381,23 +386,23 @@ func (k8sh *K8sHelper) DeleteResource(args ...string) error {
|
||||
}
|
||||
|
||||
// WaitForCustomResourceDeletion waits for the CRD deletion
|
||||
func (k8sh *K8sHelper) WaitForCustomResourceDeletion(namespace string, checkerFunc func() error) error {
|
||||
func (k8sh *K8sHelper) WaitForCustomResourceDeletion(namespace, name string, checkerFunc func() error) error {
|
||||
|
||||
// wait for the operator to finalize and delete the CRD
|
||||
for i := 0; i < 60; i++ {
|
||||
err := checkerFunc()
|
||||
if err == nil {
|
||||
logger.Infof("custom resource %s still exists", namespace)
|
||||
logger.Infof("custom resource %q in namespace %q still exists", name, namespace)
|
||||
time.Sleep(2 * time.Second)
|
||||
continue
|
||||
}
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Infof("custom resource %s deleted", namespace)
|
||||
logger.Infof("custom resource %q in namespace %s deleted", name, namespace)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
logger.Errorf("gave up deleting custom resource %s", namespace)
|
||||
logger.Errorf("gave up deleting custom resource %q ", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -931,6 +936,16 @@ func (k8sh *K8sHelper) PrintPVCs(namespace string, detailed bool) {
|
||||
}
|
||||
}
|
||||
|
||||
func (k8sh *K8sHelper) PrintResources(namespace, name string) {
|
||||
args := []string{"-n", namespace, "get", name, "-o", "yaml"}
|
||||
result, err := k8sh.Kubectl(args...)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to get resource %s. %v", name, err)
|
||||
} else {
|
||||
logger.Infof("%s\n", result)
|
||||
}
|
||||
}
|
||||
|
||||
func (k8sh *K8sHelper) PrintStorageClasses(detailed bool) {
|
||||
ctx := context.TODO()
|
||||
scs, err := k8sh.Clientset.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{})
|
||||
@@ -1218,6 +1233,7 @@ func (k8sh *K8sHelper) IsPodInExpectedState(podNamePattern string, namespace str
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("waiting for pod with label app=%s in namespace %q to be in state %q...", podNamePattern, namespace, state)
|
||||
time.Sleep(RetryInterval * time.Second)
|
||||
}
|
||||
|
||||
|
||||
@@ -24,7 +24,6 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
@@ -269,8 +268,6 @@ func runBlockCSITest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.
|
||||
poolName := "replicapool"
|
||||
storageClassName := "rook-ceph-block"
|
||||
blockName := "block-pv-claim"
|
||||
systemNamespace := installer.SystemNamespace(namespace)
|
||||
|
||||
podNameWithPVRetained := "block-test-retained"
|
||||
poolNameRetained := "replicapoolretained"
|
||||
storageClassNameRetained := "rook-ceph-block-retained"
|
||||
@@ -287,10 +284,10 @@ func runBlockCSITest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.
|
||||
assert.Equal(s.T(), 0, len(initBlockImages), "there should not already be any images in the pool")
|
||||
|
||||
logger.Infof("step 1: Create block storage")
|
||||
err := helper.BlockClient.CreateStorageClassAndPVC(true, defaultNamespace, namespace, systemNamespace, poolName, storageClassName, "Delete", blockName, "ReadWriteOnce")
|
||||
err := helper.BlockClient.CreateStorageClassAndPVC(defaultNamespace, poolName, storageClassName, "Delete", blockName, "ReadWriteOnce")
|
||||
require.NoError(s.T(), err)
|
||||
require.NoError(s.T(), retryBlockImageCountCheck(helper, clusterInfo, 1), "Make sure a new block is created")
|
||||
err = helper.BlockClient.CreateStorageClassAndPVC(true, defaultNamespace, namespace, systemNamespace, poolNameRetained, storageClassNameRetained, "Retain", blockNameRetained, "ReadWriteOnce")
|
||||
err = helper.BlockClient.CreateStorageClassAndPVC(defaultNamespace, poolNameRetained, storageClassNameRetained, "Retain", blockNameRetained, "ReadWriteOnce")
|
||||
require.NoError(s.T(), err)
|
||||
require.NoError(s.T(), retryBlockImageCountCheck(helper, clusterInfo, 2), "Make sure another new block is created")
|
||||
logger.Infof("Block Storage created successfully")
|
||||
@@ -405,7 +402,8 @@ func restartOSDPods(k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
|
||||
options := metav1.DeleteOptions{}
|
||||
err := k8sh.Clientset.CoreV1().Pods(namespace).Delete(ctx, pod.Name, options)
|
||||
assert.NoError(s.T(), err)
|
||||
|
||||
}
|
||||
for _, pod := range pods.Items {
|
||||
logger.Infof("Waiting for osd pod %s to be deleted", pod.Name)
|
||||
deleted := k8sh.WaitUntilPodIsDeleted(pod.Name, namespace)
|
||||
assert.True(s.T(), deleted)
|
||||
@@ -417,20 +415,20 @@ func restartOSDPods(k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
|
||||
assert.NoError(s.T(), err)
|
||||
}
|
||||
|
||||
func runBlockCSITestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, clusterNamespace, systemNamespace string, version cephv1.CephVersionSpec) {
|
||||
func runBlockCSITestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings) {
|
||||
checkSkipCSITest(s.T(), k8sh)
|
||||
|
||||
logger.Infof("Block Storage End to End Integration Test - create storageclass,pool and pvc")
|
||||
logger.Infof("Running on Rook Cluster %s", clusterNamespace)
|
||||
clusterInfo := client.AdminClusterInfo(clusterNamespace)
|
||||
logger.Infof("Running on Rook Cluster %s", settings.Namespace)
|
||||
clusterInfo := client.AdminClusterInfo(settings.Namespace)
|
||||
poolName := "rookpool"
|
||||
storageClassName := "rook-ceph-block-lite"
|
||||
blockName := "test-block-claim-lite"
|
||||
podName := "test-pod-lite"
|
||||
defer blockTestDataCleanUp(helper, k8sh, s, clusterInfo, poolName, storageClassName, blockName, podName, true)
|
||||
setupBlockLite(helper, k8sh, s, clusterInfo, systemNamespace, poolName, storageClassName, blockName, podName, version)
|
||||
setupBlockLite(helper, k8sh, s, clusterInfo, poolName, storageClassName, blockName, podName)
|
||||
if !skipSnapshotTest(k8sh) {
|
||||
blockCSISnapshotTest(helper, k8sh, s, storageClassName, clusterNamespace)
|
||||
blockCSISnapshotTest(helper, k8sh, s, storageClassName, settings.Namespace)
|
||||
}
|
||||
|
||||
if !skipCloneTest(k8sh) {
|
||||
@@ -439,7 +437,7 @@ func runBlockCSITestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s su
|
||||
}
|
||||
|
||||
func setupBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, clusterInfo *client.ClusterInfo,
|
||||
systemNamespace, poolName, storageClassName, blockName, podName string, version cephv1.CephVersionSpec) {
|
||||
poolName, storageClassName, blockName, podName string) {
|
||||
|
||||
// Check initial number of blocks
|
||||
initialBlocks, err := helper.BlockClient.ListAllImages(clusterInfo)
|
||||
@@ -449,7 +447,7 @@ func setupBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S
|
||||
|
||||
logger.Infof("step : Create Pool,StorageClass and PVC")
|
||||
|
||||
err = helper.BlockClient.CreateStorageClassAndPVC(true, defaultNamespace, clusterInfo.Namespace, systemNamespace, poolName, storageClassName, "Delete", blockName, "ReadWriteOnce")
|
||||
err = helper.BlockClient.CreateStorageClassAndPVC(defaultNamespace, poolName, storageClassName, "Delete", blockName, "ReadWriteOnce")
|
||||
require.NoError(s.T(), err)
|
||||
|
||||
require.True(s.T(), k8sh.WaitUntilPVCIsBound(defaultNamespace, blockName))
|
||||
|
||||
@@ -25,7 +25,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
@@ -87,36 +86,15 @@ func checkIfRookClusterIsHealthy(s suite.Suite, testClient *clients.TestClient,
|
||||
require.Nil(s.T(), err)
|
||||
}
|
||||
|
||||
func HandlePanics(r interface{}, op installer.TestSuite, t func() *testing.T) {
|
||||
func HandlePanics(r interface{}, uninstaller func(), t func() *testing.T) {
|
||||
if r != nil {
|
||||
logger.Infof("unexpected panic occurred during test %s, --> %v", t().Name(), r)
|
||||
t().Fail()
|
||||
op.Teardown()
|
||||
uninstaller()
|
||||
t().FailNow()
|
||||
}
|
||||
}
|
||||
|
||||
// TestCluster struct for handling panic and test suite tear down
|
||||
type TestCluster struct {
|
||||
installer *installer.CephInstaller
|
||||
kh *utils.K8sHelper
|
||||
helper *clients.TestClient
|
||||
T func() *testing.T
|
||||
clusterName string
|
||||
namespace string
|
||||
storeType string
|
||||
storageClassName string
|
||||
useHelm bool
|
||||
usePVC bool
|
||||
mons int
|
||||
rbdMirrorWorkers int
|
||||
rookCephCleanup bool
|
||||
skipOSDCreation bool
|
||||
minimalMatrixK8sVersion string
|
||||
rookVersion string
|
||||
cephVersion cephv1.CephVersionSpec
|
||||
}
|
||||
|
||||
func checkIfShouldRunForMinimalTestMatrix(t func() *testing.T, k8sh *utils.K8sHelper, version string) {
|
||||
testArgs := os.Getenv("TEST_ARGUMENTS")
|
||||
if !strings.Contains(testArgs, "min-test-matrix") {
|
||||
@@ -140,50 +118,27 @@ func checkIfShouldRunForMinimalTestMatrix(t func() *testing.T, k8sh *utils.K8sHe
|
||||
logger.Infof("Running test suite since kube version is %q", kubeVersion)
|
||||
}
|
||||
|
||||
// StartTestCluster creates new instance of TestCluster struct
|
||||
func StartTestCluster(t func() *testing.T, cluster *TestCluster) (*TestCluster, *utils.K8sHelper) {
|
||||
kh, err := utils.CreateK8sHelper(t)
|
||||
// StartTestCluster creates new instance of TestCephSettings struct
|
||||
func StartTestCluster(t func() *testing.T, settings *installer.TestCephSettings, minimalMatrixK8sVersion string) (*installer.CephInstaller, *utils.K8sHelper) {
|
||||
k8shelper, err := utils.CreateK8sHelper(t)
|
||||
require.NoError(t(), err)
|
||||
checkIfShouldRunForMinimalTestMatrix(t, kh, cluster.minimalMatrixK8sVersion)
|
||||
checkIfShouldRunForMinimalTestMatrix(t, k8shelper, minimalMatrixK8sVersion)
|
||||
|
||||
cluster.installer = installer.NewCephInstaller(t, kh.Clientset, cluster.useHelm, cluster.clusterName, cluster.rookVersion, cluster.cephVersion, cluster.rookCephCleanup)
|
||||
cluster.kh = kh
|
||||
cluster.helper = nil
|
||||
cluster.T = t
|
||||
|
||||
if cluster.rookVersion != installer.VersionMaster {
|
||||
// make sure we have the images from a previous release locally so the test doesn't hit a timeout
|
||||
assert.NoError(t(), kh.GetDockerImage("rook/ceph:"+cluster.rookVersion))
|
||||
}
|
||||
|
||||
assert.NoError(t(), kh.GetDockerImage(cluster.cephVersion.Image))
|
||||
|
||||
cluster.Setup()
|
||||
return cluster, kh
|
||||
}
|
||||
|
||||
// Setup is a wrapper for setting up rook
|
||||
func (op *TestCluster) Setup() {
|
||||
// Turn on DEBUG logging
|
||||
capnslog.SetGlobalLogLevel(capnslog.DEBUG)
|
||||
isRookInstalled, err := op.installer.InstallRook(op.namespace, op.storeType, op.usePVC, op.storageClassName,
|
||||
cephv1.MonSpec{Count: op.mons, AllowMultiplePerNode: true}, false /* startWithAllNodes */, op.rbdMirrorWorkers, op.skipOSDCreation, op.rookVersion)
|
||||
|
||||
installer := installer.NewCephInstaller(t, k8shelper.Clientset, settings)
|
||||
isRookInstalled, err := installer.InstallRook()
|
||||
|
||||
if !isRookInstalled || err != nil {
|
||||
logger.Errorf("Rook was not installed successfully: %v", err)
|
||||
if !op.installer.T().Failed() {
|
||||
op.installer.GatherAllRookLogs(op.installer.T().Name(), op.namespace, installer.SystemNamespace(op.namespace))
|
||||
if !installer.T().Failed() {
|
||||
installer.GatherAllRookLogs(t().Name(), settings.Namespace, settings.OperatorNamespace)
|
||||
}
|
||||
op.T().Fail()
|
||||
op.Teardown()
|
||||
op.T().FailNow()
|
||||
t().Fail()
|
||||
installer.UninstallRook()
|
||||
t().FailNow()
|
||||
}
|
||||
}
|
||||
|
||||
// SetInstallData updates the installer helper based on the version of Rook desired
|
||||
func (op *TestCluster) SetInstallData(version string) {}
|
||||
|
||||
// Teardown is a wrapper for tearDown after Suite
|
||||
func (op *TestCluster) Teardown() {
|
||||
op.installer.UninstallRook(op.namespace)
|
||||
return installer, k8shelper
|
||||
}
|
||||
@@ -239,83 +239,83 @@ func fileSystemCSISnapshotTest(helper *clients.TestClient, k8sh *utils.K8sHelper
|
||||
|
||||
// Smoke Test for File System Storage - Test check the following operations on Filesystem Storage in order
|
||||
// Create,Mount,Write,Read,Unmount and Delete.
|
||||
func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName string, useCSI bool, preserveFilesystemOnDelete bool) {
|
||||
if useCSI {
|
||||
func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string, preserveFilesystemOnDelete bool) {
|
||||
if settings.UseCSI {
|
||||
checkSkipCSITest(s.T(), k8sh)
|
||||
}
|
||||
|
||||
defer fileTestDataCleanUp(helper, k8sh, s, filePodName, namespace, filesystemName)
|
||||
logger.Infof("Running on Rook Cluster %s", namespace)
|
||||
defer fileTestDataCleanUp(helper, k8sh, s, filePodName, settings.Namespace, filesystemName)
|
||||
logger.Infof("Running on Rook Cluster %s", settings.Namespace)
|
||||
logger.Infof("File Storage End To End Integration Test - create, mount, write to, read from, and unmount")
|
||||
activeCount := 2
|
||||
createFilesystem(helper, k8sh, s, namespace, filesystemName, activeCount)
|
||||
createFilesystem(helper, k8sh, s, settings, filesystemName, activeCount)
|
||||
|
||||
if preserveFilesystemOnDelete {
|
||||
_, err := k8sh.Kubectl("-n", namespace, "patch", "CephFilesystem", filesystemName, "--type=merge", "-p", `{"spec": {"preserveFilesystemOnDelete": true}}`)
|
||||
_, err := k8sh.Kubectl("-n", settings.Namespace, "patch", "CephFilesystem", filesystemName, "--type=merge", "-p", `{"spec": {"preserveFilesystemOnDelete": true}}`)
|
||||
assert.NoError(s.T(), err)
|
||||
}
|
||||
|
||||
// Create a test pod where CephFS is consumed without user creds
|
||||
storageClassName := "cephfs-storageclass"
|
||||
err := helper.FSClient.CreateStorageClass(filesystemName, installer.SystemNamespace(namespace), namespace, storageClassName)
|
||||
err := helper.FSClient.CreateStorageClass(filesystemName, settings.OperatorNamespace, settings.Namespace, storageClassName)
|
||||
assert.NoError(s.T(), err)
|
||||
createFilesystemConsumerPod(helper, k8sh, s, namespace, filesystemName, storageClassName, useCSI)
|
||||
createFilesystemConsumerPod(helper, k8sh, s, settings, filesystemName, storageClassName)
|
||||
|
||||
// Test reading and writing to the first pod
|
||||
err = writeAndReadToFilesystem(helper, k8sh, s, namespace, filePodName, "test_file")
|
||||
err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, filePodName, "test_file")
|
||||
assert.NoError(s.T(), err)
|
||||
|
||||
// TODO: Also mount with user credentials with the CSI driver
|
||||
if !useCSI {
|
||||
if !settings.UseCSI {
|
||||
// Create a test pod where CephFS is consumed with a mountUser and mountSecret specified.
|
||||
createFilesystemMountCephCredentials(helper, k8sh, s, namespace, filesystemName)
|
||||
createFilesystemMountUserConsumerPod(helper, k8sh, s, namespace, filesystemName, storageClassName)
|
||||
createFilesystemMountCephCredentials(helper, k8sh, s, settings, filesystemName)
|
||||
createFilesystemMountUserConsumerPod(helper, k8sh, s, settings, filesystemName, storageClassName)
|
||||
|
||||
// Test reading and writing to the second pod
|
||||
err = writeAndReadToFilesystem(helper, k8sh, s, namespace, fileMountUserPodName, "canttouchthis")
|
||||
err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, fileMountUserPodName, "canttouchthis")
|
||||
assert.Error(s.T(), err, "we should not be able to write to file canttouchthis on CephFS `/`")
|
||||
err = writeAndReadToFilesystem(helper, k8sh, s, namespace, fileMountUserPodName, "foo/test_file")
|
||||
err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, fileMountUserPodName, "foo/test_file")
|
||||
assert.NoError(s.T(), err, "we should be able to write to the `/foo` directory on CephFS")
|
||||
|
||||
cleanupFilesystemConsumer(helper, k8sh, s, namespace, fileMountUserPodName)
|
||||
cleanupFilesystemConsumer(helper, k8sh, s, settings.Namespace, fileMountUserPodName)
|
||||
assert.NoError(s.T(), err)
|
||||
}
|
||||
|
||||
// Start the NFS daemons
|
||||
testNFSDaemons(helper, k8sh, s, namespace, filesystemName)
|
||||
testNFSDaemons(helper, k8sh, s, settings, filesystemName)
|
||||
|
||||
// Cleanup the filesystem and its clients
|
||||
cleanupFilesystemConsumer(helper, k8sh, s, namespace, filePodName)
|
||||
cleanupFilesystemConsumer(helper, k8sh, s, settings.Namespace, filePodName)
|
||||
assert.NoError(s.T(), err)
|
||||
downscaleMetadataServers(helper, k8sh, s, namespace, filesystemName)
|
||||
cleanupFilesystem(helper, k8sh, s, namespace, filesystemName)
|
||||
downscaleMetadataServers(helper, k8sh, s, settings.Namespace, filesystemName)
|
||||
cleanupFilesystem(helper, k8sh, s, settings.Namespace, filesystemName)
|
||||
err = helper.FSClient.DeleteStorageClass(storageClassName)
|
||||
assertNoErrorUnlessNotFound(s, err)
|
||||
|
||||
if preserveFilesystemOnDelete {
|
||||
fses, err := helper.FSClient.List(namespace)
|
||||
fses, err := helper.FSClient.List(settings.Namespace)
|
||||
assert.NoError(s.T(), err)
|
||||
assert.Len(s.T(), fses, 1)
|
||||
assert.Equal(s.T(), fses[0].Name, filesystemName)
|
||||
|
||||
err = helper.FSClient.Delete(filesystemName, namespace)
|
||||
err = helper.FSClient.Delete(filesystemName, settings.Namespace)
|
||||
assert.NoError(s.T(), err)
|
||||
}
|
||||
}
|
||||
|
||||
func testNFSDaemons(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) {
|
||||
func testNFSDaemons(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) {
|
||||
name := "my-nfs"
|
||||
err := helper.NFSClient.Create(namespace, name, filesystemName+"-data0", 2)
|
||||
err := helper.NFSClient.Create(settings.Namespace, name, filesystemName+"-data0", 2)
|
||||
require.Nil(s.T(), err)
|
||||
|
||||
err = helper.NFSClient.Delete(namespace, name)
|
||||
err = helper.NFSClient.Delete(settings.Namespace, name)
|
||||
assert.Nil(s.T(), err)
|
||||
}
|
||||
|
||||
func createFilesystemConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName, storageClassName string, useCSI bool) {
|
||||
err := createPodWithFilesystem(k8sh, s, filePodName, namespace, filesystemName, storageClassName, false, useCSI)
|
||||
func createFilesystemConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName, storageClassName string) {
|
||||
err := createPodWithFilesystem(k8sh, s, settings, filePodName, filesystemName, storageClassName, false)
|
||||
require.NoError(s.T(), err)
|
||||
filePodRunning := k8sh.IsPodRunning(filePodName, namespace)
|
||||
filePodRunning := k8sh.IsPodRunning(filePodName, settings.Namespace)
|
||||
require.True(s.T(), filePodRunning, "make sure file-test pod is in running state")
|
||||
logger.Infof("File system mounted successfully")
|
||||
}
|
||||
@@ -358,36 +358,36 @@ func cleanupFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suit
|
||||
}
|
||||
|
||||
// Test File System Creation on Rook that was installed on a custom namespace i.e. Namespace != "rook" and delete it again
|
||||
func runFileE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) {
|
||||
func runFileE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) {
|
||||
checkSkipCSITest(s.T(), k8sh)
|
||||
logger.Infof("File Storage End to End Integration Test - create Filesystem and make sure mds pod is running")
|
||||
logger.Infof("Running on Rook Cluster %s", namespace)
|
||||
logger.Infof("Running on Rook Cluster %s", settings.Namespace)
|
||||
activeCount := 1
|
||||
createFilesystem(helper, k8sh, s, namespace, filesystemName, activeCount)
|
||||
createFilesystem(helper, k8sh, s, settings, filesystemName, activeCount)
|
||||
// Create a test pod where CephFS is consumed without user creds
|
||||
storageClassName := "cephfs-storageclass"
|
||||
err := helper.FSClient.CreateStorageClass(filesystemName, namespace, namespace, storageClassName)
|
||||
err := helper.FSClient.CreateStorageClass(filesystemName, settings.OperatorNamespace, settings.Namespace, storageClassName)
|
||||
assert.NoError(s.T(), err)
|
||||
assert.NoError(s.T(), err)
|
||||
if !skipSnapshotTest(k8sh) {
|
||||
fileSystemCSISnapshotTest(helper, k8sh, s, storageClassName, namespace)
|
||||
fileSystemCSISnapshotTest(helper, k8sh, s, storageClassName, settings.Namespace)
|
||||
}
|
||||
|
||||
if !skipCloneTest(k8sh) {
|
||||
fileSystemCSICloneTest(helper, k8sh, s, storageClassName, namespace)
|
||||
fileSystemCSICloneTest(helper, k8sh, s, storageClassName, settings.Namespace)
|
||||
}
|
||||
cleanupFilesystem(helper, k8sh, s, namespace, filesystemName)
|
||||
cleanupFilesystem(helper, k8sh, s, settings.Namespace, filesystemName)
|
||||
err = helper.FSClient.DeleteStorageClass(storageClassName)
|
||||
assertNoErrorUnlessNotFound(s, err)
|
||||
}
|
||||
|
||||
func createFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName string, activeCount int) {
|
||||
func createFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string, activeCount int) {
|
||||
logger.Infof("Create file System")
|
||||
fscErr := helper.FSClient.Create(filesystemName, namespace, activeCount)
|
||||
fscErr := helper.FSClient.Create(filesystemName, settings.Namespace, activeCount)
|
||||
require.Nil(s.T(), fscErr)
|
||||
logger.Infof("File system %s created", filesystemName)
|
||||
|
||||
filesystemList, _ := helper.FSClient.List(namespace)
|
||||
filesystemList, _ := helper.FSClient.List(settings.Namespace)
|
||||
require.Equal(s.T(), 1, len(filesystemList), "There should be one shared file system present")
|
||||
}
|
||||
|
||||
@@ -399,13 +399,12 @@ func fileTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s su
|
||||
assert.NoError(s.T(), err)
|
||||
}
|
||||
|
||||
func createPodWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podName, namespace, filesystemName, storageClassName string, mountUser, useCSI bool) error {
|
||||
func createPodWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, podName, filesystemName, storageClassName string, mountUser bool) error {
|
||||
var testPodManifest string
|
||||
if useCSI {
|
||||
testPodManifest = getFilesystemCSITestPod(podName, namespace, storageClassName)
|
||||
if settings.UseCSI {
|
||||
testPodManifest = getFilesystemCSITestPod(settings, podName, storageClassName)
|
||||
} else {
|
||||
driverName := installer.SystemNamespace(namespace)
|
||||
testPodManifest = getFilesystemFlexTestPod(podName, namespace, filesystemName, driverName, mountUser)
|
||||
testPodManifest = getFilesystemFlexTestPod(settings, podName, filesystemName, mountUser)
|
||||
}
|
||||
if err := k8sh.ResourceOperation("create", testPodManifest); err != nil {
|
||||
return fmt.Errorf("failed to create pod -- %s. %+v", testPodManifest, err)
|
||||
@@ -413,7 +412,7 @@ func createPodWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podName, name
|
||||
return nil
|
||||
}
|
||||
|
||||
func getFilesystemFlexTestPod(podName, namespace, filesystemName, driverName string, mountUser bool) string {
|
||||
func getFilesystemFlexTestPod(settings *installer.TestCephSettings, podName, filesystemName string, mountUser bool) string {
|
||||
mountUserInsert := ""
|
||||
if mountUser {
|
||||
mountUserInsert = `
|
||||
@@ -425,7 +424,7 @@ func getFilesystemFlexTestPod(podName, namespace, filesystemName, driverName str
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: ` + podName + `
|
||||
namespace: ` + namespace + `
|
||||
namespace: ` + settings.Namespace + `
|
||||
spec:
|
||||
containers:
|
||||
- name: ` + podName + `
|
||||
@@ -444,23 +443,23 @@ spec:
|
||||
volumes:
|
||||
- name: ` + filesystemName + `
|
||||
flexVolume:
|
||||
driver: ceph.rook.io/` + driverName + `
|
||||
driver: ceph.rook.io/` + settings.OperatorNamespace + `
|
||||
fsType: ceph
|
||||
options:
|
||||
fsName: ` + filesystemName + `
|
||||
clusterNamespace: ` + namespace + mountUserInsert + `
|
||||
clusterNamespace: ` + settings.Namespace + mountUserInsert + `
|
||||
restartPolicy: Always
|
||||
`
|
||||
}
|
||||
|
||||
func getFilesystemCSITestPod(podName, namespace, storageClassName string) string {
|
||||
func getFilesystemCSITestPod(settings *installer.TestCephSettings, podName, storageClassName string) string {
|
||||
claimName := podName
|
||||
return `
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: ` + claimName + `
|
||||
namespace: ` + namespace + `
|
||||
namespace: ` + settings.Namespace + `
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
@@ -473,7 +472,7 @@ apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: ` + podName + `
|
||||
namespace: ` + namespace + `
|
||||
namespace: ` + settings.Namespace + `
|
||||
spec:
|
||||
containers:
|
||||
- name: ` + podName + `
|
||||
@@ -496,20 +495,20 @@ spec:
|
||||
`
|
||||
}
|
||||
|
||||
func createFilesystemMountCephCredentials(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) {
|
||||
func createFilesystemMountCephCredentials(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) {
|
||||
ctx := context.TODO()
|
||||
// Create agent binding for access to Secrets
|
||||
err := k8sh.ResourceOperation("apply", getFilesystemAgentMountSecretsBinding(namespace))
|
||||
err := k8sh.ResourceOperation("apply", getFilesystemAgentMountSecretsBinding(settings.Namespace))
|
||||
require.Nil(s.T(), err)
|
||||
// Mount CephFS in toolbox and create /foo directory on it
|
||||
logger.Info("Creating /foo directory on CephFS")
|
||||
_, err = k8sh.Exec(namespace, "rook-ceph-tools", "mkdir", []string{"-p", utils.TestMountPath})
|
||||
_, err = k8sh.Exec(settings.Namespace, client.RunAllCephCommandsInToolboxPod, "mkdir", []string{"-p", utils.TestMountPath})
|
||||
require.Nil(s.T(), err)
|
||||
_, err = k8sh.ExecWithRetry(10, namespace, "rook-ceph-tools", "bash", []string{"-c", fmt.Sprintf("mount -t ceph -o mds_namespace=%s,name=admin,secret=$(grep key /etc/ceph/keyring | awk '{print $3}') $(grep mon_host /etc/ceph/ceph.conf | awk '{print $3}'):/ %s", filesystemName, utils.TestMountPath)})
|
||||
_, err = k8sh.ExecWithRetry(10, settings.Namespace, client.RunAllCephCommandsInToolboxPod, "bash", []string{"-c", fmt.Sprintf("mount -t ceph -o mds_namespace=%s,name=admin,secret=$(grep key /etc/ceph/keyring | awk '{print $3}') $(grep mon_host /etc/ceph/ceph.conf | awk '{print $3}'):/ %s", filesystemName, utils.TestMountPath)})
|
||||
require.Nil(s.T(), err)
|
||||
_, err = k8sh.Exec(namespace, "rook-ceph-tools", "mkdir", []string{"-p", fmt.Sprintf("%s/foo", utils.TestMountPath)})
|
||||
_, err = k8sh.Exec(settings.Namespace, client.RunAllCephCommandsInToolboxPod, "mkdir", []string{"-p", fmt.Sprintf("%s/foo", utils.TestMountPath)})
|
||||
require.Nil(s.T(), err)
|
||||
_, err = k8sh.Exec(namespace, "rook-ceph-tools", "umount", []string{utils.TestMountPath})
|
||||
_, err = k8sh.Exec(settings.Namespace, client.RunAllCephCommandsInToolboxPod, "umount", []string{utils.TestMountPath})
|
||||
require.Nil(s.T(), err)
|
||||
logger.Info("Created /foo directory on CephFS")
|
||||
|
||||
@@ -523,15 +522,15 @@ func createFilesystemMountCephCredentials(helper *clients.TestClient, k8sh *util
|
||||
),
|
||||
}
|
||||
logger.Infof("ceph credentials command args: %s", commandArgs[1])
|
||||
result, err := k8sh.Exec(namespace, "rook-ceph-tools", "bash", commandArgs)
|
||||
result, err := k8sh.Exec(settings.Namespace, client.RunAllCephCommandsInToolboxPod, "bash", commandArgs)
|
||||
logger.Infof("Ceph filesystem credentials output: %s", result)
|
||||
logger.Info("Created Ceph credentials")
|
||||
require.Nil(s.T(), err)
|
||||
// Save Ceph credentials to Kubernetes
|
||||
_, err = k8sh.Clientset.CoreV1().Secrets(namespace).Create(ctx, &v1.Secret{
|
||||
_, err = k8sh.Clientset.CoreV1().Secrets(settings.Namespace).Create(ctx, &v1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: fileMountSecret,
|
||||
Namespace: namespace,
|
||||
Namespace: settings.Namespace,
|
||||
},
|
||||
Data: map[string][]byte{
|
||||
"mykey": []byte(result),
|
||||
@@ -541,12 +540,11 @@ func createFilesystemMountCephCredentials(helper *clients.TestClient, k8sh *util
|
||||
logger.Info("Created Ceph credentials Secret in Kubernetes")
|
||||
}
|
||||
|
||||
func createFilesystemMountUserConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName, storageClassName string) {
|
||||
func createFilesystemMountUserConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName, storageClassName string) {
|
||||
// TODO: Mount with user credentials for the CSI driver
|
||||
useCSI := false
|
||||
mtfsErr := createPodWithFilesystem(k8sh, s, fileMountUserPodName, namespace, filesystemName, storageClassName, true, useCSI)
|
||||
mtfsErr := createPodWithFilesystem(k8sh, s, settings, fileMountUserPodName, filesystemName, storageClassName, true)
|
||||
require.Nil(s.T(), mtfsErr)
|
||||
filePodRunning := k8sh.IsPodRunning(fileMountUserPodName, namespace)
|
||||
filePodRunning := k8sh.IsPodRunning(fileMountUserPodName, settings.Namespace)
|
||||
require.True(s.T(), filePodRunning, "make sure file-mountuser-test pod is in running state")
|
||||
logger.Infof("File system mounted successfully")
|
||||
}
|
||||
|
||||
@@ -24,9 +24,9 @@ import (
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
rgw "github.com/rook/rook/pkg/operator/ceph/object"
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
@@ -59,11 +59,11 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
|
||||
|
||||
logger.Infof("Step 0 : Create Object Store User")
|
||||
cosuErr := helper.ObjectUserClient.Create(namespace, userid, userdisplayname, storeName)
|
||||
require.Nil(s.T(), cosuErr)
|
||||
assert.Nil(s.T(), cosuErr)
|
||||
|
||||
logger.Infof("Step 1 : Create Object Store")
|
||||
cobsErr := helper.ObjectClient.Create(namespace, storeName, 3)
|
||||
require.Nil(s.T(), cobsErr)
|
||||
assert.Nil(s.T(), cobsErr)
|
||||
|
||||
// check that ObjectStore is created
|
||||
logger.Infof("Check that RGW pods are Running")
|
||||
@@ -86,7 +86,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
|
||||
|
||||
assert.True(s.T(), helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid))
|
||||
userInfo, err := helper.ObjectUserClient.GetUser(namespace, storeName, userid)
|
||||
require.NoError(s.T(), err)
|
||||
assert.NoError(s.T(), err)
|
||||
assert.Equal(s.T(), userid, userInfo.UserID)
|
||||
assert.Equal(s.T(), userdisplayname, *userInfo.DisplayName)
|
||||
logger.Infof("Done creating object store user")
|
||||
@@ -107,11 +107,11 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
|
||||
assert.NotEmpty(s.T(), objectStore.Status.Info["endpoint"])
|
||||
break
|
||||
}
|
||||
require.NotEqual(s.T(), i, 4)
|
||||
assert.NotEqual(s.T(), i, 4)
|
||||
|
||||
logger.Infof("Step 2 : Test Deleting User")
|
||||
dosuErr := helper.ObjectUserClient.Delete(namespace, userid)
|
||||
require.Nil(s.T(), dosuErr)
|
||||
assert.Nil(s.T(), dosuErr)
|
||||
logger.Infof("Object store user deleted successfully")
|
||||
logger.Infof("Checking to see if the user secret has been deleted")
|
||||
for i = 0; i < 4 && helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid) == true; i++ {
|
||||
@@ -119,7 +119,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
assert.False(s.T(), helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid))
|
||||
require.NotEqual(s.T(), i, 4)
|
||||
assert.NotEqual(s.T(), i, 4)
|
||||
|
||||
logger.Infof("Check that MGRs are not in a crashloop")
|
||||
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mgr", namespace, 1, "Running"))
|
||||
@@ -129,14 +129,14 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
|
||||
logger.Infof("Step 3 : Create Object Bucket Claim with reclaim policy delete")
|
||||
bucketStorageClassName := "rook-smoke-delete-bucket"
|
||||
cobErr := helper.BucketClient.CreateBucketStorageClass(namespace, storeName, bucketStorageClassName, "Delete", region)
|
||||
require.Nil(s.T(), cobErr)
|
||||
assert.Nil(s.T(), cobErr)
|
||||
cobcErr := helper.BucketClient.CreateObc(obcName, bucketStorageClassName, bucketname, maxObject, true)
|
||||
require.Nil(s.T(), cobcErr)
|
||||
assert.Nil(s.T(), cobcErr)
|
||||
|
||||
created := utils.Retry(12, 2*time.Second, "OBC is created", func() bool {
|
||||
return helper.BucketClient.CheckOBC(obcName, "bound")
|
||||
})
|
||||
require.True(s.T(), created)
|
||||
assert.True(s.T(), created)
|
||||
|
||||
logger.Infof("Check if bucket was created")
|
||||
context := k8sh.MakeContext()
|
||||
@@ -153,7 +153,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
assert.NotEqual(s.T(), i, 4)
|
||||
require.Equal(s.T(), bucketname, bkt.Name)
|
||||
assert.Equal(s.T(), bucketname, bkt.Name)
|
||||
logger.Infof("OBC, Secret and ConfigMap created")
|
||||
|
||||
logger.Infof("Step 4 : Create s3 client")
|
||||
@@ -161,37 +161,37 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
|
||||
s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
|
||||
s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
|
||||
s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true)
|
||||
require.Nil(s.T(), err)
|
||||
assert.Nil(s.T(), err)
|
||||
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
|
||||
|
||||
logger.Infof("Step 5 : Put Object on bucket")
|
||||
_, poErr := s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey1, contentType)
|
||||
require.Nil(s.T(), poErr)
|
||||
assert.Nil(s.T(), poErr)
|
||||
|
||||
logger.Infof("Step 6 : Get Object from bucket")
|
||||
read, err := s3client.GetObjectInBucket(bucketname, ObjectKey1)
|
||||
require.Nil(s.T(), err)
|
||||
require.Equal(s.T(), ObjBody, read)
|
||||
assert.Nil(s.T(), err)
|
||||
assert.Equal(s.T(), ObjBody, read)
|
||||
logger.Infof("Object Created and Retrieved on bucket successfully")
|
||||
|
||||
logger.Infof("Step 7 : Testing Quota for the OBC")
|
||||
logger.Infof("Adding one more object to the bucket")
|
||||
_, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey2, contentType)
|
||||
require.Nil(s.T(), poErr)
|
||||
assert.Nil(s.T(), poErr)
|
||||
logger.Infof("Testing the max object limit")
|
||||
_, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey3, contentType)
|
||||
require.Error(s.T(), poErr)
|
||||
assert.Error(s.T(), poErr)
|
||||
|
||||
logger.Infof("Step 8 : Delete objects on bucket")
|
||||
_, delobjErr := s3client.DeleteObjectInBucket(bucketname, ObjectKey1)
|
||||
require.Nil(s.T(), delobjErr)
|
||||
assert.Nil(s.T(), delobjErr)
|
||||
_, delobjErr = s3client.DeleteObjectInBucket(bucketname, ObjectKey2)
|
||||
require.Nil(s.T(), delobjErr)
|
||||
assert.Nil(s.T(), delobjErr)
|
||||
logger.Infof("Objects deleted on bucket successfully")
|
||||
|
||||
logger.Infof("Step 9 : Delete Object Bucket Claim")
|
||||
dobcErr := helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketname, maxObject, true)
|
||||
require.Nil(s.T(), dobcErr)
|
||||
assert.Nil(s.T(), dobcErr)
|
||||
logger.Infof("Checking to see if the obc, secret and cm have all been deleted")
|
||||
for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "deleted"); i++ {
|
||||
logger.Infof("(%d) obc deleted check, sleeping for 5 seconds ...", i)
|
||||
@@ -220,27 +220,27 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
|
||||
}
|
||||
|
||||
// Test Object StoreCreation on Rook that was installed via helm
|
||||
func runObjectE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, name string, replicaSize int, deleteStore bool) {
|
||||
func runObjectE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, name string, replicaSize int, deleteStore bool) {
|
||||
logger.Infof("Object Storage End To End Integration Test - Create Object Store and check if rgw service is Running")
|
||||
logger.Infof("Running on Rook Cluster %s", namespace)
|
||||
logger.Infof("Running on Rook Cluster %s", settings.Namespace)
|
||||
|
||||
logger.Infof("Step 1 : Create Object Store")
|
||||
err := helper.ObjectClient.Create(namespace, name, int32(replicaSize))
|
||||
require.Nil(s.T(), err)
|
||||
err := helper.ObjectClient.Create(settings.Namespace, name, int32(replicaSize))
|
||||
assert.Nil(s.T(), err)
|
||||
|
||||
logger.Infof("Step 2 : check rook-ceph-rgw service status and count")
|
||||
require.True(s.T(), k8sh.IsPodInExpectedState("rook-ceph-rgw", namespace, "Running"),
|
||||
assert.True(s.T(), k8sh.IsPodInExpectedState("rook-ceph-rgw", settings.Namespace, "Running"),
|
||||
"Make sure rook-ceph-rgw is in running state")
|
||||
|
||||
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-rgw", namespace, replicaSize, "Running"),
|
||||
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-rgw", settings.Namespace, replicaSize, "Running"),
|
||||
"Make sure all rook-ceph-rgw pods are in Running state")
|
||||
|
||||
require.True(s.T(), k8sh.IsServiceUp("rook-ceph-rgw-"+name, namespace))
|
||||
assert.True(s.T(), k8sh.IsServiceUp("rook-ceph-rgw-"+name, settings.Namespace))
|
||||
|
||||
if deleteStore {
|
||||
logger.Infof("Delete Object Store")
|
||||
err = helper.ObjectClient.Delete(namespace, name)
|
||||
require.Nil(s.T(), err)
|
||||
err = helper.ObjectClient.Delete(settings.Namespace, name)
|
||||
assert.Nil(s.T(), err)
|
||||
logger.Infof("Done deleting object store")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,7 +61,7 @@ func TestCephFlexSuite(t *testing.T) {
|
||||
|
||||
s := new(CephFlexDriverSuite)
|
||||
defer func(s *CephFlexDriverSuite) {
|
||||
HandlePanics(recover(), s.op, s.T)
|
||||
HandlePanics(recover(), s.TearDownSuite, s.T)
|
||||
}(s)
|
||||
suite.Run(t, s)
|
||||
}
|
||||
@@ -72,47 +72,45 @@ type CephFlexDriverSuite struct {
|
||||
clusterInfo *client.ClusterInfo
|
||||
bc *clients.BlockOperation
|
||||
kh *utils.K8sHelper
|
||||
namespace string
|
||||
installer *installer.CephInstaller
|
||||
settings *installer.TestCephSettings
|
||||
pvcNameRWO string
|
||||
pvcNameRWX string
|
||||
op *TestCluster
|
||||
}
|
||||
|
||||
func (s *CephFlexDriverSuite) SetupSuite() {
|
||||
|
||||
s.namespace = "flex-ns"
|
||||
namespace := "flex-ns"
|
||||
s.pvcNameRWO = "block-persistent-rwo"
|
||||
s.pvcNameRWX = "block-persistent-rwx"
|
||||
|
||||
flexTestCluster := TestCluster{
|
||||
namespace: s.namespace,
|
||||
storeType: "bluestore",
|
||||
storageClassName: "",
|
||||
useHelm: false,
|
||||
usePVC: false,
|
||||
mons: 1,
|
||||
rbdMirrorWorkers: 1,
|
||||
rookCephCleanup: true,
|
||||
skipOSDCreation: false,
|
||||
minimalMatrixK8sVersion: flexDriverMinimalTestVersion,
|
||||
rookVersion: installer.VersionMaster,
|
||||
cephVersion: installer.OctopusVersion,
|
||||
s.settings = &installer.TestCephSettings{
|
||||
ClusterName: "flex-test",
|
||||
Namespace: namespace,
|
||||
OperatorNamespace: installer.SystemNamespace(namespace),
|
||||
StorageClassName: "",
|
||||
UseHelm: false,
|
||||
UsePVC: false,
|
||||
Mons: 1,
|
||||
SkipOSDCreation: false,
|
||||
UseCSI: false,
|
||||
DirectMountToolbox: true,
|
||||
RookVersion: installer.VersionMaster,
|
||||
CephVersion: installer.OctopusVersion,
|
||||
}
|
||||
|
||||
s.clusterInfo = client.AdminClusterInfo(s.namespace)
|
||||
s.op, s.kh = StartTestCluster(s.T, &flexTestCluster)
|
||||
s.testClient = clients.CreateTestClient(s.kh, s.op.installer.Manifests)
|
||||
s.settings.ApplyEnvVars()
|
||||
s.clusterInfo = client.AdminClusterInfo(namespace)
|
||||
s.installer, s.kh = StartTestCluster(s.T, s.settings, flexDriverMinimalTestVersion)
|
||||
s.testClient = clients.CreateTestClient(s.kh, s.installer.Manifests)
|
||||
s.bc = s.testClient.BlockClient
|
||||
}
|
||||
|
||||
func (s *CephFlexDriverSuite) AfterTest(suiteName, testName string) {
|
||||
s.op.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(s.namespace))
|
||||
s.installer.CollectOperatorLog(suiteName, testName)
|
||||
}
|
||||
|
||||
func (s *CephFlexDriverSuite) TestFileSystem() {
|
||||
useCSI := false
|
||||
preserveFilesystemOnDelete := false
|
||||
runFileE2ETest(s.testClient, s.kh, s.Suite, s.namespace, "smoke-test-fs", useCSI, preserveFilesystemOnDelete)
|
||||
runFileE2ETest(s.testClient, s.kh, s.Suite, s.settings, "flex-fs", preserveFilesystemOnDelete)
|
||||
}
|
||||
|
||||
func (s *CephFlexDriverSuite) TestBlockStorageMountUnMountForStatefulSets() {
|
||||
@@ -127,9 +125,9 @@ func (s *CephFlexDriverSuite) TestBlockStorageMountUnMountForStatefulSets() {
|
||||
logger.Infof("Test case when block persistent volumes are scaled up and down along with StatefulSet")
|
||||
logger.Info("Step 1: Create pool and storageClass")
|
||||
|
||||
err := s.testClient.PoolClient.Create(poolName, s.namespace, 1)
|
||||
err := s.testClient.PoolClient.Create(poolName, s.settings.Namespace, 1)
|
||||
assert.Nil(s.T(), err)
|
||||
err = s.testClient.BlockClient.CreateStorageClass(false, poolName, storageClassName, reclaimPolicy, s.namespace)
|
||||
err = s.testClient.BlockClient.CreateStorageClass(false, poolName, storageClassName, reclaimPolicy, s.settings.Namespace)
|
||||
assert.Nil(s.T(), err)
|
||||
logger.Info("Step 2 : Deploy statefulSet with 1X replication")
|
||||
service, statefulset := getBlockStatefulSetAndServiceDefinition(defaultNamespace, statefulSetName, statefulPodsName, storageClassName)
|
||||
@@ -190,11 +188,9 @@ func (s *CephFlexDriverSuite) setupPVCs() {
|
||||
logger.Infof("creating the test PVCs")
|
||||
poolNameRWO := "block-pool-rwo"
|
||||
storageClassNameRWO := "rook-ceph-block-rwo"
|
||||
systemNamespace := installer.SystemNamespace(s.namespace)
|
||||
|
||||
// Create PVCs
|
||||
useCSI := false
|
||||
err := s.testClient.BlockClient.CreateStorageClassAndPVC(useCSI, defaultNamespace, s.namespace, systemNamespace, poolNameRWO, storageClassNameRWO, "Delete", s.pvcNameRWO, "ReadWriteOnce")
|
||||
err := s.testClient.BlockClient.CreateStorageClassAndPVC(defaultNamespace, poolNameRWO, storageClassNameRWO, "Delete", s.pvcNameRWO, "ReadWriteOnce")
|
||||
require.Nil(s.T(), err)
|
||||
require.True(s.T(), s.kh.WaitUntilPVCIsBound(defaultNamespace, s.pvcNameRWO), "Make sure PVC is Bound")
|
||||
|
||||
@@ -207,13 +203,13 @@ func (s *CephFlexDriverSuite) setupPVCs() {
|
||||
require.Nil(s.T(), err)
|
||||
crdName, err := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWO)
|
||||
require.Nil(s.T(), err)
|
||||
s.kh.IsVolumeResourcePresent(systemNamespace, crdName)
|
||||
s.kh.IsVolumeResourcePresent(s.settings.OperatorNamespace, crdName)
|
||||
|
||||
err = s.bc.CreateClientPod(getFlexBlockPodDefinition("setup-block-rwx", s.pvcNameRWX, false))
|
||||
require.Nil(s.T(), err)
|
||||
crdName, err = s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWX)
|
||||
require.Nil(s.T(), err)
|
||||
s.kh.IsVolumeResourcePresent(systemNamespace, crdName)
|
||||
s.kh.IsVolumeResourcePresent(s.settings.OperatorNamespace, crdName)
|
||||
require.True(s.T(), s.kh.IsPodRunning("setup-block-rwo", defaultNamespace), "make sure setup-block-rwo pod is in running state")
|
||||
require.True(s.T(), s.kh.IsPodRunning("setup-block-rwx", defaultNamespace), "make sure setup-block-rwx pod is in running state")
|
||||
|
||||
@@ -239,9 +235,9 @@ func (s *CephFlexDriverSuite) TearDownSuite() {
|
||||
"setup-block-rwo", "setup-block-rwx", "rwo-block-rw-one", "rwo-block-rw-two", "rwo-block-ro-one",
|
||||
"rwo-block-ro-two", "rwx-block-rw-one", "rwx-block-rw-two", "rwx-block-ro-one", "rwx-block-ro-two")
|
||||
assert.NoError(s.T(), err)
|
||||
err = s.testClient.BlockClient.DeletePVC(s.namespace, s.pvcNameRWO)
|
||||
err = s.testClient.BlockClient.DeletePVC(s.settings.Namespace, s.pvcNameRWO)
|
||||
assertNoErrorUnlessNotFound(s.Suite, err)
|
||||
err = s.testClient.BlockClient.DeletePVC(s.namespace, s.pvcNameRWX)
|
||||
err = s.testClient.BlockClient.DeletePVC(s.settings.Namespace, s.pvcNameRWX)
|
||||
assertNoErrorUnlessNotFound(s.Suite, err)
|
||||
err = s.testClient.BlockClient.DeleteStorageClass("rook-ceph-block-rwo")
|
||||
assert.NoError(s.T(), err)
|
||||
@@ -251,7 +247,7 @@ func (s *CephFlexDriverSuite) TearDownSuite() {
|
||||
assert.NoError(s.T(), err)
|
||||
err = s.testClient.PoolClient.DeletePool(s.testClient.BlockClient, s.clusterInfo, "block-pool-rwx")
|
||||
assert.NoError(s.T(), err)
|
||||
s.op.Teardown()
|
||||
s.installer.UninstallRook()
|
||||
}
|
||||
|
||||
func (s *CephFlexDriverSuite) TestBlockStorageMountUnMountForDifferentAccessModes() {
|
||||
@@ -267,12 +263,12 @@ func (s *CephFlexDriverSuite) TestBlockStorageMountUnMountForDifferentAccessMode
|
||||
require.Nil(s.T(), err)
|
||||
crdName, err := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWO)
|
||||
assert.Nil(s.T(), err)
|
||||
assert.True(s.T(), s.kh.IsVolumeResourcePresent(installer.SystemNamespace(s.namespace), crdName), fmt.Sprintf("make sure Volume %s is created", crdName))
|
||||
assert.True(s.T(), s.kh.IsVolumeResourcePresent(s.settings.OperatorNamespace, crdName), fmt.Sprintf("make sure Volume %s is created", crdName))
|
||||
assert.True(s.T(), s.kh.IsPodRunning("rwo-block-rw-one", defaultNamespace), "make sure block-rw-one pod is in running state")
|
||||
|
||||
crdName, err = s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWX)
|
||||
assert.Nil(s.T(), err)
|
||||
assert.True(s.T(), s.kh.IsVolumeResourcePresent(installer.SystemNamespace(s.namespace), crdName), fmt.Sprintf("make sure Volume %s is created", crdName))
|
||||
assert.True(s.T(), s.kh.IsVolumeResourcePresent(s.settings.OperatorNamespace, crdName), fmt.Sprintf("make sure Volume %s is created", crdName))
|
||||
assert.True(s.T(), s.kh.IsPodRunning("rwx-block-rw-one", defaultNamespace), "make sure rwx-block-rw-one pod is in running state")
|
||||
|
||||
logger.Infof("Step 2: Check if previously persisted data is readable from ReadWriteOnce and ReadWriteMany PVC")
|
||||
|
||||
@@ -47,7 +47,7 @@ func TestCephHelmSuite(t *testing.T) {
|
||||
|
||||
s := new(HelmSuite)
|
||||
defer func(s *HelmSuite) {
|
||||
HandlePanics(recover(), s.op, s.T)
|
||||
HandlePanics(recover(), s.TearDownSuite, s.T)
|
||||
}(s)
|
||||
suite.Run(t, s)
|
||||
}
|
||||
@@ -55,56 +55,56 @@ func TestCephHelmSuite(t *testing.T) {
|
||||
type HelmSuite struct {
|
||||
suite.Suite
|
||||
helper *clients.TestClient
|
||||
kh *utils.K8sHelper
|
||||
op *TestCluster
|
||||
namespace string
|
||||
installer *installer.CephInstaller
|
||||
settings *installer.TestCephSettings
|
||||
k8shelper *utils.K8sHelper
|
||||
}
|
||||
|
||||
func (hs *HelmSuite) SetupSuite() {
|
||||
hs.namespace = "helm-ns"
|
||||
helmTestCluster := TestCluster{
|
||||
namespace: hs.namespace,
|
||||
storeType: "bluestore",
|
||||
storageClassName: "",
|
||||
useHelm: true,
|
||||
usePVC: false,
|
||||
mons: 1,
|
||||
rbdMirrorWorkers: 1,
|
||||
rookCephCleanup: true,
|
||||
skipOSDCreation: false,
|
||||
minimalMatrixK8sVersion: helmMinimalTestVersion,
|
||||
rookVersion: installer.VersionMaster,
|
||||
cephVersion: installer.NautilusVersion,
|
||||
func (h *HelmSuite) SetupSuite() {
|
||||
namespace := "helm-ns"
|
||||
h.settings = &installer.TestCephSettings{
|
||||
Namespace: namespace,
|
||||
OperatorNamespace: namespace,
|
||||
StorageClassName: "",
|
||||
UseHelm: true,
|
||||
UsePVC: false,
|
||||
Mons: 1,
|
||||
UseCSI: true,
|
||||
SkipOSDCreation: false,
|
||||
EnableAdmissionController: true,
|
||||
EnableDiscovery: true,
|
||||
RookVersion: installer.VersionMaster,
|
||||
CephVersion: installer.OctopusVersion,
|
||||
}
|
||||
|
||||
hs.op, hs.kh = StartTestCluster(hs.T, &helmTestCluster)
|
||||
hs.helper = clients.CreateTestClient(hs.kh, hs.op.installer.Manifests)
|
||||
h.settings.ApplyEnvVars()
|
||||
h.installer, h.k8shelper = StartTestCluster(h.T, h.settings, helmMinimalTestVersion)
|
||||
h.helper = clients.CreateTestClient(h.k8shelper, h.installer.Manifests)
|
||||
}
|
||||
|
||||
func (hs *HelmSuite) TearDownSuite() {
|
||||
hs.op.Teardown()
|
||||
func (h *HelmSuite) TearDownSuite() {
|
||||
h.installer.UninstallRook()
|
||||
}
|
||||
|
||||
func (hs *HelmSuite) AfterTest(suiteName, testName string) {
|
||||
hs.op.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(hs.namespace))
|
||||
func (h *HelmSuite) AfterTest(suiteName, testName string) {
|
||||
h.installer.CollectOperatorLog(suiteName, testName)
|
||||
}
|
||||
|
||||
// Test to make sure all rook components are installed and Running
|
||||
func (hs *HelmSuite) TestARookInstallViaHelm() {
|
||||
checkIfRookClusterIsInstalled(hs.Suite, hs.kh, hs.namespace, hs.namespace, 1)
|
||||
func (h *HelmSuite) TestARookInstallViaHelm() {
|
||||
checkIfRookClusterIsInstalled(h.Suite, h.k8shelper, h.settings.Namespace, h.settings.Namespace, 1)
|
||||
}
|
||||
|
||||
// Test BlockCreation on Rook that was installed via Helm
|
||||
func (hs *HelmSuite) TestBlockStoreOnRookInstalledViaHelm() {
|
||||
runBlockCSITestLite(hs.helper, hs.kh, hs.Suite, hs.namespace, hs.namespace, hs.op.installer.CephVersion)
|
||||
func (h *HelmSuite) TestBlockStoreOnRookInstalledViaHelm() {
|
||||
runBlockCSITestLite(h.helper, h.k8shelper, h.Suite, h.settings)
|
||||
}
|
||||
|
||||
// Test File System Creation on Rook that was installed via helm
|
||||
func (hs *HelmSuite) TestFileStoreOnRookInstalledViaHelm() {
|
||||
runFileE2ETestLite(hs.helper, hs.kh, hs.Suite, hs.namespace, "testfs")
|
||||
func (h *HelmSuite) TestFileStoreOnRookInstalledViaHelm() {
|
||||
runFileE2ETestLite(h.helper, h.k8shelper, h.Suite, h.settings, "testfs")
|
||||
}
|
||||
|
||||
// Test Object StoreCreation on Rook that was installed via helm
|
||||
func (hs *HelmSuite) TestObjectStoreOnRookInstalledViaHelm() {
|
||||
runObjectE2ETestLite(hs.helper, hs.kh, hs.Suite, hs.namespace, "default", 3, true)
|
||||
func (h *HelmSuite) TestObjectStoreOnRookInstalledViaHelm() {
|
||||
runObjectE2ETestLite(h.helper, h.k8shelper, h.Suite, h.settings, "default", 3, true)
|
||||
}
|
||||
@@ -55,15 +55,16 @@ func TestCephMgrSuite(t *testing.T) {
|
||||
|
||||
s := new(CephMgrSuite)
|
||||
defer func(s *CephMgrSuite) {
|
||||
HandlePanics(recover(), s.cluster, s.T)
|
||||
HandlePanics(recover(), s.TearDownSuite, s.T)
|
||||
}(s)
|
||||
suite.Run(t, s)
|
||||
}
|
||||
|
||||
type CephMgrSuite struct {
|
||||
suite.Suite
|
||||
cluster *TestCluster
|
||||
settings *installer.TestCephSettings
|
||||
k8sh *utils.K8sHelper
|
||||
installer *installer.CephInstaller
|
||||
namespace string
|
||||
}
|
||||
|
||||
@@ -87,46 +88,44 @@ type service struct {
|
||||
Status serviceStatus
|
||||
}
|
||||
|
||||
func (suite *CephMgrSuite) SetupSuite() {
|
||||
suite.namespace = "mgr-ns"
|
||||
func (s *CephMgrSuite) SetupSuite() {
|
||||
s.namespace = "mgr-ns"
|
||||
|
||||
mgrTestCluster := TestCluster{
|
||||
clusterName: suite.namespace,
|
||||
namespace: suite.namespace,
|
||||
storeType: "bluestore",
|
||||
storageClassName: "",
|
||||
useHelm: false,
|
||||
usePVC: false,
|
||||
mons: 1,
|
||||
rbdMirrorWorkers: 0,
|
||||
rookCephCleanup: true,
|
||||
skipOSDCreation: true,
|
||||
minimalMatrixK8sVersion: cephMasterSuiteMinimalTestVersion,
|
||||
rookVersion: installer.VersionMaster,
|
||||
cephVersion: installer.MasterVersion,
|
||||
s.settings = &installer.TestCephSettings{
|
||||
ClusterName: s.namespace,
|
||||
OperatorNamespace: installer.SystemNamespace(s.namespace),
|
||||
Namespace: s.namespace,
|
||||
StorageClassName: "",
|
||||
UseHelm: false,
|
||||
UsePVC: false,
|
||||
Mons: 1,
|
||||
UseCSI: true,
|
||||
SkipOSDCreation: true,
|
||||
RookVersion: installer.VersionMaster,
|
||||
CephVersion: installer.MasterVersion,
|
||||
}
|
||||
|
||||
suite.cluster, suite.k8sh = StartTestCluster(suite.T, &mgrTestCluster)
|
||||
suite.waitForOrchestrationModule()
|
||||
s.settings.ApplyEnvVars()
|
||||
s.installer, s.k8sh = StartTestCluster(s.T, s.settings, cephMasterSuiteMinimalTestVersion)
|
||||
s.waitForOrchestrationModule()
|
||||
}
|
||||
|
||||
func (suite *CephMgrSuite) AfterTest(suiteName, testName string) {
|
||||
suite.cluster.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(suite.namespace))
|
||||
func (s *CephMgrSuite) AfterTest(suiteName, testName string) {
|
||||
s.installer.CollectOperatorLog(suiteName, testName)
|
||||
}
|
||||
|
||||
func (suite *CephMgrSuite) TearDownSuite() {
|
||||
suite.cluster.Teardown()
|
||||
func (s *CephMgrSuite) TearDownSuite() {
|
||||
s.installer.UninstallRook()
|
||||
}
|
||||
|
||||
func (suite *CephMgrSuite) execute(command []string) (error, string) {
|
||||
func (s *CephMgrSuite) execute(command []string) (error, string) {
|
||||
orchCommand := append([]string{"orch"}, command...)
|
||||
return suite.cluster.installer.Execute("ceph", orchCommand, suite.namespace)
|
||||
return s.installer.Execute("ceph", orchCommand, s.namespace)
|
||||
}
|
||||
|
||||
func (suite *CephMgrSuite) waitForOrchestrationModule() {
|
||||
func (s *CephMgrSuite) waitForOrchestrationModule() {
|
||||
var err error
|
||||
for timeout := 0; timeout < 30; timeout++ {
|
||||
err, output := suite.execute([]string{"status"})
|
||||
err, output := s.execute([]string{"status"})
|
||||
logger.Infof("%s", output)
|
||||
if err == nil {
|
||||
logger.Info("Rook Toolbox ready to execute commands")
|
||||
@@ -135,30 +134,30 @@ func (suite *CephMgrSuite) waitForOrchestrationModule() {
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
logger.Error("Giving up waiting for Rook Toolbox to be ready")
|
||||
assert.Nil(suite.T(), err)
|
||||
assert.Nil(s.T(), err)
|
||||
}
|
||||
func (suite *CephMgrSuite) TestDeviceLs() {
|
||||
func (s *CephMgrSuite) TestDeviceLs() {
|
||||
logger.Info("Testing .... <ceph orch device ls>")
|
||||
err, device_list := suite.execute([]string{"device", "ls"})
|
||||
assert.Nil(suite.T(), err)
|
||||
err, device_list := s.execute([]string{"device", "ls"})
|
||||
assert.Nil(s.T(), err)
|
||||
logger.Infof("output = %s", device_list)
|
||||
}
|
||||
|
||||
func (suite *CephMgrSuite) TestStatus() {
|
||||
func (s *CephMgrSuite) TestStatus() {
|
||||
logger.Info("Testing .... <ceph orch status>")
|
||||
err, status := suite.execute([]string{"status"})
|
||||
assert.Nil(suite.T(), err)
|
||||
err, status := s.execute([]string{"status"})
|
||||
assert.Nil(s.T(), err)
|
||||
logger.Infof("output = %s", status)
|
||||
|
||||
assert.Equal(suite.T(), status, "Backend: rook\nAvailable: True")
|
||||
assert.Equal(s.T(), status, "Backend: rook\nAvailable: True")
|
||||
}
|
||||
|
||||
func (suite *CephMgrSuite) TestHostLs() {
|
||||
func (s *CephMgrSuite) TestHostLs() {
|
||||
logger.Info("Testing .... <ceph orch host ls>")
|
||||
|
||||
// Get the orchestrator hosts
|
||||
err, output := suite.execute([]string{"host", "ls", "json"})
|
||||
assert.Nil(suite.T(), err)
|
||||
err, output := s.execute([]string{"host", "ls", "json"})
|
||||
assert.Nil(s.T(), err)
|
||||
logger.Infof("output = %s", output)
|
||||
|
||||
hosts := []byte(output)
|
||||
@@ -166,7 +165,7 @@ func (suite *CephMgrSuite) TestHostLs() {
|
||||
|
||||
err = json.Unmarshal(hosts, &hostsList)
|
||||
if err != nil {
|
||||
assert.Nil(suite.T(), err)
|
||||
assert.Nil(s.T(), err)
|
||||
}
|
||||
|
||||
var hostOutput []string
|
||||
@@ -176,8 +175,8 @@ func (suite *CephMgrSuite) TestHostLs() {
|
||||
sort.Strings(hostOutput)
|
||||
|
||||
// get the k8s nodes
|
||||
nodes, err := k8sutil.GetNodeHostNames(suite.k8sh.Clientset)
|
||||
assert.Nil(suite.T(), err)
|
||||
nodes, err := k8sutil.GetNodeHostNames(s.k8sh.Clientset)
|
||||
assert.Nil(s.T(), err)
|
||||
|
||||
k8sNodes := make([]string, 0, len(nodes))
|
||||
for k := range nodes {
|
||||
@@ -186,21 +185,21 @@ func (suite *CephMgrSuite) TestHostLs() {
|
||||
sort.Strings(k8sNodes)
|
||||
|
||||
// nodes and hosts must be the same
|
||||
assert.Equal(suite.T(), hostOutput, k8sNodes)
|
||||
assert.Equal(s.T(), hostOutput, k8sNodes)
|
||||
}
|
||||
|
||||
func (suite *CephMgrSuite) TestCreateOSD() {
|
||||
func (s *CephMgrSuite) TestCreateOSD() {
|
||||
logger.Info("Testing .... <ceph orch create OSD>")
|
||||
|
||||
// Get the first available device
|
||||
err, deviceList := suite.execute([]string{"device", "ls", "--format", "json"})
|
||||
assert.Nil(suite.T(), err)
|
||||
err, deviceList := s.execute([]string{"device", "ls", "--format", "json"})
|
||||
assert.Nil(s.T(), err)
|
||||
logger.Infof("output = %s", deviceList)
|
||||
|
||||
inventory := make([]map[string]interface{}, 0)
|
||||
|
||||
err = json.Unmarshal([]byte(deviceList), &inventory)
|
||||
assert.Nil(suite.T(), err)
|
||||
assert.Nil(s.T(), err)
|
||||
|
||||
selectedNode := ""
|
||||
selectedDevice := ""
|
||||
@@ -216,40 +215,40 @@ func (suite *CephMgrSuite) TestCreateOSD() {
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.NotEqual(suite.T(), "", selectedDevice, "No devices available to create test OSD")
|
||||
assert.NotEqual(suite.T(), "", selectedNode, "No nodes available to create test OSD")
|
||||
assert.NotEqual(s.T(), "", selectedDevice, "No devices available to create test OSD")
|
||||
assert.NotEqual(s.T(), "", selectedNode, "No nodes available to create test OSD")
|
||||
|
||||
if selectedDevice == "" || selectedNode == "" {
|
||||
return
|
||||
}
|
||||
// Create the OSD
|
||||
err, output := suite.execute([]string{"daemon", "add", "osd", fmt.Sprintf("%s:%s", selectedNode, selectedDevice)})
|
||||
err, output := s.execute([]string{"daemon", "add", "osd", fmt.Sprintf("%s:%s", selectedNode, selectedDevice)})
|
||||
|
||||
assert.Nil(suite.T(), err)
|
||||
assert.Nil(s.T(), err)
|
||||
logger.Infof("output = %s", output)
|
||||
|
||||
err = suite.k8sh.WaitForPodCount("app=rook-ceph-osd", suite.namespace, 1)
|
||||
assert.Nil(suite.T(), err)
|
||||
err = s.k8sh.WaitForPodCount("app=rook-ceph-osd", s.namespace, 1)
|
||||
assert.Nil(s.T(), err)
|
||||
}
|
||||
|
||||
func (suite *CephMgrSuite) TestServiceLs() {
|
||||
func (s *CephMgrSuite) TestServiceLs() {
|
||||
logger.Info("Testing .... <ceph orch ls>")
|
||||
err, output := suite.execute([]string{"ls", "--format", "json"})
|
||||
assert.Nil(suite.T(), err)
|
||||
err, output := s.execute([]string{"ls", "--format", "json"})
|
||||
assert.Nil(s.T(), err)
|
||||
logger.Infof("output = %s", output)
|
||||
|
||||
services := []byte(output)
|
||||
var servicesList []service
|
||||
|
||||
err = json.Unmarshal(services, &servicesList)
|
||||
assert.Nil(suite.T(), err)
|
||||
assert.Nil(s.T(), err)
|
||||
|
||||
for _, svc := range servicesList {
|
||||
labelFilter := fmt.Sprintf("app=rook-ceph-%s", svc.ServiceName)
|
||||
k8sPods, err := k8sutil.PodsRunningWithLabel(suite.k8sh.Clientset, suite.namespace, labelFilter)
|
||||
k8sPods, err := k8sutil.PodsRunningWithLabel(s.k8sh.Clientset, s.namespace, labelFilter)
|
||||
logger.Infof("Service: %+v", svc)
|
||||
logger.Infof("k8s pods for svc %q: %d", svc.ServiceName, k8sPods)
|
||||
assert.Nil(suite.T(), err)
|
||||
assert.Equal(suite.T(), svc.Status.Running, k8sPods, fmt.Sprintf("Wrong number of pods for kind of service <%s>", svc.ServiceName))
|
||||
assert.Nil(s.T(), err)
|
||||
assert.Equal(s.T(), svc.Status.Running, k8sPods, fmt.Sprintf("Wrong number of pods for kind of service <%s>", svc.ServiceName))
|
||||
}
|
||||
}
|
||||
@@ -17,16 +17,14 @@ limitations under the License.
|
||||
package integration
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
@@ -58,180 +56,130 @@ func TestCephMultiClusterDeploySuite(t *testing.T) {
|
||||
|
||||
s := new(MultiClusterDeploySuite)
|
||||
defer func(s *MultiClusterDeploySuite) {
|
||||
HandlePanics(recover(), s.op, s.T)
|
||||
HandlePanics(recover(), s.TearDownSuite, s.T)
|
||||
}(s)
|
||||
suite.Run(t, s)
|
||||
}
|
||||
|
||||
type MultiClusterDeploySuite struct {
|
||||
suite.Suite
|
||||
testClient *clients.TestClient
|
||||
k8sh *utils.K8sHelper
|
||||
namespace1 string
|
||||
namespace2 string
|
||||
op *MCTestOperations
|
||||
poolName string
|
||||
testClient *clients.TestClient
|
||||
k8sh *utils.K8sHelper
|
||||
settings *installer.TestCephSettings
|
||||
externalManifests installer.CephManifests
|
||||
installer *installer.CephInstaller
|
||||
coreToolbox string
|
||||
externalToolbox string
|
||||
poolName string
|
||||
}
|
||||
|
||||
// Deploy Multiple Rook clusters
|
||||
func (mrc *MultiClusterDeploySuite) SetupSuite() {
|
||||
mrc.poolName = "multi-cluster-pool1"
|
||||
mrc.namespace1 = "mrc-n1"
|
||||
mrc.namespace2 = "mrc-n2"
|
||||
|
||||
mrc.op, mrc.k8sh = NewMCTestOperations(mrc.T, mrc.namespace1, mrc.namespace2)
|
||||
mrc.testClient = clients.CreateTestClient(mrc.k8sh, mrc.op.installer.Manifests)
|
||||
mrc.createPools()
|
||||
}
|
||||
|
||||
func (mrc *MultiClusterDeploySuite) AfterTest(suiteName, testName string) {
|
||||
mrc.op.installer.CollectOperatorLog(suiteName, testName, mrc.op.systemNamespace)
|
||||
}
|
||||
|
||||
func (mrc *MultiClusterDeploySuite) createPools() {
|
||||
// create a test pool in each cluster so that we get some PGs
|
||||
logger.Infof("Creating pool %s", mrc.poolName)
|
||||
err := mrc.testClient.PoolClient.Create(mrc.poolName, mrc.namespace1, 1)
|
||||
require.Nil(mrc.T(), err)
|
||||
}
|
||||
|
||||
func (mrc *MultiClusterDeploySuite) deletePools() {
|
||||
// create a test pool in each cluster so that we get some PGs
|
||||
logger.Infof("Deleting pool %s", mrc.poolName)
|
||||
clusterInfo := client.AdminClusterInfo(mrc.namespace1)
|
||||
if err := mrc.testClient.PoolClient.DeletePool(mrc.testClient.BlockClient, clusterInfo, mrc.poolName); err != nil {
|
||||
logger.Errorf("failed to delete pool %q. %v", mrc.poolName, err)
|
||||
} else {
|
||||
logger.Infof("deleted pool %q", mrc.poolName)
|
||||
func (s *MultiClusterDeploySuite) SetupSuite() {
|
||||
s.poolName = "multi-cluster-pool1"
|
||||
coreNamespace := "multi-core"
|
||||
s.settings = &installer.TestCephSettings{
|
||||
ClusterName: "multi-cluster",
|
||||
Namespace: coreNamespace,
|
||||
OperatorNamespace: installer.SystemNamespace(coreNamespace),
|
||||
StorageClassName: "manual",
|
||||
UsePVC: installer.UsePVC(),
|
||||
Mons: 1,
|
||||
UseCSI: true,
|
||||
MultipleMgrs: true,
|
||||
EnableAdmissionController: true,
|
||||
RookVersion: installer.VersionMaster,
|
||||
CephVersion: installer.NautilusVersion,
|
||||
}
|
||||
}
|
||||
|
||||
func (mrc *MultiClusterDeploySuite) TearDownSuite() {
|
||||
mrc.deletePools()
|
||||
mrc.op.Teardown()
|
||||
}
|
||||
|
||||
// Test to make sure all rook components are installed and Running
|
||||
func (mrc *MultiClusterDeploySuite) TestInstallingMultipleRookClusters() {
|
||||
// Check if Rook cluster 1 is deployed successfully
|
||||
checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace1, 1)
|
||||
checkIfRookClusterIsHealthy(mrc.Suite, mrc.testClient, mrc.namespace1)
|
||||
|
||||
// Check if Rook external cluster is deployed successfully
|
||||
// Checking health status is enough to validate the connection
|
||||
checkIfRookClusterIsHealthy(mrc.Suite, mrc.testClient, mrc.namespace2)
|
||||
}
|
||||
|
||||
// MCTestOperations struct for handling panic and test suite tear down
|
||||
type MCTestOperations struct {
|
||||
installer *installer.CephInstaller
|
||||
kh *utils.K8sHelper
|
||||
T func() *testing.T
|
||||
namespace1 string
|
||||
namespace2 string
|
||||
systemNamespace string
|
||||
storageClassName string
|
||||
testOverPVC bool
|
||||
}
|
||||
|
||||
// NewMCTestOperations creates new instance of TestCluster struct
|
||||
func NewMCTestOperations(t func() *testing.T, namespace1 string, namespace2 string) (*MCTestOperations, *utils.K8sHelper) {
|
||||
|
||||
kh, err := utils.CreateK8sHelper(t)
|
||||
require.NoError(t(), err)
|
||||
checkIfShouldRunForMinimalTestMatrix(t, kh, multiClusterMinimalTestVersion)
|
||||
|
||||
cleanupHost := false
|
||||
i := installer.NewCephInstaller(t, kh.Clientset, false, "", installer.VersionMaster, installer.NautilusVersion, cleanupHost)
|
||||
|
||||
op := &MCTestOperations{i, kh, t, namespace1, namespace2, installer.SystemNamespace(namespace1), "", false}
|
||||
if kh.VersionAtLeast("v1.13.0") {
|
||||
op.testOverPVC = true
|
||||
op.storageClassName = "manual"
|
||||
s.settings.ApplyEnvVars()
|
||||
externalSettings := &installer.TestCephSettings{
|
||||
IsExternal: true,
|
||||
ClusterName: "test-external",
|
||||
Namespace: "multi-external",
|
||||
OperatorNamespace: s.settings.OperatorNamespace,
|
||||
RookVersion: s.settings.RookVersion,
|
||||
UseCSI: true,
|
||||
}
|
||||
op.Setup()
|
||||
return op, kh
|
||||
}
|
||||
externalSettings.ApplyEnvVars()
|
||||
s.externalManifests = installer.NewCephManifests(externalSettings)
|
||||
|
||||
// Setup is wrapper for setting up multiple rook clusters.
|
||||
func (o MCTestOperations) Setup() {
|
||||
var err error
|
||||
if o.testOverPVC {
|
||||
root, err := utils.FindRookRoot()
|
||||
require.NoError(o.T(), err, "failed to get rook root")
|
||||
cmdArgs := utils.CommandArgs{Command: filepath.Join(root, localPathPVCmd),
|
||||
CmdArgs: []string{installer.TestScratchDevice()}}
|
||||
cmdOut := utils.ExecuteCommand(cmdArgs)
|
||||
require.NoError(o.T(), cmdOut.Err)
|
||||
k8sh, err := utils.CreateK8sHelper(s.T)
|
||||
assert.NoError(s.T(), err)
|
||||
if !k8sh.VersionAtLeast("v1.16.0") {
|
||||
s.T().Skip("requires at least k8s 1.16, no need to run on older versions")
|
||||
}
|
||||
|
||||
err = o.installer.CreateCephOperator(installer.SystemNamespace(o.namespace1))
|
||||
require.NoError(o.T(), err)
|
||||
|
||||
require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-operator", o.systemNamespace, "Running"),
|
||||
"Make sure rook-operator is in running state")
|
||||
|
||||
require.True(o.T(), o.kh.IsPodInExpectedState("rook-discover", o.systemNamespace, "Running"),
|
||||
"Make sure rook-discover is in running state")
|
||||
|
||||
time.Sleep(10 * time.Second)
|
||||
|
||||
// start the two clusters in parallel
|
||||
logger.Infof("starting two clusters, one traditional and one external")
|
||||
err = o.startCluster(o.namespace1, "bluestore")
|
||||
require.NoError(o.T(), err)
|
||||
// Wait for the monitors to be up so that we can fetch the configmap and secrets
|
||||
require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-mon", o.namespace1, "Running"),
|
||||
"Make sure rook-ceph-mon is in running state")
|
||||
// Start the core storage cluster
|
||||
s.setupMultiClusterCore()
|
||||
s.createPools()
|
||||
|
||||
// Start the external cluster that will connect to the core cluster
|
||||
// create an external cluster
|
||||
err = o.startExternalCluster(o.namespace2)
|
||||
require.NoError(o.T(), err)
|
||||
s.startExternalCluster()
|
||||
|
||||
logger.Infof("finished starting clusters")
|
||||
}
|
||||
|
||||
// Teardown is a wrapper for tearDown after suite
|
||||
func (o MCTestOperations) Teardown() {
|
||||
o.installer.UninstallRookFromMultipleNS(installer.SystemNamespace(o.namespace1), o.namespace1, o.namespace2)
|
||||
func (s *MultiClusterDeploySuite) AfterTest(suiteName, testName string) {
|
||||
s.installer.CollectOperatorLog(suiteName, testName)
|
||||
}
|
||||
|
||||
func (o MCTestOperations) startCluster(namespace, store string) error {
|
||||
logger.Infof("starting cluster %s", namespace)
|
||||
multipleManagers := true
|
||||
err := o.installer.CreateRookCluster(namespace, o.systemNamespace, store, o.testOverPVC, o.storageClassName,
|
||||
cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, multipleManagers, false, false, installer.NautilusVersion)
|
||||
func (s *MultiClusterDeploySuite) createPools() {
|
||||
// create a test pool in each cluster so that we get some PGs
|
||||
logger.Infof("Creating pool %s", s.poolName)
|
||||
err := s.testClient.PoolClient.Create(s.poolName, s.settings.Namespace, 1)
|
||||
require.Nil(s.T(), err)
|
||||
}
|
||||
|
||||
func (s *MultiClusterDeploySuite) deletePools() {
|
||||
// create a test pool in each cluster so that we get some PGs
|
||||
clusterInfo := client.AdminClusterInfo(s.settings.Namespace)
|
||||
if err := s.testClient.PoolClient.DeletePool(s.testClient.BlockClient, clusterInfo, s.poolName); err != nil {
|
||||
logger.Errorf("failed to delete pool %q. %v", s.poolName, err)
|
||||
} else {
|
||||
logger.Infof("deleted pool %q", s.poolName)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MultiClusterDeploySuite) TearDownSuite() {
|
||||
s.deletePools()
|
||||
s.installer.UninstallRookFromMultipleNS(s.installer.Manifests, s.externalManifests)
|
||||
}
|
||||
|
||||
// Test to make sure all rook components are installed and Running
|
||||
func (s *MultiClusterDeploySuite) TestInstallingMultipleRookClusters() {
|
||||
// Check if Rook cluster 1 is deployed successfully
|
||||
client.RunAllCephCommandsInToolboxPod = s.coreToolbox
|
||||
checkIfRookClusterIsInstalled(s.Suite, s.k8sh, s.settings.OperatorNamespace, s.settings.Namespace, 1)
|
||||
checkIfRookClusterIsHealthy(s.Suite, s.testClient, s.settings.Namespace)
|
||||
|
||||
// Check if Rook external cluster is deployed successfully
|
||||
// Checking health status is enough to validate the connection
|
||||
client.RunAllCephCommandsInToolboxPod = s.externalToolbox
|
||||
checkIfRookClusterIsHealthy(s.Suite, s.testClient, s.externalManifests.Settings().Namespace)
|
||||
}
|
||||
|
||||
// Setup is wrapper for setting up multiple rook clusters.
|
||||
func (s *MultiClusterDeploySuite) setupMultiClusterCore() {
|
||||
root, err := utils.FindRookRoot()
|
||||
require.NoError(s.T(), err, "failed to get rook root")
|
||||
cmdArgs := utils.CommandArgs{Command: filepath.Join(root, localPathPVCmd),
|
||||
CmdArgs: []string{installer.TestScratchDevice()}}
|
||||
cmdOut := utils.ExecuteCommand(cmdArgs)
|
||||
require.NoError(s.T(), cmdOut.Err)
|
||||
|
||||
s.installer, s.k8sh = StartTestCluster(s.T, s.settings, multiClusterMinimalTestVersion)
|
||||
s.testClient = clients.CreateTestClient(s.k8sh, s.installer.Manifests)
|
||||
s.coreToolbox = client.RunAllCephCommandsInToolboxPod
|
||||
}
|
||||
|
||||
func (s *MultiClusterDeploySuite) startExternalCluster() {
|
||||
err := s.installer.CreateRookExternalCluster(s.externalManifests)
|
||||
if err != nil {
|
||||
o.T().Fail()
|
||||
o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace)
|
||||
return fmt.Errorf("failed to create cluster %s. %+v", namespace, err)
|
||||
s.T().Fail()
|
||||
s.installer.GatherAllRookLogs(s.T().Name(), s.externalManifests.Settings().Namespace)
|
||||
require.NoError(s.T(), err)
|
||||
}
|
||||
|
||||
if err := o.installer.CreateRookToolbox(namespace); err != nil {
|
||||
o.T().Fail()
|
||||
o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace)
|
||||
return fmt.Errorf("failed to create toolbox for %s. %+v", namespace, err)
|
||||
}
|
||||
logger.Infof("succeeded starting cluster %s", namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o MCTestOperations) startExternalCluster(namespace string) error {
|
||||
logger.Infof("starting external cluster %q", namespace)
|
||||
err := o.installer.CreateRookExternalCluster(namespace, o.namespace1)
|
||||
if err != nil {
|
||||
o.T().Fail()
|
||||
o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace)
|
||||
return fmt.Errorf("failed to create external cluster %s. %+v", namespace, err)
|
||||
}
|
||||
|
||||
logger.Infof("running toolbox on namespace %q", namespace)
|
||||
if err := o.installer.CreateRookToolbox(namespace); err != nil {
|
||||
o.T().Fail()
|
||||
o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace)
|
||||
return fmt.Errorf("failed to create toolbox for %s. %+v", namespace, err)
|
||||
}
|
||||
|
||||
logger.Infof("succeeded starting external cluster %s", namespace)
|
||||
return nil
|
||||
s.externalToolbox = client.RunAllCephCommandsInToolboxPod
|
||||
logger.Infof("succeeded starting external cluster %s", s.externalManifests.Settings().Namespace)
|
||||
}
|
||||
@@ -72,7 +72,7 @@ func TestCephSmokeSuite(t *testing.T) {
|
||||
|
||||
s := new(SmokeSuite)
|
||||
defer func(s *SmokeSuite) {
|
||||
HandlePanics(recover(), s.op, s.T)
|
||||
HandlePanics(recover(), s.TearDownSuite, s.T)
|
||||
}(s)
|
||||
suite.Run(t, s)
|
||||
}
|
||||
@@ -80,83 +80,85 @@ func TestCephSmokeSuite(t *testing.T) {
|
||||
type SmokeSuite struct {
|
||||
suite.Suite
|
||||
helper *clients.TestClient
|
||||
op *TestCluster
|
||||
settings *installer.TestCephSettings
|
||||
installer *installer.CephInstaller
|
||||
k8sh *utils.K8sHelper
|
||||
namespace string
|
||||
}
|
||||
|
||||
func (suite *SmokeSuite) SetupSuite() {
|
||||
suite.namespace = "smoke-ns"
|
||||
smokeTestCluster := TestCluster{
|
||||
namespace: suite.namespace,
|
||||
storeType: "bluestore",
|
||||
storageClassName: installer.StorageClassName(),
|
||||
useHelm: false,
|
||||
usePVC: installer.UsePVC(),
|
||||
mons: 3,
|
||||
rbdMirrorWorkers: 1,
|
||||
rookCephCleanup: true,
|
||||
skipOSDCreation: false,
|
||||
minimalMatrixK8sVersion: smokeSuiteMinimalTestVersion,
|
||||
rookVersion: installer.VersionMaster,
|
||||
cephVersion: installer.OctopusVersion,
|
||||
func (s *SmokeSuite) SetupSuite() {
|
||||
namespace := "smoke-ns"
|
||||
s.settings = &installer.TestCephSettings{
|
||||
ClusterName: "smoke-cluster",
|
||||
Namespace: namespace,
|
||||
OperatorNamespace: installer.SystemNamespace(namespace),
|
||||
StorageClassName: installer.StorageClassName(),
|
||||
UseHelm: false,
|
||||
UsePVC: installer.UsePVC(),
|
||||
Mons: 3,
|
||||
SkipOSDCreation: false,
|
||||
UseCSI: true,
|
||||
EnableAdmissionController: true,
|
||||
UseCrashPruner: true,
|
||||
RookVersion: installer.VersionMaster,
|
||||
CephVersion: installer.OctopusVersion,
|
||||
}
|
||||
s.settings.ApplyEnvVars()
|
||||
|
||||
suite.op, suite.k8sh = StartTestCluster(suite.T, &smokeTestCluster)
|
||||
suite.helper = clients.CreateTestClient(suite.k8sh, suite.op.installer.Manifests)
|
||||
s.installer, s.k8sh = StartTestCluster(s.T, s.settings, smokeSuiteMinimalTestVersion)
|
||||
s.helper = clients.CreateTestClient(s.k8sh, s.installer.Manifests)
|
||||
}
|
||||
|
||||
func (suite *SmokeSuite) AfterTest(suiteName, testName string) {
|
||||
suite.op.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(suite.namespace))
|
||||
func (s *SmokeSuite) AfterTest(suiteName, testName string) {
|
||||
s.installer.CollectOperatorLog(suiteName, testName)
|
||||
}
|
||||
|
||||
func (suite *SmokeSuite) TearDownSuite() {
|
||||
suite.op.Teardown()
|
||||
func (s *SmokeSuite) TearDownSuite() {
|
||||
s.installer.UninstallRook()
|
||||
}
|
||||
|
||||
func (suite *SmokeSuite) TestBlockStorage_SmokeTest() {
|
||||
runBlockCSITest(suite.helper, suite.k8sh, suite.Suite, suite.namespace)
|
||||
func (s *SmokeSuite) TestBlockStorage_SmokeTest() {
|
||||
runBlockCSITest(s.helper, s.k8sh, s.Suite, s.settings.Namespace)
|
||||
}
|
||||
|
||||
func (suite *SmokeSuite) TestFileStorage_SmokeTest() {
|
||||
useCSI := true
|
||||
func (s *SmokeSuite) TestFileStorage_SmokeTest() {
|
||||
preserveFilesystemOnDelete := true
|
||||
runFileE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace, "smoke-test-fs", useCSI, preserveFilesystemOnDelete)
|
||||
runFileE2ETest(s.helper, s.k8sh, s.Suite, s.settings, "smoke-test-fs", preserveFilesystemOnDelete)
|
||||
}
|
||||
|
||||
func (suite *SmokeSuite) TestObjectStorage_SmokeTest() {
|
||||
if !utils.IsPlatformOpenShift() {
|
||||
runObjectE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace)
|
||||
func (s *SmokeSuite) TestObjectStorage_SmokeTest() {
|
||||
if utils.IsPlatformOpenShift() {
|
||||
s.T().Skip("object store tests skipped on openshift")
|
||||
}
|
||||
runObjectE2ETest(s.helper, s.k8sh, s.Suite, s.settings.Namespace)
|
||||
}
|
||||
|
||||
// Test to make sure all rook components are installed and Running
|
||||
func (suite *SmokeSuite) TestARookClusterInstallation_SmokeTest() {
|
||||
checkIfRookClusterIsInstalled(suite.Suite, suite.k8sh, installer.SystemNamespace(suite.namespace), suite.namespace, 3)
|
||||
func (s *SmokeSuite) TestARookClusterInstallation_SmokeTest() {
|
||||
checkIfRookClusterIsInstalled(s.Suite, s.k8sh, s.settings.OperatorNamespace, s.settings.Namespace, 3)
|
||||
}
|
||||
|
||||
// Smoke Test for Mon failover - Test check the following operations for the Mon failover in order
|
||||
// Delete mon pod, Wait for new mon pod
|
||||
func (suite *SmokeSuite) TestMonFailover() {
|
||||
func (s *SmokeSuite) TestMonFailover() {
|
||||
ctx := context.TODO()
|
||||
logger.Infof("Mon Failover Smoke Test")
|
||||
|
||||
deployments, err := suite.getNonCanaryMonDeployments()
|
||||
require.Nil(suite.T(), err)
|
||||
require.Equal(suite.T(), 3, len(deployments))
|
||||
deployments, err := s.getNonCanaryMonDeployments()
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), 3, len(deployments))
|
||||
|
||||
monToKill := deployments[0].Name
|
||||
logger.Infof("Killing mon %s", monToKill)
|
||||
propagation := metav1.DeletePropagationForeground
|
||||
delOptions := &metav1.DeleteOptions{PropagationPolicy: &propagation}
|
||||
err = suite.k8sh.Clientset.AppsV1().Deployments(suite.namespace).Delete(ctx, monToKill, *delOptions)
|
||||
require.Nil(suite.T(), err)
|
||||
err = s.k8sh.Clientset.AppsV1().Deployments(s.settings.Namespace).Delete(ctx, monToKill, *delOptions)
|
||||
require.NoError(s.T(), err)
|
||||
|
||||
// Wait for the health check to start a new monitor
|
||||
originalMonDeleted := false
|
||||
for i := 0; i < 30; i++ {
|
||||
deployments, err := suite.getNonCanaryMonDeployments()
|
||||
require.Nil(suite.T(), err)
|
||||
deployments, err := s.getNonCanaryMonDeployments()
|
||||
require.NoError(s.T(), err)
|
||||
|
||||
// Make sure the old mon is not still alive
|
||||
foundOldMon := false
|
||||
@@ -188,31 +190,31 @@ func (suite *SmokeSuite) TestMonFailover() {
|
||||
return
|
||||
}
|
||||
|
||||
assert.Equal(suite.T(), 2, len(deployments))
|
||||
assert.Equal(s.T(), 2, len(deployments))
|
||||
}
|
||||
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
|
||||
require.Fail(suite.T(), "giving up waiting for a new monitor")
|
||||
require.Fail(s.T(), "giving up waiting for a new monitor")
|
||||
}
|
||||
|
||||
// Smoke Test for pool Resizing
|
||||
func (suite *SmokeSuite) TestPoolResize() {
|
||||
func (s *SmokeSuite) TestPoolResize() {
|
||||
ctx := context.TODO()
|
||||
logger.Infof("Pool Resize Smoke Test")
|
||||
|
||||
poolName := "testpool"
|
||||
err := suite.helper.PoolClient.Create(poolName, suite.namespace, 1)
|
||||
require.Nil(suite.T(), err)
|
||||
err := s.helper.PoolClient.Create(poolName, s.settings.Namespace, 1)
|
||||
require.NoError(s.T(), err)
|
||||
|
||||
poolFound := false
|
||||
clusterInfo := client.AdminClusterInfo(suite.namespace)
|
||||
clusterInfo := client.AdminClusterInfo(s.settings.Namespace)
|
||||
|
||||
// Wait for pool to appear
|
||||
for i := 0; i < 10; i++ {
|
||||
pools, err := suite.helper.PoolClient.ListCephPools(clusterInfo)
|
||||
require.Nil(suite.T(), err)
|
||||
pools, err := s.helper.PoolClient.ListCephPools(clusterInfo)
|
||||
require.NoError(s.T(), err)
|
||||
for _, p := range pools {
|
||||
if p.Name != poolName {
|
||||
continue
|
||||
@@ -227,24 +229,24 @@ func (suite *SmokeSuite) TestPoolResize() {
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
|
||||
require.Equal(suite.T(), true, poolFound, "pool not found")
|
||||
require.Equal(s.T(), true, poolFound, "pool not found")
|
||||
|
||||
err = suite.helper.PoolClient.Update(poolName, suite.namespace, 2)
|
||||
require.Nil(suite.T(), err)
|
||||
err = s.helper.PoolClient.Update(poolName, s.settings.Namespace, 2)
|
||||
require.NoError(s.T(), err)
|
||||
|
||||
poolResized := false
|
||||
// Wait for pool resize to happen
|
||||
for i := 0; i < 10; i++ {
|
||||
details, err := suite.helper.PoolClient.GetCephPoolDetails(clusterInfo, poolName)
|
||||
require.Nil(suite.T(), err)
|
||||
details, err := s.helper.PoolClient.GetCephPoolDetails(clusterInfo, poolName)
|
||||
require.NoError(s.T(), err)
|
||||
if details.Size > 1 {
|
||||
logger.Infof("pool %s size was updated", poolName)
|
||||
require.Equal(suite.T(), 2, int(details.Size))
|
||||
require.Equal(s.T(), 2, int(details.Size))
|
||||
poolResized = true
|
||||
|
||||
// resize the pool back to 1 to avoid hangs around not having enough OSDs to satisfy rbd
|
||||
err = suite.helper.PoolClient.Update(poolName, suite.namespace, 1)
|
||||
require.Nil(suite.T(), err)
|
||||
err = s.helper.PoolClient.Update(poolName, s.settings.Namespace, 1)
|
||||
require.NoError(s.T(), err)
|
||||
} else if poolResized && details.Size == 1 {
|
||||
logger.Infof("pool resized back to 1")
|
||||
break
|
||||
@@ -255,45 +257,45 @@ func (suite *SmokeSuite) TestPoolResize() {
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
|
||||
require.Equal(suite.T(), true, poolResized, fmt.Sprintf("pool %s not found", poolName))
|
||||
require.Equal(s.T(), true, poolResized, fmt.Sprintf("pool %s not found", poolName))
|
||||
|
||||
// Verify the Kubernetes Secret has been created (bootstrap peer token)
|
||||
pool, err := suite.k8sh.RookClientset.CephV1().CephBlockPools(suite.namespace).Get(ctx, poolName, metav1.GetOptions{})
|
||||
assert.NoError(suite.T(), err)
|
||||
pool, err := s.k8sh.RookClientset.CephV1().CephBlockPools(s.settings.Namespace).Get(ctx, poolName, metav1.GetOptions{})
|
||||
assert.NoError(s.T(), err)
|
||||
if pool.Spec.Mirroring.Enabled {
|
||||
secretName := pool.Status.Info[oppool.RBDMirrorBootstrapPeerSecretName]
|
||||
assert.NotEmpty(suite.T(), secretName)
|
||||
assert.NotEmpty(s.T(), secretName)
|
||||
// now fetch the secret which contains the bootstrap peer token
|
||||
s, err := suite.k8sh.Clientset.CoreV1().Secrets(suite.namespace).Get(ctx, secretName, metav1.GetOptions{})
|
||||
require.Nil(suite.T(), err)
|
||||
assert.NotEmpty(suite.T(), s.Data["token"])
|
||||
secret, err := s.k8sh.Clientset.CoreV1().Secrets(s.settings.Namespace).Get(ctx, secretName, metav1.GetOptions{})
|
||||
require.NoError(s.T(), err)
|
||||
assert.NotEmpty(s.T(), secret.Data["token"])
|
||||
|
||||
// Once we have a scenario with another Ceph cluster - needs to be added in the MultiCluster suite
|
||||
// We would need to add a bootstrap peer token following the below procedure
|
||||
// bootstrapSecretName := "bootstrap-peer-token"
|
||||
// token := "eyJmc2lkIjoiYzZiMDg3ZjItNzgyOS00ZGJiLWJjZmMtNTNkYzM0ZTBiMzVkIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBV1lsWmZVQ1Q2RGhBQVBtVnAwbGtubDA5YVZWS3lyRVV1NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMTExLjEwOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTA6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjEyOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTI6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjExOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTE6Njc4OV0ifQ=="
|
||||
// s = oppool.GenerateBootstrapPeerSecret(bootstrapSecretName, suite.namespace, string(pool.GetUID()), []byte(token))
|
||||
// s, err = suite.k8sh.Clientset.CoreV1().Secrets(suite.namespace).Create(s)
|
||||
// require.Nil(suite.T(), err, err.Error())
|
||||
// s = oppool.GenerateBootstrapPeerSecret(bootstrapSecretName, s.settings.Namespace, string(pool.GetUID()), []byte(token))
|
||||
// s, err = s.k8sh.Clientset.CoreV1().Secrets(s.settings.Namespace).Create(s)
|
||||
// require.Nil(s.T(), err, err.Error())
|
||||
|
||||
// // update the ceph block pool cr
|
||||
// pool.Spec.Mirrored.PeersSecretNames = append(pool.Spec.Mirrored.PeersSecretNames, bootstrapSecretName)
|
||||
// _, err = suite.k8sh.RookClientset.CephV1().CephBlockPools(suite.namespace).Update(pool)
|
||||
// require.Nil(suite.T(), err, err.Error())
|
||||
// _, err = s.k8sh.RookClientset.CephV1().CephBlockPools(s.settings.Namespace).Update(pool)
|
||||
// require.Nil(s.T(), err, err.Error())
|
||||
|
||||
// mirrorInfo, err := client.PrintPoolMirroringInfo(suite.k8sh.MakeContext(), clusterInfo, poolName)
|
||||
// require.Nil(suite.T(), err, err.Error())
|
||||
// assert.Equal(suite.T(), "image", mirrorInfo.Mode)
|
||||
// assert.Equal(suite.T(), 1, len(mirrorInfo.Peers))
|
||||
// mirrorInfo, err := client.PrintPoolMirroringInfo(s.k8sh.MakeContext(), clusterInfo, poolName)
|
||||
// require.Nil(s.T(), err, err.Error())
|
||||
// assert.Equal(s.T(), "image", mirrorInfo.Mode)
|
||||
// assert.Equal(s.T(), 1, len(mirrorInfo.Peers))
|
||||
}
|
||||
|
||||
// clean up the pool
|
||||
err = suite.helper.PoolClient.DeletePool(suite.helper.BlockClient, clusterInfo, poolName)
|
||||
assert.NoError(suite.T(), err)
|
||||
err = s.helper.PoolClient.DeletePool(s.helper.BlockClient, clusterInfo, poolName)
|
||||
assert.NoError(s.T(), err)
|
||||
}
|
||||
|
||||
// Smoke Test for Client CRD
|
||||
func (suite *SmokeSuite) TestCreateClient() {
|
||||
func (s *SmokeSuite) TestCreateClient() {
|
||||
logger.Infof("Create Client Smoke Test")
|
||||
|
||||
clientName := "client1"
|
||||
@@ -302,14 +304,14 @@ func (suite *SmokeSuite) TestCreateClient() {
|
||||
"mgr": "allow rwx",
|
||||
"osd": "allow rwx",
|
||||
}
|
||||
clusterInfo := client.AdminClusterInfo(suite.namespace)
|
||||
err := suite.helper.UserClient.Create(clientName, suite.namespace, caps)
|
||||
require.Nil(suite.T(), err)
|
||||
clusterInfo := client.AdminClusterInfo(s.settings.Namespace)
|
||||
err := s.helper.UserClient.Create(clientName, s.settings.Namespace, caps)
|
||||
require.NoError(s.T(), err)
|
||||
|
||||
clientFound := false
|
||||
|
||||
for i := 0; i < 30; i++ {
|
||||
clients, _ := suite.helper.UserClient.Get(clusterInfo, "client."+clientName)
|
||||
clients, _ := s.helper.UserClient.Get(clusterInfo, "client."+clientName)
|
||||
if clients != "" {
|
||||
clientFound = true
|
||||
}
|
||||
@@ -322,7 +324,7 @@ func (suite *SmokeSuite) TestCreateClient() {
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
|
||||
require.Equal(suite.T(), true, clientFound, "client not found")
|
||||
assert.Equal(s.T(), true, clientFound, "client not found")
|
||||
|
||||
logger.Infof("Update Client Smoke Test")
|
||||
newcaps := map[string]string{
|
||||
@@ -330,30 +332,33 @@ func (suite *SmokeSuite) TestCreateClient() {
|
||||
"mgr": "allow rw",
|
||||
"osd": "allow *",
|
||||
}
|
||||
caps, _ = suite.helper.UserClient.Update(clusterInfo, clientName, newcaps)
|
||||
caps, _ = s.helper.UserClient.Update(clusterInfo, clientName, newcaps)
|
||||
|
||||
require.Equal(suite.T(), "allow r", caps["mon"], "wrong caps")
|
||||
require.Equal(suite.T(), "allow rw", caps["mgr"], "wrong caps")
|
||||
require.Equal(suite.T(), "allow *", caps["osd"], "wrong caps")
|
||||
assert.Equal(s.T(), "allow r", caps["mon"], "wrong caps")
|
||||
assert.Equal(s.T(), "allow rw", caps["mgr"], "wrong caps")
|
||||
assert.Equal(s.T(), "allow *", caps["osd"], "wrong caps")
|
||||
|
||||
err = s.helper.UserClient.Delete(clientName, s.settings.Namespace)
|
||||
require.NoError(s.T(), err)
|
||||
}
|
||||
|
||||
// Smoke Test for RBD Mirror CRD
|
||||
func (suite *SmokeSuite) TestCreateRBDMirrorClient() {
|
||||
func (s *SmokeSuite) TestCreateRBDMirrorClient() {
|
||||
logger.Infof("Create rbd-mirror Smoke Test")
|
||||
|
||||
rbdMirrorName := "my-rbd-mirror"
|
||||
|
||||
err := suite.helper.RBDMirrorClient.Create(suite.namespace, rbdMirrorName, 1)
|
||||
require.Nil(suite.T(), err)
|
||||
err := s.helper.RBDMirrorClient.Create(s.settings.Namespace, rbdMirrorName, 1)
|
||||
require.NoError(s.T(), err)
|
||||
|
||||
err = suite.helper.RBDMirrorClient.Delete(suite.namespace, rbdMirrorName)
|
||||
require.Nil(suite.T(), err)
|
||||
err = s.helper.RBDMirrorClient.Delete(s.settings.Namespace, rbdMirrorName)
|
||||
require.NoError(s.T(), err)
|
||||
}
|
||||
|
||||
func (suite *SmokeSuite) getNonCanaryMonDeployments() ([]appsv1.Deployment, error) {
|
||||
func (s *SmokeSuite) getNonCanaryMonDeployments() ([]appsv1.Deployment, error) {
|
||||
ctx := context.TODO()
|
||||
opts := metav1.ListOptions{LabelSelector: "app=rook-ceph-mon"}
|
||||
deployments, err := suite.k8sh.Clientset.AppsV1().Deployments(suite.namespace).List(ctx, opts)
|
||||
deployments, err := s.k8sh.Clientset.AppsV1().Deployments(s.settings.Namespace).List(ctx, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -58,7 +58,7 @@ func TestCephUpgradeSuite(t *testing.T) {
|
||||
|
||||
s := new(UpgradeSuite)
|
||||
defer func(s *UpgradeSuite) {
|
||||
HandlePanics(recover(), s.op, s.T)
|
||||
HandlePanics(recover(), s.TearDownSuite, s.T)
|
||||
}(s)
|
||||
suite.Run(t, s)
|
||||
}
|
||||
@@ -66,40 +66,37 @@ func TestCephUpgradeSuite(t *testing.T) {
|
||||
type UpgradeSuite struct {
|
||||
suite.Suite
|
||||
helper *clients.TestClient
|
||||
op *TestCluster
|
||||
k8sh *utils.K8sHelper
|
||||
settings *installer.TestCephSettings
|
||||
installer *installer.CephInstaller
|
||||
namespace string
|
||||
}
|
||||
|
||||
func (s *UpgradeSuite) SetupSuite() {
|
||||
s.namespace = "upgrade-ns"
|
||||
upgradeTestCluster := TestCluster{
|
||||
clusterName: s.namespace,
|
||||
namespace: s.namespace,
|
||||
storeType: "",
|
||||
storageClassName: "",
|
||||
useHelm: false,
|
||||
usePVC: false,
|
||||
mons: 1,
|
||||
rbdMirrorWorkers: 0,
|
||||
rookCephCleanup: false,
|
||||
skipOSDCreation: false,
|
||||
minimalMatrixK8sVersion: upgradeMinimalTestVersion,
|
||||
rookVersion: installer.Version1_5,
|
||||
cephVersion: installer.NautilusVersion,
|
||||
s.settings = &installer.TestCephSettings{
|
||||
ClusterName: s.namespace,
|
||||
Namespace: s.namespace,
|
||||
OperatorNamespace: installer.SystemNamespace(s.namespace),
|
||||
StorageClassName: "",
|
||||
UseHelm: false,
|
||||
UsePVC: false,
|
||||
Mons: 1,
|
||||
UseCSI: true,
|
||||
SkipOSDCreation: false,
|
||||
RookVersion: installer.Version1_5,
|
||||
CephVersion: installer.NautilusVersion,
|
||||
}
|
||||
|
||||
s.op, s.k8sh = StartTestCluster(s.T, &upgradeTestCluster)
|
||||
s.helper = clients.CreateTestClient(s.k8sh, s.op.installer.Manifests)
|
||||
s.installer, s.k8sh = StartTestCluster(s.T, s.settings, upgradeMinimalTestVersion)
|
||||
s.helper = clients.CreateTestClient(s.k8sh, s.installer.Manifests)
|
||||
}
|
||||
|
||||
func (s *UpgradeSuite) TearDownSuite() {
|
||||
s.op.Teardown()
|
||||
s.installer.UninstallRook()
|
||||
}
|
||||
|
||||
func (s *UpgradeSuite) TestUpgradeToMaster() {
|
||||
systemNamespace := installer.SystemNamespace(s.namespace)
|
||||
|
||||
//
|
||||
// Create block, object, and file storage before the upgrade
|
||||
//
|
||||
@@ -108,7 +105,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
|
||||
blockName := "block-claim-upgrade"
|
||||
logger.Infof("Initializing block before the upgrade")
|
||||
clusterInfo := client.AdminClusterInfo(s.namespace)
|
||||
setupBlockLite(s.helper, s.k8sh, s.Suite, clusterInfo, systemNamespace, poolName, storageClassName, blockName, rbdPodName, s.op.installer.CephVersion)
|
||||
setupBlockLite(s.helper, s.k8sh, s.Suite, clusterInfo, poolName, storageClassName, blockName, rbdPodName)
|
||||
|
||||
createPodWithBlock(s.helper, s.k8sh, s.Suite, s.namespace, storageClassName, rbdPodName, blockName)
|
||||
|
||||
@@ -121,13 +118,12 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
|
||||
logger.Infof("Initializing file before the upgrade")
|
||||
filesystemName := "upgrade-test-fs"
|
||||
activeCount := 1
|
||||
createFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, activeCount)
|
||||
createFilesystem(s.helper, s.k8sh, s.Suite, s.settings, filesystemName, activeCount)
|
||||
|
||||
// Start the file test client
|
||||
fsStorageClass := "file-upgrade"
|
||||
assert.NoError(s.T(), s.helper.FSClient.CreateStorageClass(filesystemName, systemNamespace, s.namespace, fsStorageClass))
|
||||
useCSI := true
|
||||
createFilesystemConsumerPod(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, fsStorageClass, useCSI)
|
||||
assert.NoError(s.T(), s.helper.FSClient.CreateStorageClass(filesystemName, s.settings.OperatorNamespace, s.namespace, fsStorageClass))
|
||||
createFilesystemConsumerPod(s.helper, s.k8sh, s.Suite, s.settings, filesystemName, fsStorageClass)
|
||||
defer func() {
|
||||
cleanupFilesystemConsumer(s.helper, s.k8sh, s.Suite, s.namespace, filePodName)
|
||||
cleanupFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName)
|
||||
@@ -135,7 +131,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
|
||||
|
||||
logger.Infof("Initializing object before the upgrade")
|
||||
objectStoreName := "upgraded-object"
|
||||
runObjectE2ETestLite(s.helper, s.k8sh, s.Suite, s.namespace, objectStoreName, 1, false)
|
||||
runObjectE2ETestLite(s.helper, s.k8sh, s.Suite, s.settings, objectStoreName, 1, false)
|
||||
|
||||
logger.Info("Initializing object bucket claim before the upgrade")
|
||||
bucketStorageClassName := "rook-smoke-delete-bucket"
|
||||
@@ -147,6 +143,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
|
||||
defer func() {
|
||||
_ = s.helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketPrefix, maxObject, false)
|
||||
_ = s.helper.BucketClient.DeleteBucketStorageClass(s.namespace, objectStoreName, bucketStorageClassName, "Delete", region)
|
||||
objectStoreCleanUp(s.Suite, s.helper, s.k8sh, s.settings.Namespace, objectStoreName)
|
||||
}()
|
||||
|
||||
created := utils.Retry(12, 2*time.Second, "OBC is created", func() bool {
|
||||
@@ -178,7 +175,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
|
||||
// Upgrade Rook from v1.5 to master
|
||||
//
|
||||
logger.Infof("*** UPGRADING ROOK FROM %s to master ***", installer.Version1_5)
|
||||
s.gatherLogs(systemNamespace, "_before_master_upgrade")
|
||||
s.gatherLogs(s.settings.OperatorNamespace, "_before_master_upgrade")
|
||||
s.upgradeToMaster()
|
||||
|
||||
s.verifyOperatorImage(installer.VersionMaster)
|
||||
@@ -199,7 +196,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
|
||||
// Upgrade from nautilus to octopus
|
||||
//
|
||||
logger.Infof("*** UPGRADING CEPH FROM Nautilus TO Octopus ***")
|
||||
s.gatherLogs(systemNamespace, "_before_octopus_upgrade")
|
||||
s.gatherLogs(s.settings.OperatorNamespace, "_before_octopus_upgrade")
|
||||
s.upgradeCephVersion(installer.OctopusVersion.Image, numOSDs)
|
||||
// Verify reading and writing to the test clients
|
||||
newFile = "post-octopus-upgrade-file"
|
||||
@@ -213,7 +210,7 @@ func (s *UpgradeSuite) gatherLogs(systemNamespace, testSuffix string) {
|
||||
s.k8sh.PrintPodDescribe(s.namespace)
|
||||
}
|
||||
n := strings.Replace(s.T().Name(), "/", "_", -1) + testSuffix
|
||||
s.op.installer.GatherAllRookLogs(n, systemNamespace, s.namespace)
|
||||
s.installer.GatherAllRookLogs(n, systemNamespace, s.namespace)
|
||||
}
|
||||
|
||||
func (s *UpgradeSuite) upgradeCephVersion(newCephImage string, numOSDs int) {
|
||||
@@ -241,7 +238,7 @@ func (s *UpgradeSuite) verifyRookUpgrade(numOSDs int) {
|
||||
// Get some info about the currently deployed mons to determine later if they are all updated
|
||||
monDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-mon")
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), s.op.mons, len(monDepList.Items), monDepList.Items)
|
||||
require.Equal(s.T(), s.settings.Mons, len(monDepList.Items), monDepList.Items)
|
||||
|
||||
// Get some info about the currently deployed mgr to determine later if it is updated
|
||||
mgrDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-mgr")
|
||||
@@ -263,7 +260,7 @@ func (s *UpgradeSuite) verifyRookUpgrade(numOSDs int) {
|
||||
func (s *UpgradeSuite) waitForUpgradedDaemons(previousVersion, versionLabel string, numOSDs int, waitForMDS bool) {
|
||||
// wait for the mon(s) to be updated
|
||||
monsNotOldVersion := fmt.Sprintf("app=rook-ceph-mon,%s!=%s", versionLabel, previousVersion)
|
||||
err := s.k8sh.WaitForDeploymentCount(monsNotOldVersion, s.namespace, s.op.mons)
|
||||
err := s.k8sh.WaitForDeploymentCount(monsNotOldVersion, s.namespace, s.settings.Mons)
|
||||
require.NoError(s.T(), err, "mon(s) didn't update")
|
||||
err = s.k8sh.WaitForLabeledDeploymentsToBeReady(monsNotOldVersion, s.namespace)
|
||||
require.NoError(s.T(), err)
|
||||
@@ -336,19 +333,13 @@ func (s *UpgradeSuite) verifyFilesAfterUpgrade(fsName, newFileToWrite, messageFo
|
||||
// UpgradeToMaster performs the steps necessary to upgrade a Rook v1.4 cluster to master. It does not
|
||||
// verify the upgrade but merely starts the upgrade process.
|
||||
func (s *UpgradeSuite) upgradeToMaster() {
|
||||
v1ExtensionsSupported := false
|
||||
if s.k8sh.VersionAtLeast("v1.16.0") {
|
||||
v1ExtensionsSupported = true
|
||||
}
|
||||
require.NoError(s.T(), s.k8sh.ResourceOperation("apply", upgradeManifestToMaster(s.namespace, v1ExtensionsSupported)))
|
||||
// Apply the CRDs for the latest master
|
||||
s.settings.RookVersion = installer.VersionMaster
|
||||
m := installer.NewCephManifests(s.settings)
|
||||
require.NoError(s.T(), s.k8sh.ResourceOperation("apply", m.GetCRDs(s.k8sh)))
|
||||
|
||||
require.NoError(s.T(), s.k8sh.ResourceOperation("apply", m.GetCommon()))
|
||||
|
||||
require.NoError(s.T(),
|
||||
s.k8sh.SetDeploymentVersion(installer.SystemNamespace(s.namespace), operatorContainer, operatorContainer, installer.VersionMaster))
|
||||
}
|
||||
|
||||
func upgradeManifestToMaster(namespace string, useV1Extensions bool) string {
|
||||
// Always apply CRDs for the latest master
|
||||
m := installer.NewCephManifests(installer.VersionMaster)
|
||||
manifests := m.GetRookCRDs(useV1Extensions)
|
||||
return manifests
|
||||
s.k8sh.SetDeploymentVersion(s.settings.OperatorNamespace, operatorContainer, operatorContainer, installer.VersionMaster))
|
||||
}
|
||||
@@ -45,7 +45,7 @@ func TestNfsSuite(t *testing.T) {
|
||||
|
||||
s := new(NfsSuite)
|
||||
defer func(s *NfsSuite) {
|
||||
HandlePanics(recover(), s, s.T)
|
||||
HandlePanics(recover(), s.Teardown, s.T)
|
||||
}(s)
|
||||
suite.Run(t, s)
|
||||
}
|
||||
@@ -60,87 +60,87 @@ type NfsSuite struct {
|
||||
instanceCount int
|
||||
}
|
||||
|
||||
func (suite *NfsSuite) SetupSuite() {
|
||||
suite.Setup()
|
||||
func (s *NfsSuite) SetupSuite() {
|
||||
s.Setup()
|
||||
}
|
||||
|
||||
func (suite *NfsSuite) TearDownSuite() {
|
||||
suite.Teardown()
|
||||
func (s *NfsSuite) TearDownSuite() {
|
||||
s.Teardown()
|
||||
}
|
||||
|
||||
func (suite *NfsSuite) Setup() {
|
||||
suite.namespace = "rook-nfs"
|
||||
suite.systemNamespace = installer.SystemNamespace(suite.namespace)
|
||||
suite.instanceCount = 1
|
||||
func (s *NfsSuite) Setup() {
|
||||
s.namespace = "rook-nfs"
|
||||
s.systemNamespace = installer.SystemNamespace(s.namespace)
|
||||
s.instanceCount = 1
|
||||
|
||||
k8shelper, err := utils.CreateK8sHelper(suite.T)
|
||||
k8shelper, err := utils.CreateK8sHelper(s.T)
|
||||
v := version.MustParseSemantic(k8shelper.GetK8sServerVersion())
|
||||
if !v.AtLeast(version.MustParseSemantic("1.14.0")) {
|
||||
logger.Info("Skipping NFS tests when not at least K8s v1.14")
|
||||
suite.T().Skip()
|
||||
s.T().Skip()
|
||||
}
|
||||
|
||||
require.NoError(suite.T(), err)
|
||||
suite.k8shelper = k8shelper
|
||||
require.NoError(s.T(), err)
|
||||
s.k8shelper = k8shelper
|
||||
|
||||
k8sversion := suite.k8shelper.GetK8sServerVersion()
|
||||
k8sversion := s.k8shelper.GetK8sServerVersion()
|
||||
logger.Infof("Installing nfs server on k8s %s", k8sversion)
|
||||
|
||||
suite.installer = installer.NewNFSInstaller(suite.k8shelper, suite.T)
|
||||
s.installer = installer.NewNFSInstaller(s.k8shelper, s.T)
|
||||
|
||||
suite.rwClient = clients.CreateReadWriteOperation(suite.k8shelper)
|
||||
s.rwClient = clients.CreateReadWriteOperation(s.k8shelper)
|
||||
|
||||
err = suite.installer.InstallNFSServer(suite.systemNamespace, suite.namespace, suite.instanceCount)
|
||||
err = s.installer.InstallNFSServer(s.systemNamespace, s.namespace, s.instanceCount)
|
||||
if err != nil {
|
||||
logger.Errorf("nfs server installation failed: %+v", err)
|
||||
suite.T().Fail()
|
||||
suite.Teardown()
|
||||
suite.T().FailNow()
|
||||
s.T().Fail()
|
||||
s.Teardown()
|
||||
s.T().FailNow()
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *NfsSuite) Teardown() {
|
||||
suite.installer.GatherAllNFSServerLogs(suite.systemNamespace, suite.namespace, suite.T().Name())
|
||||
suite.installer.UninstallNFSServer(suite.systemNamespace, suite.namespace)
|
||||
func (s *NfsSuite) Teardown() {
|
||||
s.installer.GatherAllNFSServerLogs(s.systemNamespace, s.namespace, s.T().Name())
|
||||
s.installer.UninstallNFSServer(s.systemNamespace, s.namespace)
|
||||
}
|
||||
|
||||
func (suite *NfsSuite) TestNfsServerInstallation() {
|
||||
logger.Infof("Verifying that nfs server pod %s is running", suite.namespace)
|
||||
func (s *NfsSuite) TestNfsServerInstallation() {
|
||||
logger.Infof("Verifying that nfs server pod %s is running", s.namespace)
|
||||
|
||||
// verify nfs server operator is running OK
|
||||
assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-nfs-operator", suite.systemNamespace, 1, "Running"),
|
||||
assert.True(s.T(), s.k8shelper.CheckPodCountAndState("rook-nfs-operator", s.systemNamespace, 1, "Running"),
|
||||
"1 rook-nfs-operator must be in Running state")
|
||||
|
||||
// verify nfs server instances are running OK
|
||||
assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState(suite.namespace, suite.namespace, suite.instanceCount, "Running"),
|
||||
fmt.Sprintf("%d rook-nfs pods must be in Running state", suite.instanceCount))
|
||||
assert.True(s.T(), s.k8shelper.CheckPodCountAndState(s.namespace, s.namespace, s.instanceCount, "Running"),
|
||||
fmt.Sprintf("%d rook-nfs pods must be in Running state", s.instanceCount))
|
||||
|
||||
// verify bigger export is running OK
|
||||
assert.True(suite.T(), true, suite.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim-bigger"))
|
||||
assert.True(s.T(), true, s.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim-bigger"))
|
||||
|
||||
podList, err := suite.rwClient.CreateWriteClient("nfs-pv-claim-bigger")
|
||||
require.NoError(suite.T(), err)
|
||||
assert.True(suite.T(), true, suite.checkReadData(podList))
|
||||
err = suite.rwClient.Delete()
|
||||
assert.NoError(suite.T(), err)
|
||||
podList, err := s.rwClient.CreateWriteClient("nfs-pv-claim-bigger")
|
||||
require.NoError(s.T(), err)
|
||||
assert.True(s.T(), true, s.checkReadData(podList))
|
||||
err = s.rwClient.Delete()
|
||||
assert.NoError(s.T(), err)
|
||||
|
||||
// verify another smaller export is running OK
|
||||
assert.True(suite.T(), true, suite.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim"))
|
||||
assert.True(s.T(), true, s.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim"))
|
||||
|
||||
defer suite.rwClient.Delete() //nolint, delete a nfs consuming pod in rook
|
||||
podList, err = suite.rwClient.CreateWriteClient("nfs-pv-claim")
|
||||
require.NoError(suite.T(), err)
|
||||
assert.True(suite.T(), true, suite.checkReadData(podList))
|
||||
defer s.rwClient.Delete() //nolint, delete a nfs consuming pod in rook
|
||||
podList, err = s.rwClient.CreateWriteClient("nfs-pv-claim")
|
||||
require.NoError(s.T(), err)
|
||||
assert.True(s.T(), true, s.checkReadData(podList))
|
||||
}
|
||||
|
||||
func (suite *NfsSuite) checkReadData(podList []string) bool {
|
||||
func (s *NfsSuite) checkReadData(podList []string) bool {
|
||||
var result string
|
||||
var err error
|
||||
// the following for loop retries to read data from the first pod in the pod list
|
||||
for i := 0; i < utils.RetryLoop; i++ {
|
||||
// the nfs volume is mounted on "/mnt" and the data(hostname of the pod) is written in "/mnt/data" of the pod
|
||||
// results stores the hostname of either one of the pod which is same as the pod name, which is read from "/mnt/data"
|
||||
result, err = suite.rwClient.Read(podList[0])
|
||||
result, err = s.rwClient.Read(podList[0])
|
||||
logger.Infof("nfs volume read exited, err: %+v. result: %s", err, result)
|
||||
if err == nil {
|
||||
break
|
||||
@@ -148,7 +148,7 @@ func (suite *NfsSuite) checkReadData(podList []string) bool {
|
||||
logger.Warning("nfs volume read failed, will try again")
|
||||
time.Sleep(utils.RetryInterval * time.Second)
|
||||
}
|
||||
require.NoError(suite.T(), err)
|
||||
require.NoError(s.T(), err)
|
||||
// the value of result must be same as the name of pod.
|
||||
if result == podList[0] || result == podList[1] {
|
||||
return true
|
||||
|
||||
@@ -17,7 +17,7 @@ trap cleanup SIGINT ERR
|
||||
SERVER_VERSION=$(kubectl version --short | awk -F "." '/Server Version/ {print $2}')
|
||||
MINIMUM_VERSION=16
|
||||
|
||||
if [ ${SERVER_VERSION} -le ${MINIMUM_VERSION} ]; then
|
||||
if [ ${SERVER_VERSION} -lt ${MINIMUM_VERSION} ]; then
|
||||
echo "required minimum kubernetes version 1.$MINIMUM_VERSION.0"
|
||||
exit
|
||||
fi
|
||||
|
||||
Reference in new issue
Block a user