From 933113eac8a49ba26bb098b8ff3580f9c255d9e4 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Tue, 9 Mar 2021 22:49:59 -0700 Subject: [PATCH 1/6] ceph: remove obsolete storeType setting All clusters are configured with bluestore as the store type therefore we can remove the settings where mentioned in the docs and dead code. Signed-off-by: Travis Nielsen --- Documentation/ceph-cluster-crd.md | 2 -- Documentation/ceph-object-store-crd.md | 2 +- cluster/examples/kubernetes/ceph/cluster.yaml | 1 - cmd/rook/ceph/osd.go | 1 - pkg/operator/ceph/cluster/osd/config/config.go | 4 ---- pkg/operator/ceph/cluster/osd/spec.go | 3 --- 6 files changed, 1 insertion(+), 12 deletions(-) diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index 1ff1f3f79..7d4d28a24 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -459,7 +459,6 @@ The following are the settings for Storage Class Device Sets which can be config The following storage selection settings are specific to Ceph and do not apply to other backends. All variables are key-value pairs represented as strings. * `metadataDevice`: Name of a device to use for the metadata of OSDs on each node. Performance can be improved by using a low latency device (such as SSD or NVMe) as the metadata device, while other spinning platter (HDD) devices on a node are used to store data. Provisioning will fail if the user specifies a `metadataDevice` but that device is not used as a metadata device by Ceph. Notably, `ceph-volume` will not use a device of the same device class (HDD, SSD, NVMe) as OSD devices for metadata, resulting in this failure. -* `storeType`: `bluestore`, the underlying storage format to use for each OSD. The default is set dynamically to `bluestore` for devices and is the only supported format at this point. * `databaseSizeMB`: The size in MB of a bluestore database. Include quotes around the size. * `walSizeMB`: The size in MB of a bluestore write ahead log (WAL). Include quotes around the size. * `deviceClass`: The [CRUSH device class](https://ceph.io/community/new-luminous-crush-device-classes/) to use for this selection of storage devices. (By default, if a device's class has not already been set, OSDs will automatically set a device's class to either `hdd`, `ssd`, or `nvme` based on the hardware properties exposed by the Linux kernel.) These storage classes can then be used to select the devices backing a storage pool by specifying them as the value of [the pool spec's `deviceClass` field](ceph-pool-crd.md#spec). @@ -764,7 +763,6 @@ spec: - name: "sdc1" # One specific partition. Should not have a file system on it. - name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # both device name and explicit udev links are supported config: # configuration can be specified at the node level which overrides the cluster level config - storeType: bluestore - name: "172.17.4.301" deviceFilter: "^sd." ``` diff --git a/Documentation/ceph-object-store-crd.md b/Documentation/ceph-object-store-crd.md index 60c3a00b1..d0cf10e8c 100644 --- a/Documentation/ceph-object-store-crd.md +++ b/Documentation/ceph-object-store-crd.md @@ -12,7 +12,7 @@ Rook allows creation and customization of object stores through the custom resou ### Erasure Coded -Erasure coded pools require the OSDs to use `bluestore` for the configured [`storeType`](ceph-cluster-crd.md#osd-configuration-settings). Additionally, erasure coded pools can only be used with `dataPools`. The `metadataPool` must use a replicated pool. +Erasure coded pools can only be used with `dataPools`. The `metadataPool` must use a replicated pool. > **NOTE**: This sample requires *at least 3 bluestore OSDs*, with each OSD located on a *different node*. diff --git a/cluster/examples/kubernetes/ceph/cluster.yaml b/cluster/examples/kubernetes/ceph/cluster.yaml index 432d50950..0d50ce60e 100644 --- a/cluster/examples/kubernetes/ceph/cluster.yaml +++ b/cluster/examples/kubernetes/ceph/cluster.yaml @@ -218,7 +218,6 @@ spec: # osdsPerDevice: "5" # - name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # devices can be specified using full udev paths # config: # configuration can be specified at the node level which overrides the cluster level config -# storeType: filestore # - name: "172.17.4.301" # deviceFilter: "^sd." # The section for configuring management of daemon disruptions during upgrade or fencing. diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index e37fb23bb..203cdedda 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -115,7 +115,6 @@ func addOSDConfigFlags(command *cobra.Command) { // OSD store config flags command.Flags().IntVar(&cfg.storeConfig.WalSizeMB, "osd-wal-size", osdcfg.WalDefaultSizeMB, "default size (MB) for OSD write ahead log (WAL) (bluestore)") command.Flags().IntVar(&cfg.storeConfig.DatabaseSizeMB, "osd-database-size", 0, "default size (MB) for OSD database (bluestore)") - command.Flags().StringVar(&cfg.storeConfig.StoreType, "osd-store", "", "type of backing OSD store to use (bluestore or filestore)") command.Flags().IntVar(&cfg.storeConfig.OSDsPerDevice, "osds-per-device", 1, "the number of OSDs per device") command.Flags().BoolVar(&cfg.storeConfig.EncryptedDevice, "encrypted-device", false, "whether to encrypt the OSD with dmcrypt") command.Flags().StringVar(&cfg.storeConfig.DeviceClass, "osd-crush-device-class", "", "The device class for all OSDs configured on this node") diff --git a/pkg/operator/ceph/cluster/osd/config/config.go b/pkg/operator/ceph/cluster/osd/config/config.go index d32f5977d..4a75da798 100644 --- a/pkg/operator/ceph/cluster/osd/config/config.go +++ b/pkg/operator/ceph/cluster/osd/config/config.go @@ -33,7 +33,6 @@ func GetConfigStoreName(nodeName string) string { } const ( - StoreTypeKey = "storeType" WalSizeMBKey = "walSizeMB" DatabaseSizeMBKey = "databaseSizeMB" JournalSizeMBKey = "journalSizeMB" @@ -45,7 +44,6 @@ const ( // StoreConfig represents the configuration of an OSD on a device. type StoreConfig struct { - StoreType string `json:"storeType,omitempty"` WalSizeMB int `json:"walSizeMB,omitempty"` DatabaseSizeMB int `json:"databaseSizeMB,omitempty"` OSDsPerDevice int `json:"osdsPerDevice,omitempty"` @@ -66,8 +64,6 @@ func ToStoreConfig(config map[string]string) StoreConfig { storeConfig := NewStoreConfig() for k, v := range config { switch k { - case StoreTypeKey: - storeConfig.StoreType = v case WalSizeMBKey: storeConfig.WalSizeMB = convertToIntIgnoreErr(v) case DatabaseSizeMBKey: diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 7802ce899..fb3b59035 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -29,7 +29,6 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms" - "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" opconfig "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" @@ -325,7 +324,6 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC return nil, errors.New("empty volumes") } - storeType := config.Bluestore osdID := strconv.Itoa(osd.ID) tiniEnvVar := v1.EnvVar{Name: "TINI_SUBREAPER", Value: ""} envVars := append(c.getConfigEnvVars(osdProps, dataDir), []v1.EnvVar{ @@ -335,7 +333,6 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC envVars = append(envVars, []v1.EnvVar{ {Name: "ROOK_OSD_UUID", Value: osd.UUID}, {Name: "ROOK_OSD_ID", Value: osdID}, - {Name: "ROOK_OSD_STORE_TYPE", Value: storeType}, {Name: "ROOK_CEPH_MON_HOST", ValueFrom: &v1.EnvVarSource{ SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{ From 64e28af74195befb449af81fe234143db5b0d2b4 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Thu, 11 Mar 2021 14:36:09 -0700 Subject: [PATCH 2/6] ceph: allow flex driver and discovery to be enabled with configmap For testing purposes, we need to configure the flex driver and discovery daemon with the operator settings configmap. Signed-off-by: Travis Nielsen --- .../kubernetes/ceph/operator-openshift.yaml | 17 +++++++---------- cluster/examples/kubernetes/ceph/operator.yaml | 17 +++++++---------- cmd/rook/ceph/operator.go | 3 --- pkg/operator/ceph/cluster/controller.go | 5 ++--- .../ceph/controller/controller_utils.go | 12 ++++++++++++ pkg/operator/ceph/operator.go | 12 +++--------- 6 files changed, 31 insertions(+), 35 deletions(-) diff --git a/cluster/examples/kubernetes/ceph/operator-openshift.yaml b/cluster/examples/kubernetes/ceph/operator-openshift.yaml index 971c875fa..cbcaa0343 100644 --- a/cluster/examples/kubernetes/ceph/operator-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/operator-openshift.yaml @@ -350,6 +350,13 @@ data: # Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true" + # Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release + # in favor of the CSI driver. + ROOK_ENABLE_FLEX_DRIVER: "false" + # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster. + # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs. + ROOK_ENABLE_DISCOVERY_DAEMON: "false" + # (Optional) Admission controller NodeAffinity. # ADMISSION_CONTROLLER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" # (Optional) Admission controller tolerations list. Put here list of taints you want to tolerate in YAML format. @@ -475,16 +482,6 @@ spec: - name: DISCOVER_DAEMON_UDEV_BLACKLIST value: "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" - # Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release - # in favor of the CSI driver. - - name: ROOK_ENABLE_FLEX_DRIVER - value: "false" - - # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster. - # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs. - - name: ROOK_ENABLE_DISCOVERY_DAEMON - value: "false" - # Whether to start machineDisruptionBudget and machineLabel controller to watch for the osd pods and MDBs. - name: ROOK_ENABLE_MACHINE_DISRUPTION_BUDGET value: "false" diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index 959561825..de293c030 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -274,6 +274,13 @@ data: # Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true" + # Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release + # in favor of the CSI driver. + ROOK_ENABLE_FLEX_DRIVER: "false" + # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster. + # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs. + ROOK_ENABLE_DISCOVERY_DAEMON: "false" + # (Optional) Admission controller NodeAffinity. # ADMISSION_CONTROLLER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" # (Optional) Admission controller tolerations list. Put here list of taints you want to tolerate in YAML format. @@ -424,16 +431,6 @@ spec: - name: DISCOVER_DAEMON_UDEV_BLACKLIST value: "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" - # Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release - # in favor of the CSI driver. - - name: ROOK_ENABLE_FLEX_DRIVER - value: "false" - - # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster. - # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs. - - name: ROOK_ENABLE_DISCOVERY_DAEMON - value: "false" - # Time to wait until the node controller will move Rook pods to other # nodes after detecting an unreachable node. # Pods affected by this setting are: diff --git a/cmd/rook/ceph/operator.go b/cmd/rook/ceph/operator.go index f3fe312f8..8acffe25d 100644 --- a/cmd/rook/ceph/operator.go +++ b/cmd/rook/ceph/operator.go @@ -47,9 +47,6 @@ func init() { operatorCmd.Flags().DurationVar(&mon.HealthCheckInterval, "mon-healthcheck-interval", mon.HealthCheckInterval, "mon health check interval (duration)") operatorCmd.Flags().DurationVar(&mon.MonOutTimeout, "mon-out-timeout", mon.MonOutTimeout, "mon out timeout (duration)") - operatorCmd.Flags().BoolVar(&operator.EnableFlexDriver, "enable-flex-driver", true, "enable the rook flex driver") - operatorCmd.Flags().BoolVar(&operator.EnableDiscoveryDaemon, "enable-discovery-daemon", false, "enable the rook discovery daemon") - // csi deployment templates operatorCmd.Flags().StringVar(&csi.RBDPluginTemplatePath, "csi-rbd-plugin-template-path", csi.DefaultRBDPluginTemplatePath, "path to ceph-csi rbd plugin template") diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 0208c7f71..64c936899 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -31,6 +31,7 @@ import ( cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/daemon/ceph/osd/kms" "github.com/rook/rook/pkg/operator/ceph/cluster/osd" + controllerutil "github.com/rook/rook/pkg/operator/ceph/controller" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/csi" "github.com/rook/rook/pkg/operator/k8sutil" @@ -51,7 +52,6 @@ import ( const ( controllerName = "ceph-cluster-controller" - enableFlexDriver = "ROOK_ENABLE_FLEX_DRIVER" detectCephVersionTimeout = 15 * time.Minute ) @@ -421,8 +421,7 @@ func (c *ClusterController) checkIfVolumesExist(cluster *cephv1.CephCluster) err return err } } - flexDriverEnabled := os.Getenv(enableFlexDriver) != "false" - if !flexDriverEnabled { + if controllerutil.FlexDriverEnabled(c.context) { logger.Debugf("Flex driver disabled, skipping check for volume attachments for cluster %q", cluster.Namespace) return nil } diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index 611dce029..e03538354 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -59,6 +59,18 @@ var ( OperatorCephBaseImageVersion string ) +func FlexDriverEnabled(context *clusterd.Context) bool { + // Ignore the error. In the remote chance that the configmap fails to be read, we will default to disabling the flex driver + value, _ := k8sutil.GetOperatorSetting(context.Clientset, OperatorSettingConfigMapName, "ROOK_ENABLE_FLEX_DRIVER", "false") + return value == "true" +} + +func DiscoveryDaemonEnabled(context *clusterd.Context) bool { + // Ignore the error. In the remote chance that the configmap fails to be read, we will default to disabling the discovery daemon + value, _ := k8sutil.GetOperatorSetting(context.Clientset, OperatorSettingConfigMapName, "ROOK_ENABLE_DISCOVERY_DAEMON", "false") + return value == "true" +} + // CheckForCancelledOrchestration checks whether a cancellation has been requested func CheckForCancelledOrchestration(context *clusterd.Context) error { defer context.RequestCancelOrchestration.UnSet() diff --git a/pkg/operator/ceph/operator.go b/pkg/operator/ceph/operator.go index 92533d260..09b811d15 100644 --- a/pkg/operator/ceph/operator.go +++ b/pkg/operator/ceph/operator.go @@ -57,12 +57,6 @@ var provisionerConfigs = map[string]string{ } var ( - // EnableFlexDriver Whether to enable the flex driver. If true, the rook-ceph-agent daemonset will be started. - EnableFlexDriver = true - - // EnableDiscoveryDaemon Whether to enable the daemon for device discovery. If true, the rook-ceph-discover daemonset will be started. - EnableDiscoveryDaemon = true - // ImmediateRetryResult Return this for a immediate retry of the reconciliation loop with the same request object. ImmediateRetryResult = reconcile.Result{Requeue: true} ) @@ -119,7 +113,7 @@ func (o *Operator) Run() error { defer stopFunc() rookDiscover := discover.New(o.context.Clientset) - if EnableDiscoveryDaemon { + if opcontroller.DiscoveryDaemonEnabled(o.context) { if err := rookDiscover.Start(o.operatorNamespace, o.rookImage, o.securityAccount, true); err != nil { return errors.Wrap(err, "failed to start device discovery daemonset") } @@ -145,7 +139,7 @@ func (o *Operator) Run() error { signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) // For Flex Driver, run volume provisioner for each of the supported configurations - if EnableFlexDriver { + if opcontroller.FlexDriverEnabled(o.context) { for name, vendor := range provisionerConfigs { volumeProvisioner := provisioner.New(o.context, vendor) pc := controller.NewProvisionController( @@ -216,7 +210,7 @@ func (o *Operator) updateDrivers() error { return errors.Errorf("rook operator namespace is not provided. expose it via downward API in the rook operator manifest file using environment variable %s", k8sutil.PodNamespaceEnvVar) } - if EnableFlexDriver { + if opcontroller.FlexDriverEnabled(o.context) { rookAgent := agent.New(o.context.Clientset) if err := rookAgent.Start(o.operatorNamespace, o.rookImage, o.securityAccount); err != nil { return errors.Wrap(err, "error starting agent daemonset") From f50e49be3a95102b55136631fd0b8a18f83ac00b Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Fri, 12 Mar 2021 14:22:53 -0700 Subject: [PATCH 3/6] ceph: object store user initialization check for nil If the object store status is not yet initialized, the object store user should fail its initialization and requeue the reconcile. Signed-off-by: Travis Nielsen --- pkg/operator/ceph/object/user/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go index e570a9cf0..7f84d11ea 100644 --- a/pkg/operator/ceph/object/user/controller.go +++ b/pkg/operator/ceph/object/user/controller.go @@ -312,7 +312,7 @@ func (r *ReconcileObjectStoreUser) initializeObjectStoreContext(u *cephv1.CephOb r.objContext = objContext r.objContext.Endpoint = store.Status.Info["endpoint"] - if store.Status.Info["secureEndpoint"] != "" { + if store.Status != nil && store.Status.Info["secureEndpoint"] != "" { r.objContext.Endpoint = store.Status.Info["secureEndpoint"] } if r.objContext.Endpoint == "" { From c23238cddbf09394e53902f505bbff8279edf88f Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Tue, 9 Mar 2021 22:52:21 -0700 Subject: [PATCH 4/6] ceph: refactor integration tests for simplification The integration tests have long been painful to maintain with settings in various places and copied to multiple types, inconsistent variable names, and otherwise difficult to maintain code. Now the settings for a test suite are all in one place and they remain in the same settings type throughout the test. The multi-cluster suite is also refactored to use the same install and uninstall helpers as the other suites. Signed-off-by: Travis Nielsen --- pkg/daemon/ceph/client/command.go | 9 +- pkg/daemon/ceph/client/command_test.go | 8 +- pkg/daemon/ceph/client/mon_test.go | 4 +- pkg/operator/ceph/cluster/controller.go | 4 +- .../ceph/controller/controller_utils.go | 1 + tests/framework/clients/block.go | 26 +- tests/framework/clients/bucket.go | 8 +- tests/framework/clients/client.go | 4 +- tests/framework/clients/filesystem.go | 24 +- tests/framework/clients/nfs.go | 2 +- tests/framework/clients/object.go | 2 +- tests/framework/clients/object_user.go | 2 +- tests/framework/clients/pool.go | 4 +- tests/framework/clients/rbd-mirror.go | 2 +- .../installer/cassandra_installer.go | 2 +- tests/framework/installer/ceph_installer.go | 611 +++---- tests/framework/installer/ceph_manifests.go | 1606 ++--------------- .../installer/ceph_manifests_v1.5.go | 1517 +--------------- tests/framework/installer/ceph_settings.go | 126 ++ .../framework/installer/general_manifests.go | 110 ++ tests/framework/installer/installer.go | 5 - tests/framework/installer/nfs_installer.go | 2 +- tests/framework/utils/helm_helper.go | 2 +- tests/framework/utils/k8s_helper.go | 16 +- tests/integration/ceph_base_block_test.go | 24 +- tests/integration/ceph_base_deploy_test.go | 75 +- tests/integration/ceph_base_file_test.go | 122 +- tests/integration/ceph_base_object_test.go | 58 +- tests/integration/ceph_flex_test.go | 68 +- tests/integration/ceph_helm_test.go | 66 +- tests/integration/ceph_mgr_test.go | 119 +- tests/integration/ceph_multi_cluster_test.go | 254 +-- tests/integration/ceph_smoke_test.go | 190 +- tests/integration/ceph_upgrade_test.go | 81 +- tests/integration/nfs_test.go | 84 +- tests/scripts/deploy_admission_controller.sh | 2 +- 36 files changed, 1307 insertions(+), 3933 deletions(-) create mode 100644 tests/framework/installer/ceph_settings.go create mode 100644 tests/framework/installer/general_manifests.go diff --git a/pkg/daemon/ceph/client/command.go b/pkg/daemon/ceph/client/command.go index 32f68e256..9712a2658 100644 --- a/pkg/daemon/ceph/client/command.go +++ b/pkg/daemon/ceph/client/command.go @@ -25,9 +25,10 @@ import ( "github.com/rook/rook/pkg/clusterd" ) -// RunAllCephCommandsInToolbox - when running the e2e tests, all ceph commands need to be run in the toolbox. +// RunAllCephCommandsInToolboxPod - when running the e2e tests, all ceph commands need to be run in the toolbox. // Everywhere else, the ceph tools are assumed to be in the container where we can shell out. -var RunAllCephCommandsInToolbox = false +// This is the name of the pod. +var RunAllCephCommandsInToolboxPod string const ( // AdminUsername is the name of the admin user @@ -66,8 +67,8 @@ func FinalizeCephCommandArgs(command string, clusterInfo *ClusterInfo, args []st } // If the command should be run inside the toolbox pod, include the kubectl args to call the toolbox - if RunAllCephCommandsInToolbox { - toolArgs := []string{"exec", "-i", "rook-ceph-tools", "-n", clusterInfo.Namespace, "--", command} + if RunAllCephCommandsInToolboxPod != "" { + toolArgs := []string{"exec", "-i", RunAllCephCommandsInToolboxPod, "-n", clusterInfo.Namespace, "--", command} return Kubectl, append(toolArgs, args...) } diff --git a/pkg/daemon/ceph/client/command_test.go b/pkg/daemon/ceph/client/command_test.go index f6af57537..3f1ba01bc 100644 --- a/pkg/daemon/ceph/client/command_test.go +++ b/pkg/daemon/ceph/client/command_test.go @@ -23,7 +23,7 @@ import ( ) func TestFinalizeCephCommandArgs(t *testing.T) { - RunAllCephCommandsInToolbox = false + RunAllCephCommandsInToolboxPod = "" configDir := "/var/lib/rook/rook-ceph" expectedCommand := "ceph" args := []string{"quorum_status"} @@ -43,7 +43,7 @@ func TestFinalizeCephCommandArgs(t *testing.T) { } func TestFinalizeRadosGWAdminCommandArgs(t *testing.T) { - RunAllCephCommandsInToolbox = false + RunAllCephCommandsInToolboxPod = "" configDir := "/var/lib/rook/rook-ceph" expectedCommand := "radosgw-admin" args := []string{ @@ -73,7 +73,7 @@ func TestFinalizeRadosGWAdminCommandArgs(t *testing.T) { } func TestFinalizeCephCommandArgsToolBox(t *testing.T) { - RunAllCephCommandsInToolbox = true + RunAllCephCommandsInToolboxPod = "rook-ceph-tools" configDir := "/var/lib/rook/rook-ceph" expectedCommand := "ceph" args := []string{"health"} @@ -93,5 +93,5 @@ func TestFinalizeCephCommandArgsToolBox(t *testing.T) { cmd, args := FinalizeCephCommandArgs(expectedCommand, clusterInfo, args, configDir) assert.Exactly(t, "kubectl", cmd) assert.Exactly(t, expectedArgs, args) - RunAllCephCommandsInToolbox = false + RunAllCephCommandsInToolboxPod = "" } diff --git a/pkg/daemon/ceph/client/mon_test.go b/pkg/daemon/ceph/client/mon_test.go index 414cdc6d0..c283de320 100644 --- a/pkg/daemon/ceph/client/mon_test.go +++ b/pkg/daemon/ceph/client/mon_test.go @@ -38,7 +38,7 @@ func TestCephArgs(t *testing.T) { assert.Equal(t, "--name=client.admin", args[3]) assert.Equal(t, "--keyring=/etc/a/client.admin.keyring", args[4]) - RunAllCephCommandsInToolbox = true + RunAllCephCommandsInToolboxPod = "rook-ceph-tools" args = []string{} command, args = FinalizeCephCommandArgs(CephTool, clusterInfo, args, "/etc") assert.Equal(t, Kubectl, command) @@ -51,7 +51,7 @@ func TestCephArgs(t *testing.T) { assert.Equal(t, "--", args[5]) assert.Equal(t, CephTool, args[6]) assert.Equal(t, "--connect-timeout=15", args[7]) - RunAllCephCommandsInToolbox = false + RunAllCephCommandsInToolboxPod = "" // cluster under /var/lib/rook args = []string{"myarg"} diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 64c936899..3284b5e99 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -293,7 +293,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R // Remove finalizer err = removeFinalizer(r.client, request.NamespacedName) if err != nil { - return reconcile.Result{}, errors.Wrap(err, "failed to remove finalize") + return reconcile.Result{}, errors.Wrap(err, "failed to remove finalizer") } // Return and do not requeue. Successful deletion. @@ -421,7 +421,7 @@ func (c *ClusterController) checkIfVolumesExist(cluster *cephv1.CephCluster) err return err } } - if controllerutil.FlexDriverEnabled(c.context) { + if !controllerutil.FlexDriverEnabled(c.context) { logger.Debugf("Flex driver disabled, skipping check for volume attachments for cluster %q", cluster.Namespace) return nil } diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index e03538354..f3d6feb96 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -142,6 +142,7 @@ func IsReadyToReconcile(c client.Client, clustercontext *clusterd.Context, names } } + logger.Debugf("%q: CephCluster %q initial reconcile is not complete yet...", controllerName, namespacedName.Namespace) return cephCluster, false, cephClusterExists, WaitForRequeueIfCephClusterNotReady } diff --git a/tests/framework/clients/block.go b/tests/framework/clients/block.go index 43075df2f..c7f8d5cad 100644 --- a/tests/framework/clients/block.go +++ b/tests/framework/clients/block.go @@ -62,26 +62,26 @@ func (b *BlockOperation) Create(manifest string, size int) (string, error) { } -func (b *BlockOperation) CreateStorageClassAndPVC(csi bool, pvcNamespace, clusterNamespace, systemNamespace, poolName, storageClassName, reclaimPolicy, blockName, mode string) error { - if err := b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockPoolDef(poolName, clusterNamespace, "1")); err != nil { +func (b *BlockOperation) CreateStorageClassAndPVC(pvcNamespace, poolName, storageClassName, reclaimPolicy, blockName, mode string) error { + if err := b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockPool(poolName, "1")); err != nil { return err } - if err := b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockStorageClassDef(csi, poolName, storageClassName, reclaimPolicy, clusterNamespace, systemNamespace)); err != nil { + if err := b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockStorageClass(poolName, storageClassName, reclaimPolicy)); err != nil { return err } - return b.k8sClient.ResourceOperation("apply", b.manifests.GetPVC(blockName, pvcNamespace, storageClassName, mode, "1M")) + return b.k8sClient.ResourceOperation("apply", installer.GetPVC(blockName, pvcNamespace, storageClassName, mode, "1M")) } func (b *BlockOperation) CreatePVC(namespace, claimName, storageClassName, mode, size string) error { - return b.k8sClient.ResourceOperation("apply", b.manifests.GetPVC(claimName, namespace, storageClassName, mode, size)) + return b.k8sClient.ResourceOperation("apply", installer.GetPVC(claimName, namespace, storageClassName, mode, size)) } func (b *BlockOperation) CreatePod(podName, claimName, namespace, mountPoint string, readOnly bool) error { - return b.k8sClient.ResourceOperation("apply", b.manifests.GetPod(podName, claimName, namespace, mountPoint, readOnly)) + return b.k8sClient.ResourceOperation("apply", installer.GetPodWithVolume(podName, claimName, namespace, mountPoint, readOnly)) } func (b *BlockOperation) CreateStorageClass(csi bool, poolName, storageClassName, reclaimPolicy, namespace string) error { - return b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockStorageClassDef(csi, poolName, storageClassName, reclaimPolicy, namespace, installer.SystemNamespace(namespace))) + return b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockStorageClass(poolName, storageClassName, reclaimPolicy)) } func (b *BlockOperation) DeletePVC(namespace, claimName string) error { @@ -91,27 +91,27 @@ func (b *BlockOperation) DeletePVC(namespace, claimName string) error { } func (b *BlockOperation) CreatePVCRestore(namespace, claimName, snapshotName, storageClassName, mode, size string) error { - return b.k8sClient.ResourceOperation("apply", b.manifests.GetPVCRestore(claimName, snapshotName, namespace, storageClassName, mode, size)) + return b.k8sClient.ResourceOperation("apply", installer.GetPVCRestore(claimName, snapshotName, namespace, storageClassName, mode, size)) } func (b *BlockOperation) CreatePVCClone(namespace, cloneClaimName, parentClaimName, storageClassName, mode, size string) error { - return b.k8sClient.ResourceOperation("apply", b.manifests.GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, mode, size)) + return b.k8sClient.ResourceOperation("apply", installer.GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, mode, size)) } func (b *BlockOperation) CreateSnapshotClass(snapshotClassName, deletePolicy, namespace string) error { - return b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockSnapshotClass(snapshotClassName, namespace, namespace, deletePolicy)) + return b.k8sClient.ResourceOperation("apply", b.manifests.GetBlockSnapshotClass(snapshotClassName, deletePolicy)) } func (b *BlockOperation) DeleteSnapshotClass(snapshotClassName, deletePolicy, namespace string) error { - return b.k8sClient.ResourceOperation("delete", b.manifests.GetBlockSnapshotClass(snapshotClassName, namespace, namespace, deletePolicy)) + return b.k8sClient.ResourceOperation("delete", b.manifests.GetBlockSnapshotClass(snapshotClassName, deletePolicy)) } func (b *BlockOperation) CreateSnapshot(snapshotName, claimName, snapshotClassName, namespace string) error { - return b.k8sClient.ResourceOperation("apply", b.manifests.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace)) + return b.k8sClient.ResourceOperation("apply", installer.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace)) } func (b *BlockOperation) DeleteSnapshot(snapshotName, claimName, snapshotClassName, namespace string) error { - return b.k8sClient.ResourceOperation("delete", b.manifests.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace)) + return b.k8sClient.ResourceOperation("delete", installer.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace)) } func (b *BlockOperation) DeleteStorageClass(storageClassName string) error { diff --git a/tests/framework/clients/bucket.go b/tests/framework/clients/bucket.go index 4d62ac01f..70e73f8b5 100644 --- a/tests/framework/clients/bucket.go +++ b/tests/framework/clients/bucket.go @@ -37,20 +37,20 @@ func CreateBucketOperation(k8sh *utils.K8sHelper, manifests installer.CephManife } func (b *BucketOperation) CreateBucketStorageClass(namespace string, storeName string, storageClassName string, reclaimPolicy string, region string) error { - return b.k8sh.ResourceOperation("create", b.manifests.GetBucketStorageClass(namespace, storeName, storageClassName, reclaimPolicy, region)) + return b.k8sh.ResourceOperation("create", b.manifests.GetBucketStorageClass(storeName, storageClassName, reclaimPolicy, region)) } func (b *BucketOperation) DeleteBucketStorageClass(namespace string, storeName string, storageClassName string, reclaimPolicy string, region string) error { - err := b.k8sh.ResourceOperation("delete", b.manifests.GetBucketStorageClass(namespace, storeName, storageClassName, reclaimPolicy, region)) + err := b.k8sh.ResourceOperation("delete", b.manifests.GetBucketStorageClass(storeName, storageClassName, reclaimPolicy, region)) return err } func (b *BucketOperation) CreateObc(obcName string, storageClassName string, bucketName string, maxObject string, createBucket bool) error { - return b.k8sh.ResourceOperation("create", b.manifests.GetObc(obcName, storageClassName, bucketName, maxObject, createBucket)) + return b.k8sh.ResourceOperation("create", b.manifests.GetOBC(obcName, storageClassName, bucketName, maxObject, createBucket)) } func (b *BucketOperation) DeleteObc(obcName string, storageClassName string, bucketName string, maxObject string, createBucket bool) error { - return b.k8sh.ResourceOperation("delete", b.manifests.GetObc(obcName, storageClassName, bucketName, maxObject, createBucket)) + return b.k8sh.ResourceOperation("delete", b.manifests.GetOBC(obcName, storageClassName, bucketName, maxObject, createBucket)) } // CheckOBC, returns true if the obc, secret and configmap are all in the "check" state, diff --git a/tests/framework/clients/client.go b/tests/framework/clients/client.go index 19236b890..2b3834b23 100644 --- a/tests/framework/clients/client.go +++ b/tests/framework/clients/client.go @@ -42,7 +42,7 @@ func CreateClientOperation(k8sh *utils.K8sHelper, manifests installer.CephManife // Create creates a client in Rook func (c *ClientOperation) Create(name, namespace string, caps map[string]string) error { logger.Infof("creating the client via CRD") - if err := c.k8sh.ResourceOperation("apply", c.manifests.GetClient(namespace, name, caps)); err != nil { + if err := c.k8sh.ResourceOperation("apply", c.manifests.GetClient(name, caps)); err != nil { return err } return nil @@ -76,7 +76,7 @@ func (c *ClientOperation) Get(clusterInfo *client.ClusterInfo, clientName string func (c *ClientOperation) Update(clusterInfo *client.ClusterInfo, clientName string, caps map[string]string) (updatedcaps map[string]string, error error) { context := c.k8sh.MakeContext() logger.Infof("updating the client via CRD") - if err := c.k8sh.ResourceOperation("apply", c.manifests.GetClient(clusterInfo.Namespace, clientName, caps)); err != nil { + if err := c.k8sh.ResourceOperation("apply", c.manifests.GetClient(clientName, caps)); err != nil { return nil, err } diff --git a/tests/framework/clients/filesystem.go b/tests/framework/clients/filesystem.go index 3a442ec6a..049e06fdc 100644 --- a/tests/framework/clients/filesystem.go +++ b/tests/framework/clients/filesystem.go @@ -42,7 +42,7 @@ func CreateFilesystemOperation(k8sh *utils.K8sHelper, manifests installer.CephMa // Create creates a filesystem in Rook func (f *FilesystemOperation) Create(name, namespace string, activeCount int) error { logger.Infof("creating the filesystem via CRD") - if err := f.k8sh.ResourceOperation("apply", f.manifests.GetFilesystem(namespace, name, activeCount)); err != nil { + if err := f.k8sh.ResourceOperation("apply", f.manifests.GetFilesystem(name, activeCount)); err != nil { return err } @@ -58,32 +58,32 @@ func (f *FilesystemOperation) Create(name, namespace string, activeCount int) er // CreateStorageClass creates a storage class for CephFS clients func (f *FilesystemOperation) CreateStorageClass(fsName, systemNamespace, namespace, storageClassName string) error { - return f.k8sh.ResourceOperation("apply", f.manifests.GetFileStorageClassDef(fsName, storageClassName, systemNamespace, namespace)) + return f.k8sh.ResourceOperation("apply", f.manifests.GetFileStorageClass(fsName, storageClassName)) } // CreateSnapshotClass creates a snapshot class for CephFS clients func (f *FilesystemOperation) CreateSnapshotClass(snapshotClassName, reclaimPolicy, namespace string) error { - return f.k8sh.ResourceOperation("apply", f.manifests.GetFileStorageSnapshotClass(snapshotClassName, namespace, namespace, reclaimPolicy)) + return f.k8sh.ResourceOperation("apply", f.manifests.GetFileStorageSnapshotClass(snapshotClassName, reclaimPolicy)) } // CreatePVCRestore creates a pvc from snapshot func (f *FilesystemOperation) CreatePVCRestore(namespace, claimName, snapshotName, storageClassName, mode, size string) error { - return f.k8sh.ResourceOperation("apply", f.manifests.GetPVCRestore(claimName, snapshotName, namespace, storageClassName, mode, size)) + return f.k8sh.ResourceOperation("apply", installer.GetPVCRestore(claimName, snapshotName, namespace, storageClassName, mode, size)) } // CreatePVCClone creates a pvc from pvc func (f *FilesystemOperation) CreatePVCClone(namespace, cloneClaimName, parentClaimName, storageClassName, mode, size string) error { - return f.k8sh.ResourceOperation("apply", f.manifests.GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, mode, size)) + return f.k8sh.ResourceOperation("apply", installer.GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, mode, size)) } // CreateSnapshot creates a snapshot from pvc func (f *FilesystemOperation) CreateSnapshot(snapshotName, claimName, snapshotClassName, namespace string) error { - return f.k8sh.ResourceOperation("apply", f.manifests.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace)) + return f.k8sh.ResourceOperation("apply", installer.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace)) } // DeleteSnapshot deletes the snapshot func (f *FilesystemOperation) DeleteSnapshot(snapshotName, claimName, snapshotClassName, namespace string) error { - return f.k8sh.ResourceOperation("delete", f.manifests.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace)) + return f.k8sh.ResourceOperation("delete", installer.GetSnapshot(snapshotName, claimName, snapshotClassName, namespace)) } func (f *FilesystemOperation) DeletePVC(namespace, claimName string) error { @@ -104,21 +104,21 @@ func (f *FilesystemOperation) DeleteStorageClass(storageClassName string) error } func (f *FilesystemOperation) CreatePVC(namespace, claimName, storageClassName, mode, size string) error { - return f.k8sh.ResourceOperation("apply", f.manifests.GetPVC(claimName, namespace, storageClassName, mode, size)) + return f.k8sh.ResourceOperation("apply", installer.GetPVC(claimName, namespace, storageClassName, mode, size)) } func (f *FilesystemOperation) CreatePod(podName, claimName, namespace, mountPoint string, readOnly bool) error { - return f.k8sh.ResourceOperation("apply", f.manifests.GetPod(podName, claimName, namespace, mountPoint, readOnly)) + return f.k8sh.ResourceOperation("apply", installer.GetPodWithVolume(podName, claimName, namespace, mountPoint, readOnly)) } func (f *FilesystemOperation) DeleteSnapshotClass(snapshotClassName, deletePolicy, namespace string) error { - return f.k8sh.ResourceOperation("delete", f.manifests.GetFileStorageSnapshotClass(snapshotClassName, namespace, namespace, deletePolicy)) + return f.k8sh.ResourceOperation("delete", f.manifests.GetFileStorageSnapshotClass(snapshotClassName, deletePolicy)) } // ScaleDown scales down the number of active metadata servers of a filesystem in Rook func (f *FilesystemOperation) ScaleDown(name, namespace string) error { logger.Infof("scaling down the number of filesystem active metadata servers via CRD") - if err := f.k8sh.ResourceOperation("apply", f.manifests.GetFilesystem(namespace, name, 1)); err != nil { + if err := f.k8sh.ResourceOperation("apply", f.manifests.GetFilesystem(name, 1)); err != nil { return err } @@ -144,7 +144,7 @@ func (f *FilesystemOperation) Delete(name, namespace string) error { } logger.Infof("Deleted filesystem %s in namespace %s", name, namespace) - return f.k8sh.WaitForCustomResourceDeletion(namespace, crdCheckerFunc) + return f.k8sh.WaitForCustomResourceDeletion(namespace, name, crdCheckerFunc) } // List lists filesystems in Rook diff --git a/tests/framework/clients/nfs.go b/tests/framework/clients/nfs.go index eb3b60db8..c3b7ab2d8 100644 --- a/tests/framework/clients/nfs.go +++ b/tests/framework/clients/nfs.go @@ -42,7 +42,7 @@ func CreateNFSOperation(k8sh *utils.K8sHelper, manifests installer.CephManifests func (n *NFSOperation) Create(namespace, name, pool string, daemonCount int) error { logger.Infof("creating the NFS daemons via CRD") - if err := n.k8sh.ResourceOperation("apply", n.manifests.GetNFS(namespace, name, pool, daemonCount)); err != nil { + if err := n.k8sh.ResourceOperation("apply", n.manifests.GetNFS(name, pool, daemonCount)); err != nil { return err } diff --git a/tests/framework/clients/object.go b/tests/framework/clients/object.go index 27e506d6f..6fdbc7417 100644 --- a/tests/framework/clients/object.go +++ b/tests/framework/clients/object.go @@ -43,7 +43,7 @@ func CreateObjectOperation(k8sh *utils.K8sHelper, manifests installer.CephManife func (o *ObjectOperation) Create(namespace, storeName string, replicaCount int32) error { logger.Infof("creating the object store via CRD") - if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStore(namespace, storeName, int(replicaCount), rgwPort)); err != nil { + if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStore(storeName, int(replicaCount), rgwPort)); err != nil { return err } diff --git a/tests/framework/clients/object_user.go b/tests/framework/clients/object_user.go index c89a7beda..aee822228 100644 --- a/tests/framework/clients/object_user.go +++ b/tests/framework/clients/object_user.go @@ -68,7 +68,7 @@ func (o *ObjectUserOperation) UserSecretExists(namespace string, store string, u func (o *ObjectUserOperation) Create(namespace string, userid string, displayName string, store string) error { logger.Infof("creating the object store user via CRD") - if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStoreUser(namespace, userid, displayName, store)); err != nil { + if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStoreUser(userid, displayName, store)); err != nil { return err } return nil diff --git a/tests/framework/clients/pool.go b/tests/framework/clients/pool.go index 6bfed39bd..8571a4ad4 100644 --- a/tests/framework/clients/pool.go +++ b/tests/framework/clients/pool.go @@ -50,7 +50,7 @@ func (p *PoolOperation) Update(name, namespace string, replicas int) error { } func (p *PoolOperation) createOrUpdatePool(name, namespace, action string, replicas int) error { - return p.k8sh.ResourceOperation(action, p.manifests.GetBlockPoolDef(name, namespace, strconv.Itoa(replicas))) + return p.k8sh.ResourceOperation(action, p.manifests.GetBlockPool(name, strconv.Itoa(replicas))) } func (p *PoolOperation) ListCephPools(clusterInfo *client.ClusterInfo) ([]client.CephStoragePoolSummary, error) { @@ -147,5 +147,5 @@ func (p *PoolOperation) DeletePool(blockClient *BlockOperation, clusterInfo *cli return err } - return p.k8sh.WaitForCustomResourceDeletion(clusterInfo.Namespace, crdCheckerFunc) + return p.k8sh.WaitForCustomResourceDeletion(clusterInfo.Namespace, poolName, crdCheckerFunc) } diff --git a/tests/framework/clients/rbd-mirror.go b/tests/framework/clients/rbd-mirror.go index 2ae2aeedb..5e9d0bc2e 100644 --- a/tests/framework/clients/rbd-mirror.go +++ b/tests/framework/clients/rbd-mirror.go @@ -41,7 +41,7 @@ func CreateRBDMirrorOperation(k8sh *utils.K8sHelper, manifests installer.CephMan func (r *RBDMirrorOperation) Create(namespace, name string, daemonCount int) error { logger.Infof("creating the RBDMirror daemons via CRD") - if err := r.k8sh.ResourceOperation("apply", r.manifests.GetRBDMirror(namespace, name, daemonCount)); err != nil { + if err := r.k8sh.ResourceOperation("apply", r.manifests.GetRBDMirror(name, daemonCount)); err != nil { return err } diff --git a/tests/framework/installer/cassandra_installer.go b/tests/framework/installer/cassandra_installer.go index 4e0f6c858..a18216a2f 100644 --- a/tests/framework/installer/cassandra_installer.go +++ b/tests/framework/installer/cassandra_installer.go @@ -131,7 +131,7 @@ func (ci *CassandraInstaller) DeleteCassandraCluster(namespace string) { _, err := ci.k8sHelper.RookClientset.CassandraV1alpha1().Clusters(namespace).Get(ctx, namespace, metav1.GetOptions{}) return err } - err = ci.k8sHelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc) + err = ci.k8sHelper.WaitForCustomResourceDeletion(namespace, namespace, crdCheckerFunc) assert.NoError(ci.T(), err) // Delete Namespace diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index cbd786029..1e65cc2a2 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -28,6 +28,7 @@ import ( "time" "github.com/google/uuid" + "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/cluster" @@ -59,33 +60,25 @@ var ( // CephInstaller wraps installing and uninstalling rook on a platform type CephInstaller struct { + settings *TestCephSettings Manifests CephManifests k8shelper *utils.K8sHelper hostPathToDelete string helmHelper *utils.HelmHelper - useHelm bool - clusterName string k8sVersion string changeHostnames bool - CephVersion cephv1.CephVersionSpec T func() *testing.T - cleanupHost bool } // CreateCephOperator creates rook-operator via kubectl -func (h *CephInstaller) CreateCephOperator(namespace string) (err error) { +func (h *CephInstaller) CreateCephOperator() (err error) { logger.Infof("Starting Rook Operator") // creating clusterrolebinding for kubeadm env. h.k8shelper.CreateAnonSystemClusterBinding() - var v1ExtensionsSupported bool - minVersion := "v1.16.0" - if h.k8shelper.VersionAtLeast(minVersion) { - v1ExtensionsSupported = true - } // creating rook resources logger.Info("Creating Rook CRDs") - resources := h.Manifests.GetRookCRDs(v1ExtensionsSupported) + resources := h.Manifests.GetCRDs(h.k8shelper) if _, err = h.k8shelper.KubectlWithStdin(resources, createFromStdinArgs...); err != nil { return err } @@ -96,47 +89,46 @@ func (h *CephInstaller) CreateCephOperator(namespace string) (err error) { assert.NoError(h.T(), err) } - err = h.k8shelper.CreateNamespace(namespace) - if err != nil { - if kerrors.IsAlreadyExists(err) { - logger.Warningf("Namespace %q already exists!!!", namespace) - } else { - return fmt.Errorf("failed to create namespace %q. %v", namespace, err) - } + // The operator namespace needs to be created explicitly, while the cluster namespace is created with the common.yaml + if err := h.k8shelper.CreateNamespace(h.settings.OperatorNamespace); err != nil { + return err } - // disable admission controller for Kubernetes version older than v1.16.0 and for upgrade test as api version v1 require minimum v0.7 controller runtime and upgrade test still using - // older version of controller runtime. - if !utils.IsPlatformOpenShift() && namespace != "upgrade-ns-system" { - err = h.startAdmissionController(namespace) - if err != nil { - return fmt.Errorf("Failed to start admission controllers: %v", err) - } - } - - rookOperator := h.Manifests.GetRookOperator(namespace) - _, err = h.k8shelper.KubectlWithStdin(rookOperator, createFromStdinArgs...) + // Create the namespace and RBAC before starting the operator + _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetCommon(), createFromStdinArgs...) if err != nil { return fmt.Errorf("Failed to create rook-operator pod: %v ", err) } - logger.Infof("Rook Operator started") + err = h.startAdmissionController() + if err != nil { + return fmt.Errorf("Failed to start admission controllers: %v", err) + } + _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetOperator(), createFromStdinArgs...) + if err != nil { + return fmt.Errorf("Failed to create rook-operator pod: %v", err) + } + + logger.Infof("Rook operator started") return nil } -func (h *CephInstaller) startAdmissionController(namespace string) error { - err := h.k8shelper.CreateNamespace(namespace) - if err != nil { - if kerrors.IsAlreadyExists(err) { - logger.Warningf("Namespace %q already exists!!!", namespace) - } else { - return fmt.Errorf("failed to create namespace %q. %v", namespace, err) - } - } +func (h *CephInstaller) startAdmissionController() error { if !h.k8shelper.VersionAtLeast("v1.16.0") { + logger.Info("skipping the admission controller on K8s version older than v1.16") return nil } + if !h.settings.EnableAdmissionController { + logger.Info("skipping admission controller for this test suite") + return nil + } + // disable admission controller for Kubernetes version older than v1.16.0 + if utils.IsPlatformOpenShift() { + logger.Info("skipping the admission controller on OpenShift") + return nil + } + rootPath, err := utils.FindRookRoot() if err != nil { return fmt.Errorf("failed to find rook root. %v", err) @@ -146,7 +138,7 @@ func (h *CephInstaller) startAdmissionController(namespace string) error { return fmt.Errorf("failed to find user home directory. %v", err) } scriptPath := path.Join(rootPath, "tests/scripts/deploy_admission_controller_test.sh") - err = h.k8shelper.MakeContext().Executor.ExecuteCommandWithEnv([]string{fmt.Sprintf("NAMESPACE=%s", namespace), fmt.Sprintf("HOME=%s", userHome)}, "bash", scriptPath) + err = h.k8shelper.MakeContext().Executor.ExecuteCommandWithEnv([]string{fmt.Sprintf("NAMESPACE=%s", h.settings.OperatorNamespace), fmt.Sprintf("HOME=%s", userHome)}, "bash", scriptPath) if err != nil { return err } @@ -155,41 +147,51 @@ func (h *CephInstaller) startAdmissionController(namespace string) error { } // CreateRookOperatorViaHelm creates rook operator via Helm chart named local/rook present in local repo -func (h *CephInstaller) CreateRookOperatorViaHelm(namespace, chartSettings string) error { +func (h *CephInstaller) CreateRookOperatorViaHelm(chartSettings string) error { // creating clusterrolebinding for kubeadm env. h.k8shelper.CreateAnonSystemClusterBinding() - // disable admission controller for Kubernetes version older than v1.16.0 - if !utils.IsPlatformOpenShift() { - if err := h.startAdmissionController(namespace); err != nil { - return fmt.Errorf("Failed to start admission controllers: %v", err) - } + // create the operator namespace before the admission controller is created + if err := h.k8shelper.CreateNamespace(h.settings.OperatorNamespace); err != nil { + return fmt.Errorf("failed to create namespace %s. %v", h.settings.Namespace, err) } - - if err := h.helmHelper.InstallLocalRookHelmChart(namespace, chartSettings); err != nil { + if err := h.startAdmissionController(); err != nil { + return fmt.Errorf("failed to start admission controllers: %v", err) + } + if err := h.helmHelper.InstallLocalRookHelmChart(h.settings.Namespace, chartSettings); err != nil { return fmt.Errorf("failed to install rook operator via helm, err : %v", err) } + // create the cluster namespace since it wasn't created by common.yaml + if err := h.k8shelper.CreateNamespace(h.settings.Namespace); err != nil { + return fmt.Errorf("failed to create namespace %s. %v", h.settings.Namespace, err) + } return nil } // CreateRookToolbox creates rook-ceph-tools via kubectl -func (h *CephInstaller) CreateRookToolbox(namespace string) (err error) { +func (h *CephInstaller) CreateRookToolbox(manifests CephManifests) (err error) { logger.Infof("Starting Rook toolbox") - rookToolbox := h.Manifests.GetRookToolBox(namespace) - - _, err = h.k8shelper.KubectlWithStdin(rookToolbox, createFromStdinArgs...) - + _, err = h.k8shelper.KubectlWithStdin(manifests.GetToolbox(), createFromStdinArgs...) if err != nil { - return fmt.Errorf("Failed to create rook-toolbox pod : %v ", err) + return errors.Wrap(err, "failed to create rook-toolbox pod") } - if !h.k8shelper.IsPodRunning("rook-ceph-tools", namespace) { - return fmt.Errorf("Rook Toolbox couldn't start") + if err := h.k8shelper.WaitForLabeledPodsToRun("app=rook-ceph-tools", h.settings.Namespace); err != nil { + return errors.Wrap(err, "Rook Toolbox couldn't start") } logger.Infof("Rook Toolbox started") + podNames, err := h.k8shelper.GetPodNamesForApp("rook-ceph-tools", manifests.Settings().Namespace) + assert.NoError(h.T(), err) + for _, podName := range podNames { + // All e2e tests should run ceph commands in the toolbox since we are not inside a container + logger.Infof("setting active toolbox pod to %q", podName) + client.RunAllCephCommandsInToolboxPod = podName + break + } + return nil } @@ -205,24 +207,16 @@ func (h *CephInstaller) Execute(command string, parameters []string, namespace s return nil, result } -// CreateRookCluster creates rook cluster via kubectl -func (h *CephInstaller) CreateRookCluster(namespace, systemNamespace, storeType string, usePVC bool, storageClassName string, - mon cephv1.MonSpec, startWithAllNodes, multipleMgrs, skipOSDCreation bool, useCrashPruner bool, cephVersion cephv1.CephVersionSpec) error { +// CreateCephCluster creates rook cluster via kubectl +func (h *CephInstaller) CreateCephCluster() error { ctx := context.TODO() - dataDirHostPath, err := h.initTestDir(namespace) + var err error + h.settings.DataDirHostPath, err = h.initTestDir(h.settings.Namespace) if err != nil { return fmt.Errorf("failed to create test dir. %+v", err) } - logger.Infof("Creating cluster: namespace=%s, systemNamespace=%s, storeType=%s, dataDirHostPath=%s, usePVC=%v, storageClassName=%s, startWithAllNodes=%t, mons=%+v", - namespace, systemNamespace, storeType, dataDirHostPath, usePVC, storageClassName, startWithAllNodes, mon) - - logger.Infof("Creating namespace %s", namespace) - ns := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}} - _, err = h.k8shelper.Clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) - if err != nil && !kerrors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create namespace %s. %+v", namespace, err) - } + logger.Infof("Creating cluster with settings: %+v", h.settings) logger.Infof("Creating custom ceph.conf settings") customSettings := map[string]string{ @@ -233,63 +227,53 @@ osd_pool_default_size = 1 customCM := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "rook-config-override", - Namespace: namespace, + Namespace: h.settings.Namespace, }, Data: customSettings, } - if _, err := h.k8shelper.Clientset.CoreV1().ConfigMaps(namespace).Create(ctx, customCM, metav1.CreateOptions{}); err != nil { + if _, err := h.k8shelper.Clientset.CoreV1().ConfigMaps(h.settings.Namespace).Create(ctx, customCM, metav1.CreateOptions{}); err != nil { return fmt.Errorf("failed to create custom ceph.conf. %+v", err) } - // Skip this step since the helm chart already includes the roles and bindings - if !h.useHelm { - logger.Infof("Creating cluster roles") - roles := h.Manifests.GetClusterRoles(namespace, systemNamespace) - if _, err := h.k8shelper.KubectlWithStdin(roles, createFromStdinArgs...); err != nil { - return fmt.Errorf("Failed to create cluster roles. %+v", err) - } - } - logger.Info("Starting Rook Cluster with yaml") - settings := &clusterSettings{h.clusterName, namespace, storeType, dataDirHostPath, mon.Count, multipleMgrs, 0, usePVC, storageClassName, skipOSDCreation, cephVersion, useCrashPruner} - rookCluster := h.Manifests.GetRookCluster(settings) + rookCluster := h.Manifests.GetCephCluster() logger.Info(rookCluster) - maxTry := 3 + maxTry := 10 for i := 0; i < maxTry; i++ { _, err := h.k8shelper.KubectlWithStdin(rookCluster, createFromStdinArgs...) if err == nil { break } if i == maxTry-1 { - return fmt.Errorf("failed to create rook cluster. %v ", err) + return fmt.Errorf("failed to create rook cluster. %v", err) } logger.Infof("failed to create rook cluster, trying again... %v", err) time.Sleep(5 * time.Second) } - if err := h.k8shelper.WaitForPodCount("app=rook-ceph-mon", namespace, mon.Count); err != nil { + if err := h.k8shelper.WaitForPodCount("app=rook-ceph-mon", h.settings.Namespace, h.settings.Mons); err != nil { return err } - if err := h.k8shelper.WaitForPodCount("app=rook-ceph-mgr", namespace, 1); err != nil { + if err := h.k8shelper.WaitForPodCount("app=rook-ceph-mgr", h.settings.Namespace, 1); err != nil { return err } - if !skipOSDCreation { - if err := h.k8shelper.WaitForPodCount("app=rook-ceph-osd", namespace, 1); err != nil { + if !h.settings.SkipOSDCreation { + if err := h.k8shelper.WaitForPodCount("app=rook-ceph-osd", h.settings.Namespace, 1); err != nil { return err } } - if useCrashPruner { - if err := h.k8shelper.WaitForCronJob("rook-ceph-crashcollector-pruner", namespace); err != nil { + if h.settings.UseCrashPruner { + if err := h.k8shelper.WaitForCronJob("rook-ceph-crashcollector-pruner", h.settings.Namespace); err != nil { return err } } logger.Infof("Rook Cluster started") - if !skipOSDCreation { - err = h.k8shelper.WaitForLabeledPodsToRun("app=rook-ceph-osd", namespace) + if !h.settings.SkipOSDCreation { + err = h.k8shelper.WaitForLabeledPodsToRun("app=rook-ceph-osd", h.settings.Namespace) return err } @@ -297,43 +281,39 @@ osd_pool_default_size = 1 } // CreateRookExternalCluster creates rook external cluster via kubectl -func (h *CephInstaller) CreateRookExternalCluster(namespace, firstClusterNamespace string) error { - ctx := context.TODO() - dataDirHostPath, err := h.initTestDir(namespace) +func (h *CephInstaller) CreateRookExternalCluster(externalManifests CephManifests) error { + var err error + externalSettings := externalManifests.Settings() + externalSettings.DataDirHostPath, err = h.initTestDir(externalSettings.Namespace) if err != nil { return fmt.Errorf("failed to create test dir. %+v", err) } - logger.Infof("Creating external cluster: namespace=%q, firstClusterNamespace=%q", namespace, firstClusterNamespace) - logger.Infof("Creating namespace %s", namespace) - ns := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}} - _, err = h.k8shelper.Clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) - if err != nil && !kerrors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create namespace %q. %v", namespace, err) - } + logger.Infof("Creating external cluster %q with core storage namespace %q", externalSettings.Namespace, h.settings.Namespace) - // Skip this step since the helm chart already includes the roles and bindings - if !h.useHelm { - logger.Infof("Creating external cluster roles") - roles := h.Manifests.GetClusterExternalRoles(namespace, firstClusterNamespace) - if _, err := h.k8shelper.KubectlWithStdin(roles, createFromStdinArgs...); err != nil { - return fmt.Errorf("failed to create cluster roles. %v", err) - } + logger.Infof("Creating external cluster roles") + roles := externalManifests.GetCommonExternal() + if _, err := h.k8shelper.KubectlWithStdin(roles, createFromStdinArgs...); err != nil { + return errors.Wrap(err, "failed to create cluster roles") } // Inject connection information from the first cluster logger.Info("Injecting cluster connection information") - err = h.InjectRookExternalClusterInfo(namespace, firstClusterNamespace) + err = h.injectRookExternalClusterInfo(externalSettings) if err != nil { - return fmt.Errorf("failed to inject cluster information into the external cluster. %v", err) + return errors.Wrap(err, "failed to inject cluster information into the external cluster") } // Start the external cluster logger.Infof("Starting Rook External Cluster with yaml") - settings := &clusterExternalSettings{namespace, dataDirHostPath} - rookCluster := h.Manifests.GetRookExternalCluster(settings) + rookCluster := externalManifests.GetExternalCephCluster() if _, err := h.k8shelper.KubectlWithStdin(rookCluster, createFromStdinArgs...); err != nil { - return fmt.Errorf("failed to create rook external cluster. %v ", err) + return errors.Wrap(err, "failed to create rook external cluster") + } + + logger.Infof("Running toolbox on external namespace %q", externalSettings.Namespace) + if err := h.CreateRookToolbox(externalManifests); err != nil { + return errors.Wrap(err, "failed to start toolbox on external cluster") } logger.Info("Rook external cluster started") @@ -341,28 +321,28 @@ func (h *CephInstaller) CreateRookExternalCluster(namespace, firstClusterNamespa } // InjectRookExternalClusterInfo inject connection information for an external cluster -func (h *CephInstaller) InjectRookExternalClusterInfo(namespace, firstClusterNamespace string) error { +func (h *CephInstaller) injectRookExternalClusterInfo(externalSettings *TestCephSettings) error { ctx := context.TODO() // get config map - cm, err := h.GetRookExternalClusterMonConfigMap(firstClusterNamespace) + cm, err := h.GetRookExternalClusterMonConfigMap() if err != nil { return fmt.Errorf("failed to get configmap. %v", err) } // create config map - _, err = h.k8shelper.Clientset.CoreV1().ConfigMaps(namespace).Create(ctx, cm, metav1.CreateOptions{}) + _, err = h.k8shelper.Clientset.CoreV1().ConfigMaps(externalSettings.Namespace).Create(ctx, cm, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("failed to create configmap. %v", err) } // get secret - secret, err := h.GetRookExternalClusterMonSecret(firstClusterNamespace) + secret, err := h.GetRookExternalClusterMonSecret() if err != nil { return fmt.Errorf("failed to get secret. %v", err) } // create secret - _, err = h.k8shelper.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + _, err = h.k8shelper.Clientset.CoreV1().Secrets(externalSettings.Namespace).Create(ctx, secret, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("failed to create secret. %v", err) } @@ -371,10 +351,10 @@ func (h *CephInstaller) InjectRookExternalClusterInfo(namespace, firstClusterNam } // GetRookExternalClusterMonConfigMap gets the monitor kubernetes configmap of the external cluster -func (h *CephInstaller) GetRookExternalClusterMonConfigMap(namespace string) (*v1.ConfigMap, error) { +func (h *CephInstaller) GetRookExternalClusterMonConfigMap() (*v1.ConfigMap, error) { ctx := context.TODO() configMapName := "rook-ceph-mon-endpoints" - externalCM, err := h.k8shelper.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, configMapName, metav1.GetOptions{}) + externalCM, err := h.k8shelper.Clientset.CoreV1().ConfigMaps(h.settings.Namespace).Get(ctx, configMapName, metav1.GetOptions{}) if err != nil { return nil, fmt.Errorf("failed to get secret. %v", err) } @@ -386,11 +366,11 @@ func (h *CephInstaller) GetRookExternalClusterMonConfigMap(namespace string) (*v } // GetRookExternalClusterMonSecret gets the monitor kubernetes secret of the external cluster -func (h *CephInstaller) GetRookExternalClusterMonSecret(namespace string) (*v1.Secret, error) { +func (h *CephInstaller) GetRookExternalClusterMonSecret() (*v1.Secret, error) { ctx := context.TODO() secretName := "rook-ceph-mon" //nolint:gosec // We safely suppress gosec in tests file - externalSecret, err := h.k8shelper.Clientset.CoreV1().Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{}) + externalSecret, err := h.k8shelper.Clientset.CoreV1().Secrets(h.settings.Namespace).Get(ctx, secretName, metav1.GetOptions{}) if err != nil { return nil, fmt.Errorf("failed to get secret. %v", err) } @@ -444,57 +424,53 @@ func (h *CephInstaller) GetNodeHostnames() ([]string, error) { return names, nil } -// InstallRook installs rook on k8s -func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, storageClassName string, - mon cephv1.MonSpec, startWithAllNodes bool, rbdMirrorWorkers int, skipOSDCreation bool, rookVersion string) (bool, error) { +func (h *CephInstaller) InstallRook() (bool, error) { + + if h.settings.RookVersion != VersionMaster { + // make sure we have the images from a previous release locally so the test doesn't hit a timeout + assert.NoError(h.T(), h.k8shelper.GetDockerImage("rook/ceph:"+h.settings.RookVersion)) + } + + assert.NoError(h.T(), h.k8shelper.GetDockerImage(h.settings.CephVersion.Image)) ctx := context.TODO() var err error k8sversion := h.k8shelper.GetK8sServerVersion() - logger.Infof("Installing rook on k8s %s", k8sversion) + logger.Infof("Installing rook on K8s %s", k8sversion) - startDiscovery := true - onamespace := namespace + startDiscovery := false // Create rook operator - if h.useHelm { + if h.settings.UseHelm { // disable the discovery daemonset with the helm chart - settings := "enableDiscoveryDaemon=false,image.tag=master" - startDiscovery = false - err = h.CreateRookOperatorViaHelm(namespace, settings) + startDiscovery = true + err = h.CreateRookOperatorViaHelm("enableDiscoveryDaemon=true,image.tag=master") if err != nil { - logger.Errorf("Rook Operator not installed ,error -> %v", err) + logger.Errorf("helm install failed. %v", err) return false, err - } } else { - onamespace = SystemNamespace(namespace) - err := h.CreateCephOperator(onamespace) + err := h.CreateCephOperator() if err != nil { - logger.Errorf("Rook Operator not installed ,error -> %v", err) + logger.Errorf("operator install failed. %v", err) return false, err } } - if !h.k8shelper.IsPodInExpectedState("rook-ceph-operator", onamespace, "Running") { + if !h.k8shelper.IsPodInExpectedState("rook-ceph-operator", h.settings.OperatorNamespace, "Running") { logger.Error("rook-ceph-operator is not running") - h.k8shelper.GetLogsFromNamespace(onamespace, "test-setup", utils.TestEnvName()) + h.k8shelper.GetLogsFromNamespace(h.settings.OperatorNamespace, "test-setup", utils.TestEnvName()) logger.Error("rook-ceph-operator is not Running, abort!") return false, err } - useCrashPruner := rookVersion == VersionMaster - // Create rook cluster - multipleManagers := false - err = h.CreateRookCluster(namespace, onamespace, storeType, usePVC, storageClassName, - cephv1.MonSpec{Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode}, startWithAllNodes, - multipleManagers, skipOSDCreation, useCrashPruner, h.CephVersion) + err = h.CreateCephCluster() if err != nil { - logger.Errorf("Rook cluster %s not installed, error -> %v", namespace, err) + logger.Errorf("Cluster %q install failed. %v", h.settings.Namespace, err) return false, err } - discovery, err := h.k8shelper.Clientset.AppsV1().DaemonSets(onamespace).Get(ctx, "rook-discover", metav1.GetOptions{}) + discovery, err := h.k8shelper.Clientset.AppsV1().DaemonSets(h.settings.OperatorNamespace).Get(ctx, "rook-discover", metav1.GetOptions{}) if startDiscovery { assert.NoError(h.T(), err) assert.NotNil(h.T(), discovery) @@ -504,17 +480,16 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st } // Create rook client - err = h.CreateRookToolbox(namespace) + err = h.CreateRookToolbox(h.Manifests) if err != nil { - logger.Errorf("Rook toolbox in cluster %s not installed, error -> %v", namespace, err) + logger.Errorf("Rook toolbox in cluster %s not installed. %v", h.settings.Namespace, err) return false, err } - logger.Infof("installed rook operator and cluster : %s on k8s %s", namespace, h.k8sVersion) + logger.Infof("installed rook operator and cluster %s on k8s %s", h.settings.Namespace, h.k8sVersion) - // disable admission controller test for Kubernetes version older than v1.16.0 and for upgrade test as api version v1 require minimum v0.7 controller runtime and upgrade test still using - // older version of controller runtime. - if !utils.IsPlatformOpenShift() && h.k8shelper.VersionAtLeast("v1.16.0") && namespace != "upgrade-ns" { - if !h.k8shelper.IsPodInExpectedState("rook-ceph-admission-controller", onamespace, "Running") { + // disable admission controller test for Kubernetes version older than v1.16.0 + if h.settings.EnableAdmissionController && !utils.IsPlatformOpenShift() && h.k8shelper.VersionAtLeast("v1.16.0") { + if !h.k8shelper.IsPodInExpectedState("rook-ceph-admission-controller", h.settings.OperatorNamespace, "Running") { assert.Fail(h.T(), "admission controller is not running") } } @@ -523,34 +498,41 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st } // UninstallRook uninstalls rook from k8s -func (h *CephInstaller) UninstallRook(namespace string) { - h.UninstallRookFromMultipleNS(SystemNamespace(namespace), namespace) +func (h *CephInstaller) UninstallRook() { + h.UninstallRookFromMultipleNS(h.Manifests) } // UninstallRookFromMultipleNS uninstalls rook from multiple namespaces in k8s -func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, namespaces ...string) { +func (h *CephInstaller) UninstallRookFromMultipleNS(manifests ...CephManifests) { ctx := context.TODO() + var clusterNamespaces []string + for _, manifest := range manifests { + clusterNamespaces = append(clusterNamespaces, manifest.Settings().Namespace) + } + // Gather logs after status checks - h.GatherAllRookLogs(h.T().Name(), append([]string{systemNamespace}, namespaces...)...) + h.GatherAllRookLogs(h.T().Name(), append([]string{h.settings.OperatorNamespace}, clusterNamespaces...)...) logger.Infof("Uninstalling Rook") var err error - for clusterNum, namespace := range namespaces { - if h.cleanupHost { - //Add cleanup policy to the ceph cluster - // Add cleanup policy to the ceph cluster but NOT the external one - if clusterNum == 0 { - err = h.addCleanupPolicy(namespace) - assert.NoError(h.T(), err) - } - } + for _, manifest := range manifests { + namespace := manifest.Settings().Namespace + clusterName := manifest.Settings().ClusterName - if !h.T().Failed() { - // Only check the Ceph status for the first cluster - // The second cluster is external so the check won't work since the first cluster is gone - if clusterNum == 0 { - // if the test passed, check that the ceph status is HEALTH_OK before we tear the cluster down - h.checkCephHealthStatus(namespace) + if !manifest.Settings().IsExternal { + // Add cleanup policy to the core ceph cluster + err = h.addCleanupPolicy(namespace) + if err != nil { + assert.NoError(h.T(), err) + // no need to check for cleanup policy later if it already failed + testCleanupPolicy = false + } + + // if the test passed, check that the ceph status is HEALTH_OK before we tear the cluster down + if !h.T().Failed() { + // Only check the Ceph status for the core cluster + // The check won't work for an external cluster since the core cluster is already gone + h.checkCephHealthStatus() } } @@ -561,155 +543,114 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, name logger.Infof("found pools: %v", pools) assert.Fail(h.T(), fmt.Sprintf("pool %q still exists", pool.Name)) // Get the operator log - h.GatherAllRookLogs(h.T().Name()+"poolcheck", systemNamespace) + h.GatherAllRookLogs(h.T().Name()+"poolcheck", h.settings.OperatorNamespace) } - err = h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cephcluster", h.clusterName) + err = h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cephcluster", clusterName) checkError(h.T(), err, fmt.Sprintf("cannot remove cluster %s", namespace)) - if h.cleanupHost { - // The second cluster is external so the cleanup pod will never exist! - if clusterNum == 0 { - err = h.waitForCleanupJobs(namespace) + clusterDeleteRetries := 0 + crdCheckerFunc := func() error { + _, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(ctx, clusterName, metav1.GetOptions{}) + clusterDeleteRetries++ + if clusterDeleteRetries > 10 { + // If the operator really isn't going to remove the finalizer, just force remove it + h.removeClusterFinalizers(namespace) + } + return err + } + err = h.k8shelper.WaitForCustomResourceDeletion(namespace, clusterName, crdCheckerFunc) + checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace)) + + if !manifest.Settings().IsExternal { + err = h.waitForCleanupJobs(namespace) + if err != nil { assert.NoError(h.T(), err) + h.GatherAllRookLogs(h.T().Name()+"cleanup-job", append([]string{h.settings.OperatorNamespace}, clusterNamespaces...)...) } } - roles := h.Manifests.GetClusterRoles(namespace, systemNamespace) - _, err = h.k8shelper.KubectlWithStdin(roles, deleteFromStdinArgs...) - if err != nil { - logger.Errorf("failed to delete cluster roles. %v ", err) - } - - crdCheckerFunc := func() error { - _, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(ctx, h.clusterName, metav1.GetOptions{}) - // ensure the finalizer(s) are removed - h.removeClusterFinalizers(namespace) - return err - } - err = h.k8shelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc) - checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace)) - - if h.useHelm { + // helm cleanup + if h.settings.UseHelm { err = h.helmHelper.DeleteLocalRookHelmChart(namespace, utils.HelmDeployName) checkError(h.T(), err, "cannot uninstall helm chart") + + // delete the entire namespace (in non-helm installs it's removed with the common.yaml) + err = h.k8shelper.DeleteResourceAndWait(false, "namespace", namespace) + checkError(h.T(), err, fmt.Sprintf("cannot delete namespace %s", namespace)) + continue } - // delete the entire namespace - err = h.k8shelper.DeleteResourceAndWait(false, "namespace", namespace) - checkError(h.T(), err, fmt.Sprintf("cannot delete namespace %s", namespace)) + // non-helm cleanup + if manifest.Settings().IsExternal { + logger.Infof("Deleting all the resources in the common external manifest") + _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetCommonExternal(), deleteFromStdinArgs...) + if err != nil { + logger.Warningf("failed to remove common external resources. %v", err) + } else { + logger.Infof("done deleting all the resources in the common external manifest") + } + } else { + h.k8shelper.PrintResources(namespace, "cephblockpools.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephclients.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephclusters.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephfilesystemmirrors.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephfilesystems.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephnfses.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephobjectrealms.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephobjectstores.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephobjectstoreusers.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephobjectzonegroups.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephobjectzones.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "cephrbdmirrors.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "objectbucketclaims.ceph.rook.io") + h.k8shelper.PrintResources(namespace, "objectbuckets.ceph.rook.io") + h.k8shelper.PrintPodStatus(namespace) + h.k8shelper.PrintPVs(true) + logger.Infof("Deleting all the resources in the common manifest") + _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetCommon(), deleteFromStdinArgs...) + if err != nil { + logger.Warningf("failed to remove common manifest. %v", err) + } else { + logger.Infof("done deleting all the resources in the common manifest") + } + } } - err = h.k8shelper.DeleteResourceAndWait(false, "namespace", systemNamespace) - checkError(h.T(), err, fmt.Sprintf("cannot delete system namespace %s", systemNamespace)) - - logger.Infof("removing the operator from namespace %s", systemNamespace) - err = h.k8shelper.DeleteResource( - "crd", - "cephclusters.ceph.rook.io", - "cephblockpools.ceph.rook.io", - "cephobjectstores.ceph.rook.io", - "cephobjectstoreusers.ceph.rook.io", - "cephobjectrealms.ceph.rook.io", - "cephobjectzonegroups.ceph.rook.io", - "cephobjectzones.ceph.rook.io", - "cephfilesystems.ceph.rook.io", - "cephnfses.ceph.rook.io", - "cephclients.ceph.rook.io", - "volumes.rook.io", - "objectbuckets.objectbucket.io", - "objectbucketclaims.objectbucket.io", - "cephrbdmirrors.ceph.rook.io", - "cephfilesystemmirrors.ceph.rook.io") - checkError(h.T(), err, "cannot delete CRDs") - - if !h.useHelm { + if !h.settings.UseHelm { logger.Infof("Deleting all the resources in the operator manifest") - rookOperator := h.Manifests.GetRookOperator(systemNamespace) - _, err = h.k8shelper.KubectlWithStdin(rookOperator, deleteFromStdinArgs...) - logger.Infof("DONE deleting all the resources in the operator manifest") - checkError(h.T(), err, "cannot uninstall rook-operator") + _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetOperator(), deleteFromStdinArgs...) + if err != nil { + logger.Errorf("failed to remove operator resources. %v", err) + } else { + logger.Infof("done deleting all the resources in the operator manifest") + } } + logger.Info("removing the CRDs") + _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetCRDs(h.k8shelper), deleteFromStdinArgs...) + if err != nil { + logger.Errorf("failed to remove CRDS. %v", err) + } else { + logger.Infof("done deleting all the CRDs") + } + + err = h.k8shelper.DeleteResourceAndWait(false, "namespace", h.settings.OperatorNamespace) + checkError(h.T(), err, fmt.Sprintf("cannot delete operator namespace %s", h.settings.OperatorNamespace)) + err = h.k8shelper.Clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, "rook-ceph-webhook", metav1.DeleteOptions{}) checkError(h.T(), err, "failed to delete webhook configuration") - err = h.k8shelper.Clientset.RbacV1().RoleBindings(systemNamespace).Delete(ctx, "rook-ceph-system", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rook-ceph-global", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rook-ceph-mgr-cluster", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete role binding") - err = h.k8shelper.Clientset.CoreV1().ServiceAccounts(systemNamespace).Delete(ctx, "rook-ceph-system", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete service account") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "rook-ceph-cluster-mgmt", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "rook-ceph-mgr-cluster", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "rook-ceph-mgr-system", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "rook-ceph-global", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - err = h.k8shelper.Clientset.RbacV1().Roles(systemNamespace).Delete(ctx, "rook-ceph-system", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete role") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rbd-csi-nodeplugin", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "rbd-csi-nodeplugin", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rbd-csi-provisioner-role", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "rbd-external-provisioner-runner", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "cephfs-csi-nodeplugin", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "cephfs-csi-nodeplugin", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "cephfs-csi-provisioner-role", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "cephfs-external-provisioner-runner", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - - err = h.k8shelper.Clientset.PolicyV1beta1().PodSecurityPolicies().Delete(ctx, "rook-privileged", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete policy") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "psp:rook", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rook-ceph-system-psp", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rook-csi-rbd-provisioner-sa-psp", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rook-csi-rbd-plugin-sa-psp", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rook-csi-cephfs-provisioner-sa-psp", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rook-csi-cephfs-plugin-sa-psp", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - - err = h.k8shelper.Clientset.CoreV1().ConfigMaps(systemNamespace).Delete(ctx, "csi-rbd-config", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete config map") - err = h.k8shelper.Clientset.CoreV1().ConfigMaps(systemNamespace).Delete(ctx, "csi-cephfs-config", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete config map") - - err = h.k8shelper.Clientset.RbacV1().ClusterRoleBindings().Delete(ctx, "rook-ceph-object-bucket", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role binding") - err = h.k8shelper.Clientset.RbacV1().ClusterRoles().Delete(ctx, "rook-ceph-object-bucket", metav1.DeleteOptions{}) - checkError(h.T(), err, "failed to delete cluster role") - - logger.Infof("done removing the operator from namespace %s", systemNamespace) + logger.Infof("done removing the operator from namespace %s", h.settings.OperatorNamespace) logger.Infof("removing host data dir %s", h.hostPathToDelete) // removing data dir if exists if h.hostPathToDelete != "" { nodes, err := h.GetNodeHostnames() checkError(h.T(), err, "cannot get node names") for _, node := range nodes { - if h.cleanupHost { - err = h.verifyDirCleanup(node, h.hostPathToDelete) - logger.Infof("verifying clean up of %s from node %s. err=%v", h.hostPathToDelete, node, err) - assert.NoError(h.T(), err) - } else { - err = h.cleanupDir(node, h.hostPathToDelete) - logger.Infof("removing %s from node %s. err=%v", h.hostPathToDelete, node, err) - } + err = h.verifyDirCleanup(node, h.hostPathToDelete) + logger.Infof("verified cleanup of %s from node %s", h.hostPathToDelete, node) + assert.NoError(h.T(), err) } } if h.changeHostnames { @@ -720,12 +661,12 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, name // wait a bit longer for the system namespace to be cleaned up after their deletion for i := 0; i < 15; i++ { - _, err := h.k8shelper.Clientset.CoreV1().Namespaces().Get(ctx, systemNamespace, metav1.GetOptions{}) + _, err := h.k8shelper.Clientset.CoreV1().Namespaces().Get(ctx, h.settings.OperatorNamespace, metav1.GetOptions{}) if err != nil && kerrors.IsNotFound(err) { - logger.Infof("system namespace %q removed", systemNamespace) + logger.Infof("operator namespace %q removed", h.settings.OperatorNamespace) break } - logger.Infof("system namespace %q still found...", systemNamespace) + logger.Infof("operator namespace %q still found...", h.settings.OperatorNamespace) time.Sleep(5 * time.Second) } } @@ -733,7 +674,7 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, name func (h *CephInstaller) removeClusterFinalizers(namespace string) { ctx := context.TODO() // Get the latest cluster instead of using the same instance in case it has been changed - cluster, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(ctx, h.clusterName, metav1.GetOptions{}) + cluster, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(ctx, h.settings.ClusterName, metav1.GetOptions{}) if err != nil { logger.Errorf("failed to remove finalizer. failed to get cluster. %+v", err) return @@ -752,9 +693,9 @@ func (h *CephInstaller) removeClusterFinalizers(namespace string) { logger.Infof("removed finalizers from cluster %s", objectMeta.Name) } -func (h *CephInstaller) checkCephHealthStatus(namespace string) { +func (h *CephInstaller) checkCephHealthStatus() { ctx := context.TODO() - clusterResource, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(ctx, h.clusterName, metav1.GetOptions{}) + clusterResource, err := h.k8shelper.RookClientset.CephV1().CephClusters(h.settings.Namespace).Get(ctx, h.settings.ClusterName, metav1.GetOptions{}) assert.Nil(h.T(), err) clusterPhase := string(clusterResource.Status.Phase) if clusterPhase != "Ready" && clusterPhase != "Connected" && clusterPhase != "Progressing" { @@ -763,9 +704,10 @@ func (h *CephInstaller) checkCephHealthStatus(namespace string) { // Depending on the tests, the health may be fluctuating with different components being started or stopped. // If needed, give it a few seconds to settle and check the status again. + logger.Infof("checking ceph cluster health in namespace %q", h.settings.Namespace) if clusterResource.Status.CephStatus.Health != "HEALTH_OK" { time.Sleep(10 * time.Second) - clusterResource, err = h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(ctx, h.clusterName, metav1.GetOptions{}) + clusterResource, err = h.k8shelper.RookClientset.CephV1().CephClusters(h.settings.Namespace).Get(ctx, h.settings.ClusterName, metav1.GetOptions{}) assert.Nil(h.T(), err) } @@ -783,24 +725,18 @@ func (h *CephInstaller) checkCephHealthStatus(namespace string) { } } -func (h *CephInstaller) cleanupDir(node, dir string) error { - resources := h.GetCleanupPod(node, dir) - _, err := h.k8shelper.KubectlWithStdin(resources, createFromStdinArgs...) - return err -} - func (h *CephInstaller) verifyDirCleanup(node, dir string) error { resources := h.GetCleanupVerificationPod(node, dir) _, err := h.k8shelper.KubectlWithStdin(resources, createFromStdinArgs...) return err } -func (h *CephInstaller) CollectOperatorLog(suiteName, testName, namespace string) { +func (h *CephInstaller) CollectOperatorLog(suiteName, testName string) { if !h.T().Failed() && TestLogCollectionLevel() != "all" { return } name := fmt.Sprintf("%s_%s", suiteName, testName) - h.k8shelper.CollectPodLogsFromLabel(cephOperatorLabel, namespace, name, utils.TestEnvName()) + h.k8shelper.CollectPodLogsFromLabel(cephOperatorLabel, h.settings.OperatorNamespace, name, utils.TestEnvName()) } func (h *CephInstaller) GatherAllRookLogs(testName string, namespaces ...string) { @@ -816,18 +752,14 @@ func (h *CephInstaller) GatherAllRookLogs(testName string, namespaces ...string) } // NewCephInstaller creates new instance of CephInstaller -func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, useHelm bool, clusterName, rookVersion string, - cephVersion cephv1.CephVersionSpec, cleanupHost bool) *CephInstaller { +func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, settings *TestCephSettings) *CephInstaller { // By default set a cluster name that is different from the namespace so we don't rely on the namespace // in expected places - if clusterName == "" { - clusterName = defaultclusterName + if settings.ClusterName == "" { + settings.ClusterName = defaultclusterName } - // All e2e tests should run ceph commands in the toolbox since we are not inside a container - client.RunAllCephCommandsInToolbox = true - version, err := clientset.ServerVersion() if err != nil { logger.Infof("failed to get kubectl server version. %+v", err) @@ -837,17 +769,14 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, useH if err != nil { panic("failed to get kubectl client :" + err.Error()) } - logger.Infof("Rook Version: %s", rookVersion) - logger.Infof("Ceph Version: %s", cephVersion.Image) + logger.Infof("Rook Version: %s", settings.RookVersion) + logger.Infof("Ceph Version: %s", settings.CephVersion.Image) h := &CephInstaller{ - Manifests: NewCephManifests(rookVersion), + settings: settings, + Manifests: NewCephManifests(settings), k8shelper: k8shelp, helmHelper: utils.NewHelmHelper(testHelmPath()), - useHelm: useHelm, - clusterName: clusterName, k8sVersion: version.String(), - CephVersion: cephVersion, - cleanupHost: cleanupHost, changeHostnames: k8shelp.VersionAtLeast("v1.18.0"), T: t, } @@ -921,18 +850,26 @@ spec: } func (h *CephInstaller) addCleanupPolicy(namespace string) error { - ctx := context.TODO() - cluster, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(ctx, h.clusterName, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("failed to get ceph cluster. %+v", err) + // Retry updating the CR a few times in case of random failure + var returnErr error + for i := 0; i < 3; i++ { + ctx := context.TODO() + cluster, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(ctx, h.settings.ClusterName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get ceph cluster. %+v", err) + } + cluster.Spec.CleanupPolicy.Confirmation = cephv1.DeleteDataDirOnHostsConfirmation + cluster.Spec.CleanupPolicy.AllowUninstallWithVolumes = true + _, err = h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Update(ctx, cluster, metav1.UpdateOptions{}) + if err != nil { + returnErr = fmt.Errorf("failed to add clean up policy to the cluster. %+v", err) + logger.Warningf("could not add cleanup policy, trying again... %v", err) + } else { + logger.Info("successfully added cleanup policy to the ceph cluster and skipping checks for existing volumes") + return nil + } } - cluster.Spec.CleanupPolicy.Confirmation = cephv1.DeleteDataDirOnHostsConfirmation - _, err = h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Update(ctx, cluster, metav1.UpdateOptions{}) - if err != nil { - return fmt.Errorf("failed to add clean up policy to the cluster. %+v", err) - } - logger.Info("successfully added cleanup policy to the ceph cluster") - return nil + return returnErr } func (h *CephInstaller) waitForCleanupJobs(namespace string) error { diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index c0a1e781a..283072307 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -18,1292 +18,130 @@ package installer import ( "fmt" - "io/ioutil" - "path" "strconv" "strings" - "github.com/pkg/errors" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/tests/framework/utils" ) type CephManifests interface { - GetRookCRDs(v1ExtensionsSupported bool) string - GetRookOperator(namespace string) string - GetClusterRoles(namespace, operatorNamespace string) string - GetClusterExternalRoles(namespace, operatorNamespace string) string - GetRookCluster(settings *clusterSettings) string - GetRookExternalCluster(settings *clusterExternalSettings) string - GetRookToolBox(namespace string) string - GetBlockPoolDef(poolName, namespace, replicaSize string) string - GetBlockStorageClassDef(csi bool, poolName, storageClassName, reclaimPolicy, namespace, operatorNamespace string) string - GetFileStorageClassDef(fsName, storageClassName, operatorNamespace, namespace string) string - GetPVC(claimName, namespace, storageClassName, accessModes, size string) string - GetBlockSnapshotClass(snapshotClassName, namespace, operatorNamespace, reclaimPolicy string) string - GetFileStorageSnapshotClass(snapshotClassName, namespace, operatorNamespace, reclaimPolicy string) string - GetPVCRestore(claimName, snapshotName, namespace, storageClassName, accessModes, size string) string - GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, accessModes, size string) string - GetSnapshot(snapshotName, claimName, snapshotClassName, namespace string) string - GetPod(podName, claimName, namespace, mountPoint string, readOnly bool) string - GetFilesystem(namespace, name string, activeCount int) string - GetNFS(namespace, name, pool string, daemonCount int) string - GetRBDMirror(namespace, name string, daemonCount int) string - GetObjectStore(namespace, name string, replicaCount, port int) string - GetObjectStoreUser(namespace, name, displayName, store string) string - GetBucketStorageClass(namespace, storeName, storageClassName, reclaimPolicy, region string) string - GetObc(obcName, storageClassName, bucketName string, maxObject string, createBucket bool) string - GetClient(namespace, name string, caps map[string]string) string -} - -type clusterSettings struct { - ClusterName string - Namespace string - StoreType string - DataDirHostPath string - Mons int - MultipleMgrs bool - RBDMirrorWorkers int - UsePVCs bool - StorageClassName string - skipOSDCreation bool - CephVersion cephv1.CephVersionSpec - useCrashPruner bool -} - -// clusterExternalSettings represents the settings of an external cluster -type clusterExternalSettings struct { - Namespace string - DataDirHostPath string + Settings() *TestCephSettings + GetCRDs(k8shelper *utils.K8sHelper) string + GetOperator() string + GetCommon() string + GetCommonExternal() string + GetCephCluster() string + GetExternalCephCluster() string + GetToolbox() string + GetBlockPool(poolName, replicaSize string) string + GetBlockStorageClass(poolName, storageClassName, reclaimPolicy string) string + GetFileStorageClass(fsName, storageClassName string) string + GetBlockSnapshotClass(snapshotClassName, reclaimPolicy string) string + GetFileStorageSnapshotClass(snapshotClassName, reclaimPolicy string) string + GetFilesystem(name string, activeCount int) string + GetNFS(name, pool string, daemonCount int) string + GetRBDMirror(name string, daemonCount int) string + GetObjectStore(name string, replicaCount, port int) string + GetObjectStoreUser(name, displayName, store string) string + GetBucketStorageClass(storeName, storageClassName, reclaimPolicy, region string) string + GetOBC(obcName, storageClassName, bucketName string, maxObject string, createBucket bool) string + GetClient(name string, caps map[string]string) string } // CephManifestsMaster wraps rook yaml definitions type CephManifestsMaster struct { - imageTag string + settings *TestCephSettings } // NewCephManifests gets the manifest type depending on the Rook version desired -func NewCephManifests(version string) CephManifests { - switch version { +func NewCephManifests(settings *TestCephSettings) CephManifests { + switch settings.RookVersion { case VersionMaster: - return &CephManifestsMaster{imageTag: VersionMaster} + return &CephManifestsMaster{settings} case Version1_5: - return &CephManifestsV1_5{imageTag: Version1_5} + return &CephManifestsV1_5{settings} } - panic(fmt.Errorf("unrecognized ceph manifest version: %s", version)) + panic(fmt.Errorf("unrecognized ceph manifest version: %s", settings.RookVersion)) } -func readManifest(filename string) string { - rootDir, err := utils.FindRookRoot() - if err != nil { - panic(err) - } - manifest := path.Join(rootDir, "cluster/examples/kubernetes/ceph", filename) - logger.Infof("Reading manifest: %s", manifest) - contents, err := ioutil.ReadFile(manifest) - if err != nil { - panic(errors.Wrapf(err, "failed to read manifest at %s", manifest)) - } - return string(contents) +func (m *CephManifestsMaster) Settings() *TestCephSettings { + return m.settings } -func (m *CephManifestsMaster) GetRookCRDs(v1ExtensionsSupported bool) string { - if v1ExtensionsSupported { - return readManifest("crds.yaml") +func (m *CephManifestsMaster) GetCRDs(k8shelper *utils.K8sHelper) string { + if k8shelper.VersionAtLeast("v1.16.0") { + return m.settings.readManifest("crds.yaml") } - return readManifest("pre-k8s-1.16/crds.yaml") + return m.settings.readManifest("pre-k8s-1.16/crds.yaml") } -func getOpenshiftSCC(namespace string) string { - manifest := readManifest("scc.yaml") - return strings.ReplaceAll(manifest, "system:serviceaccount:rook-ceph:", fmt.Sprintf("system:serviceaccount:%s:", namespace)) -} - -// GetRookOperator returns rook Operator manifest -func (m *CephManifestsMaster) GetRookOperator(operatorNamespace string) string { - var operatorManifest string - openshiftEnv := "" - +func (m *CephManifestsMaster) GetOperator() string { + var manifest string if utils.IsPlatformOpenShift() { - openshiftEnv = ` - - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED - value: "true" - - name: FLEXVOLUME_DIR_PATH - value: "/etc/kubernetes/kubelet-plugins/volume/exec"` - operatorManifest = getOpenshiftSCC(operatorNamespace) + manifest = m.settings.readManifest("operator-openshift.yaml") + } else { + manifest = m.settings.readManifest("operator.yaml") } - - operatorManifest = operatorManifest + `--- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: rook-ceph-system - namespace: ` + operatorNamespace + ` - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - - apps - - extensions - resources: - - pods - - configmaps - - services - - daemonsets - - statefulsets - - deployments - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - batch - resources: - - cronjobs - verbs: - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-cluster-mgmt - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - - secrets - - pods - - pods/log - - services - - configmaps - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - apps - - extensions - resources: - - deployments - - daemonsets - verbs: - - get - - list - - watch - - create - - update - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-global - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - # Pod access is needed for fencing - - pods - # Node access is needed for determining nodes where mons should run - - nodes - - nodes/proxy - - services - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - # PVs and PVCs are managed by the Rook provisioner - - persistentvolumes - - persistentvolumeclaims - - endpoints - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - batch - resources: - - jobs - - cronjobs - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - policy - - apps - - extensions - resources: - # This is for the clusterdisruption controller - - poddisruptionbudgets - # This is for both clusterdisruption and nodedrain controllers - - deployments - - replicasets - verbs: - - "*" -- apiGroups: - - healthchecking.openshift.io - resources: - - machinedisruptionbudgets - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - machine.openshift.io - resources: - - machines - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - csidrivers - verbs: - - create - - delete - - get - - update -- apiGroups: - - k8s.cni.cncf.io - resources: - - network-attachment-definitions - verbs: - - get ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-cluster -rules: -- apiGroups: - - "" - resources: - - configmaps - - nodes - - nodes/proxy - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - list - - get - - watch ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-osd -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list ---- -# Aspects of Rook Ceph Agent that require access to secrets -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-agent-mount - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get ---- -# Aspects of ceph-mgr that require access to the system namespace -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-system -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-object-bucket - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - verbs: - - "*" - resources: - - secrets - - configmaps -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - "objectbucket.io" - verbs: - - "*" - resources: - - "*" ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-object-bucket -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-object-bucket -subjects: - - kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-system - namespace: ` + operatorNamespace + ` - labels: - operator: rook - storage-backend: ceph ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-system - namespace: ` + operatorNamespace + ` - labels: - operator: rook - storage-backend: ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-system -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-global - labels: - operator: rook - storage-backend: ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-global -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-rbd-plugin-sa - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-plugin-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: rbd-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-rbd-provisioner-sa - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-provisioner-runner -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-provisioner-role -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: rbd-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - namespace: ` + operatorNamespace + ` - name: rbd-external-provisioner-cfg -rules: - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "delete", "update"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-provisioner-role-cfg - namespace: ` + operatorNamespace + ` -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: Role - name: rbd-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-cephfs-plugin-sa - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-plugin-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: cephfs-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-cephfs-provisioner-sa - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-provisioner-runner -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-provisioner-role -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: cephfs-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - namespace: ` + operatorNamespace + ` - name: cephfs-external-provisioner-cfg -rules: - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "create", "delete"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-provisioner-role-cfg - namespace: ` + operatorNamespace + ` -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: Role - name: cephfs-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: 00-rook-privileged - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' - seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' -spec: - privileged: true - allowedCapabilities: - # required by CSI - - SYS_ADMIN - # fsGroup - the flexVolume agent has fsGroup capabilities and could potentially be any group - fsGroup: - rule: RunAsAny - # runAsUser, supplementalGroups - Rook needs to run some pods as root - # Ceph pods could be run as the Ceph user, but that user isn't always known ahead of time - runAsUser: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - # seLinux - seLinux context is unknown ahead of time; set if this is well-known - seLinux: - rule: RunAsAny - volumes: - # recommended minimum set - - configMap - - downwardAPI - - emptyDir - - persistentVolumeClaim - - secret - - projected - # required for Rook - - hostPath - - flexVolume - # allowedHostPaths can be set to Rook's known host volume mount points when they are fully-known - # allowedHostPaths: - # - /run/udev # for OSD prep - # - /dev # for OSD prep - # - /var/lib/rook # or whatever the dataDirHostPath value is set to - # Ceph requires host IPC for setting up encrypted devices - hostIPC: true - # Ceph OSDs need to share the same PID namespace - hostPID: true - # hostNetwork can be set to 'false' if host networking isn't used - hostNetwork: true - hostPorts: - # Ceph messenger protocol v1 - - min: 6789 - max: 6790 # <- support old default port - # Ceph messenger protocol v2 - - min: 3300 - max: 3300 - # Ceph RADOS ports for OSDs, MDSes - - min: 6800 - max: 7300 - # # Ceph dashboard port HTTP (not recommended) - # - min: 7000 - # max: 7000 - # Ceph dashboard port HTTPS - - min: 8443 - max: 8443 - # Ceph mgr Prometheus Metrics - - min: 9283 - max: 9283 ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: 'psp:rook' -rules: - - apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - 00-rook-privileged - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-ceph-system-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-rbd-provisioner-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: ` + operatorNamespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-rbd-plugin-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-plugin-sa - namespace: ` + operatorNamespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-cephfs-provisioner-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: ` + operatorNamespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-cephfs-plugin-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-plugin-sa - namespace: ` + operatorNamespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-admission-controller - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-admission-controller-role -rules: - - apiGroups: ["ceph.rook.io"] - resources: ["*"] - verbs: ["get", "watch", "list"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-admission-controller-rolebinding -subjects: - - kind: ServiceAccount - name: rook-ceph-admission-controller - apiGroup: "" - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: rook-ceph-admission-controller-role - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: rook-ceph-operator - namespace: ` + operatorNamespace + ` - labels: - operator: rook - storage-backend: ceph -spec: - replicas: 1 - selector: - matchLabels: - app: rook-ceph-operator - template: - metadata: - labels: - app: rook-ceph-operator - spec: - serviceAccountName: rook-ceph-system - containers: - - name: rook-ceph-operator - image: rook/ceph:` + m.imageTag + ` - args: ["ceph", "operator"] - env: - - name: ROOK_LOG_LEVEL - value: INFO - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ROOK_ENABLE_FLEX_DRIVER - value: "true" - - name: ROOK_CURRENT_NAMESPACE_ONLY - value: "false"` + openshiftEnv + ` - - name: ROOK_ENABLE_DISCOVERY_DAEMON - value: "true" - volumeMounts: - - mountPath: /var/lib/rook - name: rook-config - - mountPath: /etc/ceph - name: default-config-dir - volumes: - - name: rook-config - emptyDir: {} - - name: default-config-dir - emptyDir: {} ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: rook-ceph-operator-config - namespace: ` + operatorNamespace + ` -data: - ROOK_CSI_ENABLE_CEPHFS: "true" - ROOK_CSI_ENABLE_RBD: "true" - ROOK_CSI_ENABLE_GRPC_METRICS: "false" - ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true" - CSI_LOG_LEVEL: "5" -` - return operatorManifest + return m.settings.replaceOperatorSettings(manifest) } -// GetClusterRoles returns rook-cluster manifest -func (m *CephManifestsMaster) GetClusterRoles(namespace, operatorNamespace string) string { - return `apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-osd - namespace: ` + namespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-osd - namespace: ` + namespace + ` -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: [ "get", "list", "watch", "create", "update", "delete" ] -- apiGroups: ["ceph.rook.io"] - resources: ["cephclusters", "cephclusters/finalizers"] - verbs: [ "get", "list", "create", "update", "delete" ] ---- -# Aspects of ceph-mgr that operate within the cluster's namespace -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr - namespace: ` + namespace + ` -rules: -- apiGroups: - - "" - resources: - - pods - - services - - pods/log - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" ---- -# Allow the ceph osd to access cluster-wide resources necessary for determining their topology location -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-osd-` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-osd -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: ` + namespace + ` ---- -# Allow the ceph mgr to access cluster-wide resources necessary for the mgr modules -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-cluster-` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-mgr-cluster -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -rules: -- apiGroups: - - "" - resources: - - pods - - configmaps - verbs: - - get - - list - - watch - - create - - update - - delete ---- -# Allow the operator to create resources in this cluster's namespace -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cluster-mgmt - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-cluster-mgmt -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -# Allow the osd pods in this namespace to work with configmaps -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-osd - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-osd -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: ` + namespace + ` ---- -# Allow the ceph mgr to access the cluster-specific resources necessary for the mgr modules -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-mgr -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -# Allow the ceph mgr to access the rook system resources necessary for the mgr modules -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-system ` + namespace + ` - namespace: ` + operatorNamespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-mgr-system -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-cmd-reporter -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-default-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: default - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-osd-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-mgr-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-cmd-reporter-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -` +func (m *CephManifestsMaster) GetCommonExternal() string { + return m.settings.readManifest("common-external.yaml") } -// GetRookCluster returns rook-cluster manifest -func (m *CephManifestsMaster) GetRookCluster(settings *clusterSettings) string { - store := "# storeType not specified; Rook will use default store types" - if settings.StoreType != "" { - store = `storeType: "` + settings.StoreType + `"` - } +func (m *CephManifestsMaster) GetCommon() string { + return m.settings.readManifest("common.yaml") +} +func (m *CephManifestsMaster) GetToolbox() string { + manifest := strings.ReplaceAll(m.settings.readManifest("direct-mount.yaml"), "name: rook-direct-mount", "name: rook-ceph-tools") + return strings.ReplaceAll(manifest, "app: rook-direct-mount", "app: rook-ceph-tools") +} + +//********************************************************************************** +//********************************************************************************** +// After a release, copy the methods below this separator into the versioned file +// such as ceph_manifests_v1.6.go. Methods above this separator do not need to be +// copied since the versioned implementation will load them directly from github. +//********************************************************************************** +//********************************************************************************** + +func (m *CephManifestsMaster) GetCephCluster() string { crushRoot := "# crushRoot not specified; Rook will use `default`" - if settings.Mons == 1 { + if m.settings.Mons == 1 { crushRoot = `crushRoot: "custom-root"` } pruner := "# daysToRetain not specified;" - if settings.useCrashPruner { + if m.settings.UseCrashPruner { pruner = "daysToRetain: 5" } mgrCount := 1 - if settings.MultipleMgrs { + if m.settings.MultipleMgrs { mgrCount = 2 } - if settings.UsePVCs { + if m.settings.UsePVC { return `apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: # set the name to something different from the namespace - name: ` + settings.ClusterName + ` - namespace: ` + settings.Namespace + ` + name: ` + m.settings.ClusterName + ` + namespace: ` + m.settings.Namespace + ` spec: - dataDirHostPath: ` + settings.DataDirHostPath + ` + dataDirHostPath: ` + m.settings.DataDirHostPath + ` mon: - count: ` + strconv.Itoa(settings.Mons) + ` + count: ` + strconv.Itoa(m.settings.Mons) + ` allowMultiplePerNode: true volumeClaimTemplate: spec: - storageClassName: ` + settings.StorageClassName + ` + storageClassName: ` + m.settings.StorageClassName + ` resources: requests: storage: 5Gi cephVersion: - image: ` + settings.CephVersion.Image + ` - allowUnsupported: ` + strconv.FormatBool(settings.CephVersion.AllowUnsupported) + ` + image: ` + m.settings.CephVersion.Image + ` + allowUnsupported: ` + strconv.FormatBool(m.settings.CephVersion.AllowUnsupported) + ` skipUpgradeChecks: false continueUpgradeAfterChecksEvenIfNotHealthy: false mgr: @@ -1332,7 +170,7 @@ spec: resources: requests: storage: 10Gi - storageClassName: ` + settings.StorageClassName + ` + storageClassName: ` + m.settings.StorageClassName + ` volumeMode: Block accessModes: - ReadWriteOnce @@ -1347,31 +185,30 @@ spec: return `apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: - name: ` + settings.ClusterName + ` - namespace: ` + settings.Namespace + ` + name: ` + m.settings.ClusterName + ` + namespace: ` + m.settings.Namespace + ` spec: cephVersion: - image: ` + settings.CephVersion.Image + ` - allowUnsupported: ` + strconv.FormatBool(settings.CephVersion.AllowUnsupported) + ` - dataDirHostPath: ` + settings.DataDirHostPath + ` + image: ` + m.settings.CephVersion.Image + ` + allowUnsupported: ` + strconv.FormatBool(m.settings.CephVersion.AllowUnsupported) + ` + dataDirHostPath: ` + m.settings.DataDirHostPath + ` network: hostNetwork: false crashCollector: disable: false ` + pruner + ` mon: - count: ` + strconv.Itoa(settings.Mons) + ` + count: ` + strconv.Itoa(m.settings.Mons) + ` allowMultiplePerNode: true dashboard: enabled: true skipUpgradeChecks: true metadataDevice: storage: - useAllNodes: ` + strconv.FormatBool(!settings.skipOSDCreation) + ` - useAllDevices: ` + strconv.FormatBool(!settings.skipOSDCreation) + ` + useAllNodes: ` + strconv.FormatBool(!m.settings.SkipOSDCreation) + ` + useAllDevices: ` + strconv.FormatBool(!m.settings.SkipOSDCreation) + ` deviceFilter: ` + getDeviceFilter() + ` config: - ` + store + ` databaseSizeMB: "1024" journalSizeMB: "1024" mgr: @@ -1391,174 +228,44 @@ spec: interval: 5s` } -// GetRookToolBox returns rook-toolbox manifest -func (m *CephManifestsMaster) GetRookToolBox(namespace string) string { - return `apiVersion: v1 -kind: Pod -metadata: - name: rook-ceph-tools - namespace: ` + namespace + ` -spec: - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: rook-ceph-tools - image: rook/ceph:` + m.imageTag + ` - imagePullPolicy: IfNotPresent - command: ["/tini"] - args: ["-g", "--", "/usr/local/bin/toolbox.sh"] - env: - - name: ROOK_CEPH_USERNAME - valueFrom: - secretKeyRef: - name: rook-ceph-mon - key: ceph-username - - name: ROOK_CEPH_SECRET - valueFrom: - secretKeyRef: - name: rook-ceph-mon - key: ceph-secret - securityContext: - privileged: true - volumeMounts: - - mountPath: /dev - name: dev - - mountPath: /sys/bus - name: sysbus - - mountPath: /lib/modules - name: libmodules - - name: mon-endpoint-volume - mountPath: /etc/rook - hostNetwork: false - volumes: - - name: dev - hostPath: - path: /dev - - name: sysbus - hostPath: - path: /sys/bus - - name: libmodules - hostPath: - path: /lib/modules - - name: mon-endpoint-volume - configMap: - name: rook-ceph-mon-endpoints - items: - - key: data - path: mon-endpoints` -} - -func (m *CephManifestsMaster) GetBlockSnapshotClass(snapshotClassName, namespace, operatorNamespace, reclaimPolicy string) string { +func (m *CephManifestsMaster) GetBlockSnapshotClass(snapshotClassName, reclaimPolicy string) string { // Create a CSI driver snapshotclass return ` apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshotClass metadata: name: ` + snapshotClassName + ` -driver: ` + operatorNamespace + `.rbd.csi.ceph.com +driver: ` + m.settings.OperatorNamespace + `.rbd.csi.ceph.com deletionPolicy: ` + reclaimPolicy + ` parameters: - clusterID: ` + namespace + ` + clusterID: ` + m.settings.Namespace + ` csi.storage.k8s.io/snapshotter-secret-name: rook-csi-rbd-provisioner - csi.storage.k8s.io/snapshotter-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/snapshotter-secret-namespace: ` + m.settings.Namespace + ` ` } -func (m *CephManifestsMaster) GetFileStorageSnapshotClass(snapshotClassName, namespace, operatorNamespace, reclaimPolicy string) string { +func (m *CephManifestsMaster) GetFileStorageSnapshotClass(snapshotClassName, reclaimPolicy string) string { // Create a CSI driver snapshotclass return ` apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshotClass metadata: name: ` + snapshotClassName + ` -driver: ` + operatorNamespace + `.cephfs.csi.ceph.com +driver: ` + m.settings.OperatorNamespace + `.cephfs.csi.ceph.com deletionPolicy: ` + reclaimPolicy + ` parameters: - clusterID: ` + namespace + ` + clusterID: ` + m.settings.Namespace + ` csi.storage.k8s.io/snapshotter-secret-name: rook-csi-cephfs-provisioner - csi.storage.k8s.io/snapshotter-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/snapshotter-secret-namespace: ` + m.settings.Namespace + ` ` } -func (m *CephManifestsMaster) GetPVCRestore(claimName, snapshotName, namespace, storageClassName, accessModes, size string) string { - return `apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ` + claimName + ` - namespace: ` + namespace + ` -spec: - storageClassName: ` + storageClassName + ` - dataSource: - name: ` + snapshotName + ` - kind: VolumeSnapshot - apiGroup: snapshot.storage.k8s.io - accessModes: - - ` + accessModes + ` - resources: - requests: - storage: ` + size -} - -func (m *CephManifestsMaster) GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, accessModes, size string) string { - return `apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ` + cloneClaimName + ` - namespace: ` + namespace + ` -spec: - storageClassName: ` + storageClassName + ` - dataSource: - name: ` + parentClaimName + ` - kind: PersistentVolumeClaim - accessModes: - - ` + accessModes + ` - resources: - requests: - storage: ` + size -} - -func (m *CephManifestsMaster) GetSnapshot(snapshotName, claimName, snapshotClassName, namespace string) string { - return `apiVersion: snapshot.storage.k8s.io/v1beta1 -kind: VolumeSnapshot -metadata: - name: ` + snapshotName + ` - namespace: ` + namespace + ` -spec: - volumeSnapshotClassName: ` + snapshotClassName + ` - source: - persistentVolumeClaimName: ` + claimName -} - -func (m *CephManifestsMaster) GetPod(podName, claimName, namespace, mountPath string, readOnly bool) string { - return ` -apiVersion: v1 -kind: Pod -metadata: - name: ` + podName + ` - namespace: ` + namespace + ` -spec: - containers: - - name: ` + podName + ` - image: busybox - command: ["/bin/sleep", "infinity"] - imagePullPolicy: IfNotPresent - volumeMounts: - - mountPath: ` + mountPath + ` - name: csivol - volumes: - - name: csivol - persistentVolumeClaim: - claimName: ` + claimName + ` - readOnly: ` + strconv.FormatBool(readOnly) + ` - restartPolicy: Never -` -} - -func (m *CephManifestsMaster) GetBlockPoolDef(poolName string, namespace string, replicaSize string) string { +func (m *CephManifestsMaster) GetBlockPool(poolName string, replicaSize string) string { return `apiVersion: ceph.rook.io/v1 kind: CephBlockPool metadata: name: ` + poolName + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: replicated: size: ` + replicaSize + ` @@ -1577,23 +284,23 @@ spec: interval: 10s` } -func (m *CephManifestsMaster) GetBlockStorageClassDef(csi bool, poolName, storageClassName, reclaimPolicy, namespace, operatorNamespace string) string { +func (m *CephManifestsMaster) GetBlockStorageClass(poolName, storageClassName, reclaimPolicy string) string { // Create a CSI driver storage class - if csi { + if m.settings.UseCSI { return ` apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ` + storageClassName + ` -provisioner: ` + operatorNamespace + `.rbd.csi.ceph.com +provisioner: ` + m.settings.OperatorNamespace + `.rbd.csi.ceph.com reclaimPolicy: ` + reclaimPolicy + ` parameters: pool: ` + poolName + ` - clusterID: ` + namespace + ` + clusterID: ` + m.settings.Namespace + ` csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner - csi.storage.k8s.io/provisioner-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/provisioner-secret-namespace: ` + m.settings.Namespace + ` csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node - csi.storage.k8s.io/node-stage-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/node-stage-secret-namespace: ` + m.settings.Namespace + ` imageFeatures: layering csi.storage.k8s.io/fstype: ext4 ` @@ -1608,10 +315,10 @@ allowVolumeExpansion: true reclaimPolicy: ` + reclaimPolicy + ` parameters: blockPool: ` + poolName + ` - clusterNamespace: ` + namespace + clusterNamespace: ` + m.settings.Namespace } -func (m *CephManifestsMaster) GetFileStorageClassDef(fsName, storageClassName, operatorNamespace, namespace string) string { +func (m *CephManifestsMaster) GetFileStorageClass(fsName, storageClassName string) string { // Create a CSI driver storage class csiCephFSNodeSecret := "rook-csi-cephfs-node" //nolint:gosec // We safely suppress gosec in tests file csiCephFSProvisionerSecret := "rook-csi-cephfs-provisioner" //nolint:gosec // We safely suppress gosec in tests file @@ -1620,40 +327,25 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ` + storageClassName + ` -provisioner: ` + operatorNamespace + `.cephfs.csi.ceph.com +provisioner: ` + m.settings.OperatorNamespace + `.cephfs.csi.ceph.com parameters: - clusterID: ` + namespace + ` + clusterID: ` + m.settings.Namespace + ` fsName: ` + fsName + ` pool: ` + fsName + `-data0 csi.storage.k8s.io/provisioner-secret-name: ` + csiCephFSProvisionerSecret + ` - csi.storage.k8s.io/provisioner-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/provisioner-secret-namespace: ` + m.settings.Namespace + ` csi.storage.k8s.io/node-stage-secret-name: ` + csiCephFSNodeSecret + ` - csi.storage.k8s.io/node-stage-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/node-stage-secret-namespace: ` + m.settings.Namespace + ` ` } -func (m *CephManifestsMaster) GetPVC(claimName, namespace, storageClassName, accessModes, size string) string { - return `apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ` + claimName + ` - namespace: ` + namespace + ` -spec: - storageClassName: ` + storageClassName + ` - accessModes: - - ` + accessModes + ` - resources: - requests: - storage: ` + size -} - // GetFilesystem returns the manifest to create a Rook filesystem resource with the given config. -func (m *CephManifestsMaster) GetFilesystem(namespace, name string, activeCount int) string { +func (m *CephManifestsMaster) GetFilesystem(name string, activeCount int) string { return `apiVersion: ceph.rook.io/v1 kind: CephFilesystem metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: metadataPool: replicated: @@ -1670,12 +362,12 @@ spec: } // GetFilesystem returns the manifest to create a Rook Ceph NFS resource with the given config. -func (m *CephManifestsMaster) GetNFS(namespace, name, pool string, count int) string { +func (m *CephManifestsMaster) GetNFS(name, pool string, count int) string { return `apiVersion: ceph.rook.io/v1 kind: CephNFS metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: rados: pool: ` + pool + ` @@ -1684,12 +376,12 @@ spec: active: ` + strconv.Itoa(count) } -func (m *CephManifestsMaster) GetObjectStore(namespace, name string, replicaCount, port int) string { +func (m *CephManifestsMaster) GetObjectStore(name string, replicaCount, port int) string { return `apiVersion: ceph.rook.io/v1 kind: CephObjectStore metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: metadataPool: replicated: @@ -1711,33 +403,33 @@ spec: ` } -func (m *CephManifestsMaster) GetObjectStoreUser(namespace, name string, displayName string, store string) string { +func (m *CephManifestsMaster) GetObjectStoreUser(name string, displayName string, store string) string { return `apiVersion: ceph.rook.io/v1 kind: CephObjectStoreUser metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: displayName: ` + displayName + ` store: ` + store } //GetBucketStorageClass returns the manifest to create object bucket -func (m *CephManifestsMaster) GetBucketStorageClass(storeNameSpace string, storeName string, storageClassName string, reclaimPolicy string, region string) string { +func (m *CephManifestsMaster) GetBucketStorageClass(storeName, storageClassName, reclaimPolicy, region string) string { return `apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ` + storageClassName + ` -provisioner: ` + storeNameSpace + `.ceph.rook.io/bucket +provisioner: ` + m.settings.Namespace + `.ceph.rook.io/bucket reclaimPolicy: ` + reclaimPolicy + ` parameters: objectStoreName: ` + storeName + ` - objectStoreNamespace: ` + storeNameSpace + ` + objectStoreNamespace: ` + m.settings.Namespace + ` region: ` + region } -//GetObc returns the manifest to create object bucket claim -func (m *CephManifestsMaster) GetObc(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string { +//GetOBC returns the manifest to create object bucket claim +func (m *CephManifestsMaster) GetOBC(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string { bucketParameter := "generateBucketName" if varBucketName { bucketParameter = "bucketName" @@ -1753,7 +445,7 @@ spec: maxObjects: "` + maxObject + `"` } -func (m *CephManifestsMaster) GetClient(namespace string, claimName string, caps map[string]string) string { +func (m *CephManifestsMaster) GetClient(claimName string, caps map[string]string) string { clientCaps := []string{} for name, cap := range caps { str := name + ": " + cap @@ -1763,87 +455,31 @@ func (m *CephManifestsMaster) GetClient(namespace string, claimName string, caps kind: CephClient metadata: name: ` + claimName + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: caps: ` + strings.Join(clientCaps, "\n ") } -func (m *CephManifestsMaster) GetClusterExternalRoles(namespace, firstClusterNamespace string) string { - return `apiVersion: v1 -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cluster-mgmt - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-cluster-mgmt -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: ` + firstClusterNamespace + ` ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-cmd-reporter -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -rules: -- apiGroups: - - "" - resources: - - pods - - configmaps - verbs: - - get - - list - - watch - - create - - update - - delete` -} - -func (m *CephManifestsMaster) GetRookExternalCluster(settings *clusterExternalSettings) string { +func (m *CephManifestsMaster) GetExternalCephCluster() string { return `apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: - name: ` + settings.Namespace + ` - namespace: ` + settings.Namespace + ` + name: ` + m.settings.ClusterName + ` + namespace: ` + m.settings.Namespace + ` spec: external: enable: true - dataDirHostPath: ` + settings.DataDirHostPath + `` + dataDirHostPath: ` + m.settings.DataDirHostPath + `` } // GetRBDMirror returns the manifest to create a Rook Ceph RBD Mirror resource with the given config. -func (m *CephManifestsMaster) GetRBDMirror(namespace, name string, count int) string { +func (m *CephManifestsMaster) GetRBDMirror(name string, count int) string { return `apiVersion: ceph.rook.io/v1 kind: CephRBDMirror metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: count: ` + strconv.Itoa(count) } diff --git a/tests/framework/installer/ceph_manifests_v1.5.go b/tests/framework/installer/ceph_manifests_v1.5.go index 848009981..5376b0233 100644 --- a/tests/framework/installer/ceph_manifests_v1.5.go +++ b/tests/framework/installer/ceph_manifests_v1.5.go @@ -17,13 +17,9 @@ limitations under the License. package installer import ( - "fmt" - "io/ioutil" - "net/http" "strconv" "strings" - "github.com/pkg/errors" "github.com/rook/rook/tests/framework/utils" ) @@ -34,1201 +30,82 @@ const ( // CephManifestsV1_4 wraps rook yaml definitions type CephManifestsV1_5 struct { - imageTag string + settings *TestCephSettings } -func readManifestFromGithub(filename string) string { - url := fmt.Sprintf("https://raw.githubusercontent.com/rook/rook/%s/cluster/examples/kubernetes/ceph/%s", Version1_5, filename) - logger.Infof("Retrieving manifest: %s", url) - // #nosec G107 This is only test code and is expected to read from a url - response, err := http.Get(url) - - if err != nil { - panic(errors.Wrapf(err, "failed to read manifest from %s", url)) - } - defer response.Body.Close() - - content, err := ioutil.ReadAll(response.Body) - if err != nil { - panic(errors.Wrapf(err, "failed to read content from %s", url)) - } - return string(content) +func (m *CephManifestsV1_5) Settings() *TestCephSettings { + return m.settings } -func (m *CephManifestsV1_5) GetRookCRDs(v1ExtensionsSupported bool) string { - if v1ExtensionsSupported { - return readManifestFromGithub("crds.yaml") +func (m *CephManifestsV1_5) GetCRDs(k8shelper *utils.K8sHelper) string { + if k8shelper.VersionAtLeast("v1.16.0") { + return m.settings.readManifestFromGithub("crds.yaml") } - return readManifestFromGithub("pre-k8s-1.16/crds.yaml") + return m.settings.readManifestFromGithub("pre-k8s-1.16/crds.yaml") } // GetRookOperator returns rook Operator manifest -func (m *CephManifestsV1_5) GetRookOperator(operatorNamespace string) string { - var operatorManifest string - openshiftEnv := "" - +func (m *CephManifestsV1_5) GetOperator() string { + var manifest string if utils.IsPlatformOpenShift() { - openshiftEnv = ` - - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED - value: "true" - - name: FLEXVOLUME_DIR_PATH - value: "/etc/kubernetes/kubelet-plugins/volume/exec"` - operatorManifest = getOpenshiftSCC(operatorNamespace) + manifest = m.settings.readManifestFromGithub("operator-openshift.yaml") + } else { + manifest = m.settings.readManifestFromGithub("operator.yaml") } - - operatorManifest = operatorManifest + `--- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: rook-ceph-system - namespace: ` + operatorNamespace + ` - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - - apps - - extensions - resources: - - pods - - configmaps - - services - - daemonsets - - statefulsets - - deployments - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - k8s.cni.cncf.io - resources: - - network-attachment-definitions - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-cluster-mgmt - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - - secrets - - pods - - pods/log - - services - - configmaps - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - apps - - extensions - resources: - - deployments - - daemonsets - verbs: - - get - - list - - watch - - create - - update - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-global - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - # Pod access is needed for fencing - - pods - # Node access is needed for determining nodes where mons should run - - nodes - - nodes/proxy - - services - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - # PVs and PVCs are managed by the Rook provisioner - - persistentvolumes - - persistentvolumeclaims - - endpoints - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - policy - - apps - - extensions - resources: - # This is for the clusterdisruption controller - - poddisruptionbudgets - # This is for both clusterdisruption and nodedrain controllers - - deployments - - replicasets - verbs: - - "*" -- apiGroups: - - healthchecking.openshift.io - resources: - - machinedisruptionbudgets - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - machine.openshift.io - resources: - - machines - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - csidrivers - verbs: - - create - - delete - - get - - update ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-cluster -rules: -- apiGroups: - - "" - resources: - - configmaps - - nodes - - nodes/proxy - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - list - - get - - watch ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-osd -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list ---- -# Aspects of Rook Ceph Agent that require access to secrets -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-agent-mount - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get ---- -# Aspects of ceph-mgr that require access to the system namespace -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-system -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-object-bucket - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - verbs: - - "*" - resources: - - secrets - - configmaps -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - "objectbucket.io" - verbs: - - "*" - resources: - - "*" ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-object-bucket -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-object-bucket -subjects: - - kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-system - namespace: ` + operatorNamespace + ` - labels: - operator: rook - storage-backend: ceph ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-system - namespace: ` + operatorNamespace + ` - labels: - operator: rook - storage-backend: ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-system -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-global - labels: - operator: rook - storage-backend: ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-global -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-rbd-plugin-sa - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-plugin-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: rbd-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-rbd-provisioner-sa - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-provisioner-runner -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-provisioner-role -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: rbd-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - namespace: ` + operatorNamespace + ` - name: rbd-external-provisioner-cfg -rules: - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "delete", "update"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-provisioner-role-cfg - namespace: ` + operatorNamespace + ` -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: Role - name: rbd-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-cephfs-plugin-sa - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-plugin-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: cephfs-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-cephfs-provisioner-sa - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-provisioner-runner -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-provisioner-role -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: cephfs-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - namespace: ` + operatorNamespace + ` - name: cephfs-external-provisioner-cfg -rules: - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "create", "delete"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-provisioner-role-cfg - namespace: ` + operatorNamespace + ` -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: ` + operatorNamespace + ` -roleRef: - kind: Role - name: cephfs-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: 00-rook-privileged - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' - seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' -spec: - privileged: true - allowedCapabilities: - # required by CSI - - SYS_ADMIN - # fsGroup - the flexVolume agent has fsGroup capabilities and could potentially be any group - fsGroup: - rule: RunAsAny - # runAsUser, supplementalGroups - Rook needs to run some pods as root - # Ceph pods could be run as the Ceph user, but that user isn't always known ahead of time - runAsUser: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - # seLinux - seLinux context is unknown ahead of time; set if this is well-known - seLinux: - rule: RunAsAny - volumes: - # recommended minimum set - - configMap - - downwardAPI - - emptyDir - - persistentVolumeClaim - - secret - - projected - # required for Rook - - hostPath - - flexVolume - # allowedHostPaths can be set to Rook's known host volume mount points when they are fully-known - # allowedHostPaths: - # - /run/udev # for OSD prep - # - /dev # for OSD prep - # - /var/lib/rook # or whatever the dataDirHostPath value is set to - # Ceph requires host IPC for setting up encrypted devices - hostIPC: true - # Ceph OSDs need to share the same PID namespace - hostPID: true - # hostNetwork can be set to 'false' if host networking isn't used - hostNetwork: true - hostPorts: - # Ceph messenger protocol v1 - - min: 6789 - max: 6790 # <- support old default port - # Ceph messenger protocol v2 - - min: 3300 - max: 3300 - # Ceph RADOS ports for OSDs, MDSes - - min: 6800 - max: 7300 - # # Ceph dashboard port HTTP (not recommended) - # - min: 7000 - # max: 7000 - # Ceph dashboard port HTTPS - - min: 8443 - max: 8443 - # Ceph mgr Prometheus Metrics - - min: 9283 - max: 9283 ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: 'psp:rook' -rules: - - apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - 00-rook-privileged - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-ceph-system-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-rbd-provisioner-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: ` + operatorNamespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-rbd-plugin-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-plugin-sa - namespace: ` + operatorNamespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-cephfs-provisioner-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: ` + operatorNamespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-cephfs-plugin-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-plugin-sa - namespace: ` + operatorNamespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-admission-controller - namespace: ` + operatorNamespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-admission-controller-role -rules: - - apiGroups: ["ceph.rook.io"] - resources: ["*"] - verbs: ["get", "watch", "list"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-admission-controller-rolebinding -subjects: - - kind: ServiceAccount - name: rook-ceph-admission-controller - apiGroup: "" - namespace: ` + operatorNamespace + ` -roleRef: - kind: ClusterRole - name: rook-ceph-admission-controller-role - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: rook-ceph-operator - namespace: ` + operatorNamespace + ` - labels: - operator: rook - storage-backend: ceph -spec: - replicas: 1 - selector: - matchLabels: - app: rook-ceph-operator - template: - metadata: - labels: - app: rook-ceph-operator - spec: - serviceAccountName: rook-ceph-system - containers: - - name: rook-ceph-operator - image: rook/ceph:` + m.imageTag + ` - args: ["ceph", "operator"] - env: - - name: ROOK_LOG_LEVEL - value: INFO - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ROOK_ENABLE_FLEX_DRIVER - value: "true" - - name: ROOK_CURRENT_NAMESPACE_ONLY - value: "false"` + openshiftEnv + ` - - name: ROOK_ENABLE_DISCOVERY_DAEMON - value: "true" - volumeMounts: - - mountPath: /var/lib/rook - name: rook-config - - mountPath: /etc/ceph - name: default-config-dir - volumes: - - name: rook-config - emptyDir: {} - - name: default-config-dir - emptyDir: {} ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: rook-ceph-operator-config - namespace: ` + operatorNamespace + ` -data: - ROOK_CSI_ENABLE_CEPHFS: "true" - ROOK_CSI_ENABLE_RBD: "true" - ROOK_CSI_ENABLE_GRPC_METRICS: "true" - ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true" - CSI_LOG_LEVEL: "5" -` - return operatorManifest + return m.settings.replaceOperatorSettings(manifest) } -// GetClusterRoles returns rook-cluster manifest -func (m *CephManifestsV1_5) GetClusterRoles(namespace, operatorNamespace string) string { - return `apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-osd - namespace: ` + namespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-osd - namespace: ` + namespace + ` -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: [ "get", "list", "watch", "create", "update", "delete" ] -- apiGroups: ["ceph.rook.io"] - resources: ["cephclusters", "cephclusters/finalizers"] - verbs: [ "get", "list", "create", "update", "delete" ] ---- -# Aspects of ceph-mgr that operate within the cluster's namespace -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr - namespace: ` + namespace + ` -rules: -- apiGroups: - - "" - resources: - - pods - - services - - pods/log - verbs: - - get - - list - - watch - - delete -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" ---- -# Allow the ceph osd to access cluster-wide resources necessary for determining their topology location -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-osd-` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-osd -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: ` + namespace + ` ---- -# Allow the ceph mgr to access cluster-wide resources necessary for the mgr modules -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-cluster-` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-mgr-cluster -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -rules: -- apiGroups: - - "" - resources: - - pods - - configmaps - verbs: - - get - - list - - watch - - create - - update - - delete ---- -# Allow the operator to create resources in this cluster's namespace -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cluster-mgmt - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-cluster-mgmt -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: ` + operatorNamespace + ` ---- -# Allow the osd pods in this namespace to work with configmaps -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-osd - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-osd -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: ` + namespace + ` ---- -# Allow the ceph mgr to access the cluster-specific resources necessary for the mgr modules -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-mgr -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -# Allow the ceph mgr to access the rook system resources necessary for the mgr modules -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-system ` + namespace + ` - namespace: ` + operatorNamespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-mgr-system -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-cmd-reporter -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-default-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: default - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-osd-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-mgr-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-cmd-reporter-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -` +// GetCommon returns rook-cluster manifest +func (m *CephManifestsV1_5) GetCommon() string { + return m.settings.readManifestFromGithub("common.yaml") } +// GetRookToolBox returns rook-toolbox manifest +func (m *CephManifestsV1_5) GetToolbox() string { + manifest := strings.ReplaceAll(m.settings.readManifestFromGithub("direct-mount.yaml"), "name: rook-direct-mount", "name: rook-ceph-tools") + return strings.ReplaceAll(manifest, "app: rook-direct-mount", "app: rook-ceph-tools") +} + +func (m *CephManifestsV1_5) GetCommonExternal() string { + return m.settings.readManifestFromGithub("common-external.yaml") +} + +//********************************************************************************** +//********************************************************************************** +// After a release, replace the methods below this separator from the +// ceph_manifests.go. Methods above this separator do not need to be +// copied since they will load them directly from github. +//********************************************************************************** +//********************************************************************************** + // GetRookCluster returns rook-cluster manifest -func (m *CephManifestsV1_5) GetRookCluster(settings *clusterSettings) string { - store := "# storeType not specified; Rook will use default store types" - if settings.StoreType != "" { - store = `storeType: "` + settings.StoreType + `"` - } - +func (m *CephManifestsV1_5) GetCephCluster() string { crushRoot := "# crushRoot not specified; Rook will use `default`" - if settings.Mons == 1 { + if m.settings.Mons == 1 { crushRoot = `crushRoot: "custom-root"` } - if settings.UsePVCs { + if m.settings.UsePVC { return `apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: # set the name to something different from the namespace - name: ` + settings.ClusterName + ` - namespace: ` + settings.Namespace + ` + name: ` + m.settings.ClusterName + ` + namespace: ` + m.settings.Namespace + ` spec: - dataDirHostPath: ` + settings.DataDirHostPath + ` + dataDirHostPath: ` + m.settings.DataDirHostPath + ` mon: - count: ` + strconv.Itoa(settings.Mons) + ` + count: ` + strconv.Itoa(m.settings.Mons) + ` allowMultiplePerNode: true volumeClaimTemplate: spec: - storageClassName: ` + settings.StorageClassName + ` + storageClassName: ` + m.settings.StorageClassName + ` resources: requests: storage: 5Gi cephVersion: - image: ` + settings.CephVersion.Image + ` - allowUnsupported: ` + strconv.FormatBool(settings.CephVersion.AllowUnsupported) + ` + image: ` + m.settings.CephVersion.Image + ` + allowUnsupported: ` + strconv.FormatBool(m.settings.CephVersion.AllowUnsupported) + ` skipUpgradeChecks: false continueUpgradeAfterChecksEvenIfNotHealthy: false dashboard: @@ -1253,7 +130,7 @@ spec: resources: requests: storage: 10Gi - storageClassName: ` + settings.StorageClassName + ` + storageClassName: ` + m.settings.StorageClassName + ` volumeMode: Block accessModes: - ReadWriteOnce @@ -1268,28 +145,27 @@ spec: return `apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: - name: ` + settings.ClusterName + ` - namespace: ` + settings.Namespace + ` + name: ` + m.settings.ClusterName + ` + namespace: ` + m.settings.Namespace + ` spec: cephVersion: - image: ` + settings.CephVersion.Image + ` - allowUnsupported: ` + strconv.FormatBool(settings.CephVersion.AllowUnsupported) + ` - dataDirHostPath: ` + settings.DataDirHostPath + ` + image: ` + m.settings.CephVersion.Image + ` + allowUnsupported: ` + strconv.FormatBool(m.settings.CephVersion.AllowUnsupported) + ` + dataDirHostPath: ` + m.settings.DataDirHostPath + ` network: hostNetwork: false mon: - count: ` + strconv.Itoa(settings.Mons) + ` + count: ` + strconv.Itoa(m.settings.Mons) + ` allowMultiplePerNode: true dashboard: enabled: true skipUpgradeChecks: true metadataDevice: storage: - useAllNodes: ` + strconv.FormatBool(!settings.skipOSDCreation) + ` - useAllDevices: ` + strconv.FormatBool(!settings.skipOSDCreation) + ` + useAllNodes: ` + strconv.FormatBool(!m.settings.SkipOSDCreation) + ` + useAllDevices: ` + strconv.FormatBool(!m.settings.SkipOSDCreation) + ` deviceFilter: '' config: - ` + store + ` databaseSizeMB: "1024" journalSizeMB: "1024" mgr: @@ -1309,174 +185,44 @@ spec: interval: 5s` } -// GetRookToolBox returns rook-toolbox manifest -func (m *CephManifestsV1_5) GetRookToolBox(namespace string) string { - return `apiVersion: v1 -kind: Pod -metadata: - name: rook-ceph-tools - namespace: ` + namespace + ` -spec: - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: rook-ceph-tools - image: rook/ceph:` + m.imageTag + ` - imagePullPolicy: IfNotPresent - command: ["/tini"] - args: ["-g", "--", "/usr/local/bin/toolbox.sh"] - env: - - name: ROOK_CEPH_USERNAME - valueFrom: - secretKeyRef: - name: rook-ceph-mon - key: ceph-username - - name: ROOK_CEPH_SECRET - valueFrom: - secretKeyRef: - name: rook-ceph-mon - key: ceph-secret - securityContext: - privileged: true - volumeMounts: - - mountPath: /dev - name: dev - - mountPath: /sys/bus - name: sysbus - - mountPath: /lib/modules - name: libmodules - - name: mon-endpoint-volume - mountPath: /etc/rook - hostNetwork: false - volumes: - - name: dev - hostPath: - path: /dev - - name: sysbus - hostPath: - path: /sys/bus - - name: libmodules - hostPath: - path: /lib/modules - - name: mon-endpoint-volume - configMap: - name: rook-ceph-mon-endpoints - items: - - key: data - path: mon-endpoints` -} - -func (m *CephManifestsV1_5) GetBlockSnapshotClass(snapshotClassName, namespace, operatorNamespace, reclaimPolicy string) string { +func (m *CephManifestsV1_5) GetBlockSnapshotClass(snapshotClassName, reclaimPolicy string) string { // Create a CSI driver snapshotclass return ` apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshotClass metadata: name: ` + snapshotClassName + ` -driver: ` + operatorNamespace + `.rbd.csi.ceph.com +driver: ` + m.settings.OperatorNamespace + `.rbd.csi.ceph.com deletionPolicy: ` + reclaimPolicy + ` parameters: - clusterID: ` + namespace + ` + clusterID: ` + m.settings.Namespace + ` csi.storage.k8s.io/snapshotter-secret-name: rook-csi-rbd-provisioner - csi.storage.k8s.io/snapshotter-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/snapshotter-secret-namespace: ` + m.settings.Namespace + ` ` } -func (m *CephManifestsV1_5) GetFileStorageSnapshotClass(snapshotClassName, namespace, operatorNamespace, reclaimPolicy string) string { +func (m *CephManifestsV1_5) GetFileStorageSnapshotClass(snapshotClassName, reclaimPolicy string) string { // Create a CSI driver snapshotclass return ` apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshotClass metadata: name: ` + snapshotClassName + ` -driver: ` + operatorNamespace + `.cephfs.csi.ceph.com +driver: ` + m.settings.OperatorNamespace + `.cephfs.csi.ceph.com deletionPolicy: ` + reclaimPolicy + ` parameters: - clusterID: ` + namespace + ` + clusterID: ` + m.settings.Namespace + ` csi.storage.k8s.io/snapshotter-secret-name: rook-csi-cephfs-provisioner - csi.storage.k8s.io/snapshotter-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/snapshotter-secret-namespace: ` + m.settings.Namespace + ` ` } -func (m *CephManifestsV1_5) GetPVCRestore(claimName, snapshotName, namespace, storageClassName, accessModes, size string) string { - return `apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ` + claimName + ` - namespace: ` + namespace + ` -spec: - storageClassName: ` + storageClassName + ` - dataSource: - name: ` + snapshotName + ` - kind: VolumeSnapshot - apiGroup: snapshot.storage.k8s.io - accessModes: - - ` + accessModes + ` - resources: - requests: - storage: ` + size -} - -func (m *CephManifestsV1_5) GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, accessModes, size string) string { - return `apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ` + cloneClaimName + ` - namespace: ` + namespace + ` -spec: - storageClassName: ` + storageClassName + ` - dataSource: - name: ` + parentClaimName + ` - kind: PersistentVolumeClaim - accessModes: - - ` + accessModes + ` - resources: - requests: - storage: ` + size -} - -func (m *CephManifestsV1_5) GetSnapshot(snapshotName, claimName, snapshotClassName, namespace string) string { - return `apiVersion: snapshot.storage.k8s.io/v1beta1 -kind: VolumeSnapshot -metadata: - name: ` + snapshotName + ` - namespace: ` + namespace + ` -spec: - volumeSnapshotClassName: ` + snapshotClassName + ` - source: - persistentVolumeClaimName: ` + claimName -} - -func (m *CephManifestsV1_5) GetPod(podName, claimName, namespace, mountPath string, readOnly bool) string { - return ` -apiVersion: v1 -kind: Pod -metadata: - name: ` + podName + ` - namespace: ` + namespace + ` -spec: - containers: - - name: ` + podName + ` - image: busybox - command: ["/bin/sleep", "infinity"] - imagePullPolicy: IfNotPresent - volumeMounts: - - mountPath: ` + mountPath + ` - name: csivol - volumes: - - name: csivol - persistentVolumeClaim: - claimName: ` + claimName + ` - readOnly: ` + strconv.FormatBool(readOnly) + ` - restartPolicy: Never -` -} - -func (m *CephManifestsV1_5) GetBlockPoolDef(poolName string, namespace string, replicaSize string) string { +func (m *CephManifestsV1_5) GetBlockPool(poolName, replicaSize string) string { return `apiVersion: ceph.rook.io/v1 kind: CephBlockPool metadata: name: ` + poolName + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: replicated: size: ` + replicaSize + ` @@ -1495,23 +241,23 @@ spec: interval: 10s` } -func (m *CephManifestsV1_5) GetBlockStorageClassDef(csi bool, poolName, storageClassName, reclaimPolicy, namespace, operatorNamespace string) string { +func (m *CephManifestsV1_5) GetBlockStorageClass(poolName, storageClassName, reclaimPolicy string) string { // Create a CSI driver storage class - if csi { + if m.settings.UseCSI { return ` apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ` + storageClassName + ` -provisioner: ` + operatorNamespace + `.rbd.csi.ceph.com +provisioner: ` + m.settings.OperatorNamespace + `.rbd.csi.ceph.com reclaimPolicy: ` + reclaimPolicy + ` parameters: pool: ` + poolName + ` - clusterID: ` + namespace + ` + clusterID: ` + m.settings.Namespace + ` csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner - csi.storage.k8s.io/provisioner-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/provisioner-secret-namespace: ` + m.settings.Namespace + ` csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node - csi.storage.k8s.io/node-stage-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/node-stage-secret-namespace: ` + m.settings.Namespace + ` imageFeatures: layering csi.storage.k8s.io/fstype: ext4 ` @@ -1526,10 +272,10 @@ allowVolumeExpansion: true reclaimPolicy: ` + reclaimPolicy + ` parameters: blockPool: ` + poolName + ` - clusterNamespace: ` + namespace + clusterNamespace: ` + m.settings.Namespace } -func (m *CephManifestsV1_5) GetFileStorageClassDef(fsName, storageClassName, operatorNamespace, namespace string) string { +func (m *CephManifestsV1_5) GetFileStorageClass(fsName, storageClassName string) string { // Create a CSI driver storage class csiCephFSNodeSecret := "rook-csi-cephfs-node" //nolint:gosec // We safely suppress gosec in tests file csiCephFSProvisionerSecret := "rook-csi-cephfs-provisioner" //nolint:gosec // We safely suppress gosec in tests file @@ -1538,40 +284,25 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ` + storageClassName + ` -provisioner: ` + operatorNamespace + `.cephfs.csi.ceph.com +provisioner: ` + m.settings.OperatorNamespace + `.cephfs.csi.ceph.com parameters: - clusterID: ` + namespace + ` + clusterID: ` + m.settings.Namespace + ` fsName: ` + fsName + ` pool: ` + fsName + `-data0 csi.storage.k8s.io/provisioner-secret-name: ` + csiCephFSProvisionerSecret + ` - csi.storage.k8s.io/provisioner-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/provisioner-secret-namespace: ` + m.settings.Namespace + ` csi.storage.k8s.io/node-stage-secret-name: ` + csiCephFSNodeSecret + ` - csi.storage.k8s.io/node-stage-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/node-stage-secret-namespace: ` + m.settings.Namespace + ` ` } -func (m *CephManifestsV1_5) GetPVC(claimName, namespace, storageClassName, accessModes, size string) string { - return `apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ` + claimName + ` - namespace: ` + namespace + ` -spec: - storageClassName: ` + storageClassName + ` - accessModes: - - ` + accessModes + ` - resources: - requests: - storage: ` + size -} - // GetFilesystem returns the manifest to create a Rook filesystem resource with the given config. -func (m *CephManifestsV1_5) GetFilesystem(namespace, name string, activeCount int) string { +func (m *CephManifestsV1_5) GetFilesystem(name string, activeCount int) string { return `apiVersion: ceph.rook.io/v1 kind: CephFilesystem metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: metadataPool: replicated: @@ -1588,12 +319,12 @@ spec: } // GetFilesystem returns the manifest to create a Rook Ceph NFS resource with the given config. -func (m *CephManifestsV1_5) GetNFS(namespace, name, pool string, count int) string { +func (m *CephManifestsV1_5) GetNFS(name, pool string, count int) string { return `apiVersion: ceph.rook.io/v1 kind: CephNFS metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: rados: pool: ` + pool + ` @@ -1602,12 +333,12 @@ spec: active: ` + strconv.Itoa(count) } -func (m *CephManifestsV1_5) GetObjectStore(namespace, name string, replicaCount, port int) string { +func (m *CephManifestsV1_5) GetObjectStore(name string, replicaCount, port int) string { return `apiVersion: ceph.rook.io/v1 kind: CephObjectStore metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: metadataPool: replicated: @@ -1630,33 +361,33 @@ spec: ` } -func (m *CephManifestsV1_5) GetObjectStoreUser(namespace, name string, displayName string, store string) string { +func (m *CephManifestsV1_5) GetObjectStoreUser(name string, displayName string, store string) string { return `apiVersion: ceph.rook.io/v1 kind: CephObjectStoreUser metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: displayName: ` + displayName + ` store: ` + store } //GetBucketStorageClass returns the manifest to create object bucket -func (m *CephManifestsV1_5) GetBucketStorageClass(storeNameSpace string, storeName string, storageClassName string, reclaimPolicy string, region string) string { +func (m *CephManifestsV1_5) GetBucketStorageClass(storeName, storageClassName, reclaimPolicy, region string) string { return `apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ` + storageClassName + ` -provisioner: ` + storeNameSpace + `.ceph.rook.io/bucket +provisioner: ` + m.settings.Namespace + `.ceph.rook.io/bucket reclaimPolicy: ` + reclaimPolicy + ` parameters: objectStoreName: ` + storeName + ` - objectStoreNamespace: ` + storeNameSpace + ` + objectStoreNamespace: ` + m.settings.Namespace + ` region: ` + region } -//GetObc returns the manifest to create object bucket claim -func (m *CephManifestsV1_5) GetObc(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string { +//GetOBC returns the manifest to create object bucket claim +func (m *CephManifestsV1_5) GetOBC(claimName, storageClassName, objectBucketName, maxObject string, varBucketName bool) string { bucketParameter := "generateBucketName" if varBucketName { bucketParameter = "bucketName" @@ -1672,7 +403,7 @@ spec: maxObjects: "` + maxObject + `"` } -func (m *CephManifestsV1_5) GetClient(namespace string, claimName string, caps map[string]string) string { +func (m *CephManifestsV1_5) GetClient(claimName string, caps map[string]string) string { clientCaps := []string{} for name, cap := range caps { str := name + ": " + cap @@ -1682,87 +413,31 @@ func (m *CephManifestsV1_5) GetClient(namespace string, claimName string, caps m kind: CephClient metadata: name: ` + claimName + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: caps: ` + strings.Join(clientCaps, "\n ") } -func (m *CephManifestsV1_5) GetClusterExternalRoles(namespace, firstClusterNamespace string) string { - return `apiVersion: v1 -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cluster-mgmt - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-cluster-mgmt -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: ` + firstClusterNamespace + ` ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-cmd-reporter -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -rules: -- apiGroups: - - "" - resources: - - pods - - configmaps - verbs: - - get - - list - - watch - - create - - update - - delete` -} - -func (m *CephManifestsV1_5) GetRookExternalCluster(settings *clusterExternalSettings) string { +func (m *CephManifestsV1_5) GetExternalCephCluster() string { return `apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: - name: ` + settings.Namespace + ` - namespace: ` + settings.Namespace + ` + name: ` + m.settings.Namespace + ` + namespace: ` + m.settings.Namespace + ` spec: external: enable: true - dataDirHostPath: ` + settings.DataDirHostPath + `` + dataDirHostPath: ` + m.settings.DataDirHostPath + `` } // GetRBDMirror returns the manifest to create a Rook Ceph RBD Mirror resource with the given config. -func (m *CephManifestsV1_5) GetRBDMirror(namespace, name string, count int) string { +func (m *CephManifestsV1_5) GetRBDMirror(name string, count int) string { return `apiVersion: ceph.rook.io/v1 kind: CephRBDMirror metadata: name: ` + name + ` - namespace: ` + namespace + ` + namespace: ` + m.settings.Namespace + ` spec: count: ` + strconv.Itoa(count) } diff --git a/tests/framework/installer/ceph_settings.go b/tests/framework/installer/ceph_settings.go new file mode 100644 index 000000000..e617f460d --- /dev/null +++ b/tests/framework/installer/ceph_settings.go @@ -0,0 +1,126 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installer + +import ( + "fmt" + "io/ioutil" + "net/http" + "path" + "strings" + + "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/tests/framework/utils" +) + +// TestCephSettings struct for handling panic and test suite tear down +type TestCephSettings struct { + DataDirHostPath string + ClusterName string + Namespace string + OperatorNamespace string + StorageClassName string + UseHelm bool + UsePVC bool + Mons int + UseCrashPruner bool + MultipleMgrs bool + SkipOSDCreation bool + UseCSI bool + EnableDiscovery bool + EnableAdmissionController bool + IsExternal bool + RookVersion string + CephVersion cephv1.CephVersionSpec +} + +func (s *TestCephSettings) readManifest(filename string) string { + rootDir, err := utils.FindRookRoot() + if err != nil { + panic(err) + } + manifest := path.Join(rootDir, "cluster/examples/kubernetes/ceph", filename) + logger.Infof("Reading manifest: %s", manifest) + contents, err := ioutil.ReadFile(manifest) + if err != nil { + panic(errors.Wrapf(err, "failed to read manifest at %s", manifest)) + } + return replaceNamespaces(manifest, string(contents), s.OperatorNamespace, s.Namespace) +} + +func (s *TestCephSettings) readManifestFromGithub(filename string) string { + return s.readManifestFromGithubWithClusterNamespace(filename, s.Namespace) +} + +func (s *TestCephSettings) readManifestFromGithubWithClusterNamespace(filename, clusterNamespace string) string { + url := fmt.Sprintf("https://raw.githubusercontent.com/rook/rook/%s/cluster/examples/kubernetes/ceph/%s", s.RookVersion, filename) + logger.Infof("Retrieving manifest: %s", url) + // #nosec G107 This is only test code and is expected to read from a url + response, err := http.Get(url) + if err != nil { + panic(errors.Wrapf(err, "failed to read manifest from %s", url)) + } + defer response.Body.Close() + + content, err := ioutil.ReadAll(response.Body) + if err != nil { + panic(errors.Wrapf(err, "failed to read content from %s", url)) + } + return replaceNamespaces(url, string(content), s.OperatorNamespace, clusterNamespace) +} + +func (s *TestCephSettings) replaceOperatorSettings(manifest string) string { + manifest = strings.ReplaceAll(manifest, `# CSI_LOG_LEVEL: "0"`, `CSI_LOG_LEVEL: "5"`) + manifest = strings.ReplaceAll(manifest, `ROOK_ENABLE_DISCOVERY_DAEMON: "false"`, fmt.Sprintf(`ROOK_ENABLE_DISCOVERY_DAEMON: "%t"`, s.EnableDiscovery)) + manifest = strings.ReplaceAll(manifest, `ROOK_ENABLE_FLEX_DRIVER: "false"`, fmt.Sprintf(`ROOK_ENABLE_FLEX_DRIVER: "%t"`, !s.UseCSI)) + manifest = strings.ReplaceAll(manifest, `ROOK_CSI_ENABLE_CEPHFS: "true"`, fmt.Sprintf(`ROOK_CSI_ENABLE_CEPHFS: "%t"`, s.UseCSI)) + manifest = strings.ReplaceAll(manifest, `ROOK_CSI_ENABLE_RBD: "true"`, fmt.Sprintf(`ROOK_CSI_ENABLE_RBD: "%t"`, s.UseCSI)) + return manifest +} + +func replaceNamespaces(name, manifest, operatorNamespace, clusterNamespace string) string { + // RBAC and related namespaces + manifest = strings.ReplaceAll(manifest, "rook-ceph # namespace:operator", operatorNamespace) + manifest = strings.ReplaceAll(manifest, "rook-ceph # namespace:cluster", clusterNamespace) + manifest = strings.ReplaceAll(manifest, "rook-ceph-external # namespace:cluster", clusterNamespace) + // Double space only needed for v1.5 upgrade test + manifest = strings.ReplaceAll(manifest, "rook-ceph # namespace:operator", operatorNamespace) + + // SCC namespaces for operator and Ceph daemons + manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-system # serviceaccount:namespace:operator", operatorNamespace+":rook-ceph-system") + manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-mgr # serviceaccount:namespace:cluster", clusterNamespace+":rook-ceph-mgr") + manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-osd # serviceaccount:namespace:cluster", clusterNamespace+":rook-ceph-osd") + + // SCC namespaces for CSI driver + manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-rbd-plugin-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-rbd-plugin-sa") + manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-rbd-provisioner-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-rbd-provisioner-sa") + manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-cephfs-plugin-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-cephfs-plugin-sa") + manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-cephfs-provisioner-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-cephfs-provisioner-sa") + + // CSI Drivers + manifest = strings.ReplaceAll(manifest, "rook-ceph.cephfs.csi.ceph.com # driver:namespace:operator", operatorNamespace+".cephfs.csi.ceph.com") + manifest = strings.ReplaceAll(manifest, "rook-ceph.rbd.csi.ceph.com # driver:namespace:operator", operatorNamespace+".rbd.csi.ceph.com") + + // Bucket storage class + manifest = strings.ReplaceAll(manifest, "rook-ceph.ceph.rook.io/bucket # driver:namespace:cluster", clusterNamespace+".ceph.rook.io/bucket") + if strings.Contains(manifest, "namespace:operator") || strings.Contains(manifest, "namespace:cluster") || strings.Contains(manifest, "driver:namespace:") || strings.Contains(manifest, "serviceaccount:namespace:") { + logger.Infof("BAD MANIFEST:\n%s", manifest) + panic(fmt.Sprintf("manifest %s still contains a namespace identifier", name)) + } + return manifest +} diff --git a/tests/framework/installer/general_manifests.go b/tests/framework/installer/general_manifests.go new file mode 100644 index 000000000..e544cbf78 --- /dev/null +++ b/tests/framework/installer/general_manifests.go @@ -0,0 +1,110 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installer + +import ( + "strconv" +) + +func GetPodWithVolume(podName, claimName, namespace, mountPath string, readOnly bool) string { + return ` +apiVersion: v1 +kind: Pod +metadata: + name: ` + podName + ` + namespace: ` + namespace + ` +spec: + containers: + - name: ` + podName + ` + image: busybox + command: ["/bin/sleep", "infinity"] + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: ` + mountPath + ` + name: csivol + volumes: + - name: csivol + persistentVolumeClaim: + claimName: ` + claimName + ` + readOnly: ` + strconv.FormatBool(readOnly) + ` + restartPolicy: Never +` +} + +func GetPVC(claimName, namespace, storageClassName, accessModes, size string) string { + return `apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ` + claimName + ` + namespace: ` + namespace + ` +spec: + storageClassName: ` + storageClassName + ` + accessModes: + - ` + accessModes + ` + resources: + requests: + storage: ` + size +} + +func GetPVCRestore(claimName, snapshotName, namespace, storageClassName, accessModes, size string) string { + return `apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ` + claimName + ` + namespace: ` + namespace + ` +spec: + storageClassName: ` + storageClassName + ` + dataSource: + name: ` + snapshotName + ` + kind: VolumeSnapshot + apiGroup: snapshot.storage.k8s.io + accessModes: + - ` + accessModes + ` + resources: + requests: + storage: ` + size +} + +func GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, accessModes, size string) string { + return `apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ` + cloneClaimName + ` + namespace: ` + namespace + ` +spec: + storageClassName: ` + storageClassName + ` + dataSource: + name: ` + parentClaimName + ` + kind: PersistentVolumeClaim + accessModes: + - ` + accessModes + ` + resources: + requests: + storage: ` + size +} + +func GetSnapshot(snapshotName, claimName, snapshotClassName, namespace string) string { + return `apiVersion: snapshot.storage.k8s.io/v1beta1 +kind: VolumeSnapshot +metadata: + name: ` + snapshotName + ` + namespace: ` + namespace + ` +spec: + volumeSnapshotClassName: ` + snapshotClassName + ` + source: + persistentVolumeClaimName: ` + claimName +} diff --git a/tests/framework/installer/installer.go b/tests/framework/installer/installer.go index 4c7d0f673..66bf6f567 100644 --- a/tests/framework/installer/installer.go +++ b/tests/framework/installer/installer.go @@ -45,11 +45,6 @@ var ( deleteFromStdinArgs = append(deleteArgs, "-") ) -type TestSuite interface { - Setup() - Teardown() -} - func SkipTestSuite(name string) bool { testsToRun := testStorageProvider() // jenkins passes "null" if the env var is not set. diff --git a/tests/framework/installer/nfs_installer.go b/tests/framework/installer/nfs_installer.go index 578d1548b..d70bf403b 100644 --- a/tests/framework/installer/nfs_installer.go +++ b/tests/framework/installer/nfs_installer.go @@ -166,7 +166,7 @@ func (h *NFSInstaller) UninstallNFSServer(systemNamespace, namespace string) { _, err := h.k8shelper.RookClientset.NfsV1alpha1().NFSServers(namespace).Get(ctx, namespace, metav1.GetOptions{}) return err } - err = h.k8shelper.WaitForCustomResourceDeletion(namespace, crdCheckerFunc) + err = h.k8shelper.WaitForCustomResourceDeletion(namespace, namespace, crdCheckerFunc) checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace)) err = h.k8shelper.DeleteResource("namespace", namespace) diff --git a/tests/framework/utils/helm_helper.go b/tests/framework/utils/helm_helper.go index 41832bb27..9264bd4d7 100644 --- a/tests/framework/utils/helm_helper.go +++ b/tests/framework/utils/helm_helper.go @@ -62,7 +62,7 @@ func (h *HelmHelper) InstallLocalRookHelmChart(namespace, chartSettings string) return errors.Wrap(err, "failed to find rook root") } chartDir := path.Join(rootDir, "cluster/charts/rook-ceph/") - cmdArgs := []string{"install", HelmDeployName, chartDir, "-f", path.Join(chartDir, "values.yaml")} + cmdArgs := []string{"install", "--create-namespace", HelmDeployName, chartDir, "-f", path.Join(chartDir, "values.yaml")} if namespace != "" { cmdArgs = append(cmdArgs, "--namespace", namespace) } diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index 8498a4571..25cbbc94a 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -347,7 +347,12 @@ func (k8sh *K8sHelper) ResourceOperationFromTemplate(action string, podDefinitio // ResourceOperation performs a kubectl action on a pod definition func (k8sh *K8sHelper) ResourceOperation(action string, manifest string) error { args := []string{action, "-f", "-"} - logger.Infof("kubectl %s manifest:\n%s", action, manifest) + maxManifestCharsToPrint := 4000 + if len(manifest) > maxManifestCharsToPrint { + logger.Infof("kubectl %s manifest (too long to print)", action) + } else { + logger.Infof("kubectl %s manifest:\n%s", action, manifest) + } _, err := k8sh.KubectlWithStdin(manifest, args...) if err == nil { return nil @@ -381,23 +386,23 @@ func (k8sh *K8sHelper) DeleteResource(args ...string) error { } // WaitForCustomResourceDeletion waits for the CRD deletion -func (k8sh *K8sHelper) WaitForCustomResourceDeletion(namespace string, checkerFunc func() error) error { +func (k8sh *K8sHelper) WaitForCustomResourceDeletion(namespace, name string, checkerFunc func() error) error { // wait for the operator to finalize and delete the CRD for i := 0; i < 60; i++ { err := checkerFunc() if err == nil { - logger.Infof("custom resource %s still exists", namespace) + logger.Infof("custom resource %q in namespace %q still exists", name, namespace) time.Sleep(2 * time.Second) continue } if kerrors.IsNotFound(err) { - logger.Infof("custom resource %s deleted", namespace) + logger.Infof("custom resource %q in namespace %s deleted", name, namespace) return nil } return err } - logger.Errorf("gave up deleting custom resource %s", namespace) + logger.Errorf("gave up deleting custom resource %q ", name) return nil } @@ -1218,6 +1223,7 @@ func (k8sh *K8sHelper) IsPodInExpectedState(podNamePattern string, namespace str } } + logger.Infof("waiting for pod with label app=%s in namespace %q to be in state %q...", podNamePattern, namespace, state) time.Sleep(RetryInterval * time.Second) } diff --git a/tests/integration/ceph_base_block_test.go b/tests/integration/ceph_base_block_test.go index aa68f4a38..a4e8e2ef9 100644 --- a/tests/integration/ceph_base_block_test.go +++ b/tests/integration/ceph_base_block_test.go @@ -24,7 +24,6 @@ import ( "testing" "time" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/tests/framework/clients" @@ -269,8 +268,6 @@ func runBlockCSITest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite. poolName := "replicapool" storageClassName := "rook-ceph-block" blockName := "block-pv-claim" - systemNamespace := installer.SystemNamespace(namespace) - podNameWithPVRetained := "block-test-retained" poolNameRetained := "replicapoolretained" storageClassNameRetained := "rook-ceph-block-retained" @@ -287,10 +284,10 @@ func runBlockCSITest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite. assert.Equal(s.T(), 0, len(initBlockImages), "there should not already be any images in the pool") logger.Infof("step 1: Create block storage") - err := helper.BlockClient.CreateStorageClassAndPVC(true, defaultNamespace, namespace, systemNamespace, poolName, storageClassName, "Delete", blockName, "ReadWriteOnce") + err := helper.BlockClient.CreateStorageClassAndPVC(defaultNamespace, poolName, storageClassName, "Delete", blockName, "ReadWriteOnce") require.NoError(s.T(), err) require.NoError(s.T(), retryBlockImageCountCheck(helper, clusterInfo, 1), "Make sure a new block is created") - err = helper.BlockClient.CreateStorageClassAndPVC(true, defaultNamespace, namespace, systemNamespace, poolNameRetained, storageClassNameRetained, "Retain", blockNameRetained, "ReadWriteOnce") + err = helper.BlockClient.CreateStorageClassAndPVC(defaultNamespace, poolNameRetained, storageClassNameRetained, "Retain", blockNameRetained, "ReadWriteOnce") require.NoError(s.T(), err) require.NoError(s.T(), retryBlockImageCountCheck(helper, clusterInfo, 2), "Make sure another new block is created") logger.Infof("Block Storage created successfully") @@ -405,7 +402,8 @@ func restartOSDPods(k8sh *utils.K8sHelper, s suite.Suite, namespace string) { options := metav1.DeleteOptions{} err := k8sh.Clientset.CoreV1().Pods(namespace).Delete(ctx, pod.Name, options) assert.NoError(s.T(), err) - + } + for _, pod := range pods.Items { logger.Infof("Waiting for osd pod %s to be deleted", pod.Name) deleted := k8sh.WaitUntilPodIsDeleted(pod.Name, namespace) assert.True(s.T(), deleted) @@ -417,20 +415,20 @@ func restartOSDPods(k8sh *utils.K8sHelper, s suite.Suite, namespace string) { assert.NoError(s.T(), err) } -func runBlockCSITestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, clusterNamespace, systemNamespace string, version cephv1.CephVersionSpec) { +func runBlockCSITestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings) { checkSkipCSITest(s.T(), k8sh) logger.Infof("Block Storage End to End Integration Test - create storageclass,pool and pvc") - logger.Infof("Running on Rook Cluster %s", clusterNamespace) - clusterInfo := client.AdminClusterInfo(clusterNamespace) + logger.Infof("Running on Rook Cluster %s", settings.Namespace) + clusterInfo := client.AdminClusterInfo(settings.Namespace) poolName := "rookpool" storageClassName := "rook-ceph-block-lite" blockName := "test-block-claim-lite" podName := "test-pod-lite" defer blockTestDataCleanUp(helper, k8sh, s, clusterInfo, poolName, storageClassName, blockName, podName, true) - setupBlockLite(helper, k8sh, s, clusterInfo, systemNamespace, poolName, storageClassName, blockName, podName, version) + setupBlockLite(helper, k8sh, s, clusterInfo, poolName, storageClassName, blockName, podName) if !skipSnapshotTest(k8sh) { - blockCSISnapshotTest(helper, k8sh, s, storageClassName, clusterNamespace) + blockCSISnapshotTest(helper, k8sh, s, storageClassName, settings.Namespace) } if !skipCloneTest(k8sh) { @@ -439,7 +437,7 @@ func runBlockCSITestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s su } func setupBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, clusterInfo *client.ClusterInfo, - systemNamespace, poolName, storageClassName, blockName, podName string, version cephv1.CephVersionSpec) { + poolName, storageClassName, blockName, podName string) { // Check initial number of blocks initialBlocks, err := helper.BlockClient.ListAllImages(clusterInfo) @@ -449,7 +447,7 @@ func setupBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S logger.Infof("step : Create Pool,StorageClass and PVC") - err = helper.BlockClient.CreateStorageClassAndPVC(true, defaultNamespace, clusterInfo.Namespace, systemNamespace, poolName, storageClassName, "Delete", blockName, "ReadWriteOnce") + err = helper.BlockClient.CreateStorageClassAndPVC(defaultNamespace, poolName, storageClassName, "Delete", blockName, "ReadWriteOnce") require.NoError(s.T(), err) require.True(s.T(), k8sh.WaitUntilPVCIsBound(defaultNamespace, blockName)) diff --git a/tests/integration/ceph_base_deploy_test.go b/tests/integration/ceph_base_deploy_test.go index 5971a707c..204de0e86 100644 --- a/tests/integration/ceph_base_deploy_test.go +++ b/tests/integration/ceph_base_deploy_test.go @@ -25,7 +25,6 @@ import ( "testing" "github.com/coreos/pkg/capnslog" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" @@ -87,36 +86,15 @@ func checkIfRookClusterIsHealthy(s suite.Suite, testClient *clients.TestClient, require.Nil(s.T(), err) } -func HandlePanics(r interface{}, op installer.TestSuite, t func() *testing.T) { +func HandlePanics(r interface{}, uninstaller func(), t func() *testing.T) { if r != nil { logger.Infof("unexpected panic occurred during test %s, --> %v", t().Name(), r) t().Fail() - op.Teardown() + uninstaller() t().FailNow() } } -// TestCluster struct for handling panic and test suite tear down -type TestCluster struct { - installer *installer.CephInstaller - kh *utils.K8sHelper - helper *clients.TestClient - T func() *testing.T - clusterName string - namespace string - storeType string - storageClassName string - useHelm bool - usePVC bool - mons int - rbdMirrorWorkers int - rookCephCleanup bool - skipOSDCreation bool - minimalMatrixK8sVersion string - rookVersion string - cephVersion cephv1.CephVersionSpec -} - func checkIfShouldRunForMinimalTestMatrix(t func() *testing.T, k8sh *utils.K8sHelper, version string) { testArgs := os.Getenv("TEST_ARGUMENTS") if !strings.Contains(testArgs, "min-test-matrix") { @@ -140,50 +118,27 @@ func checkIfShouldRunForMinimalTestMatrix(t func() *testing.T, k8sh *utils.K8sHe logger.Infof("Running test suite since kube version is %q", kubeVersion) } -// StartTestCluster creates new instance of TestCluster struct -func StartTestCluster(t func() *testing.T, cluster *TestCluster) (*TestCluster, *utils.K8sHelper) { - kh, err := utils.CreateK8sHelper(t) +// StartTestCluster creates new instance of TestCephSettings struct +func StartTestCluster(t func() *testing.T, settings *installer.TestCephSettings, minimalMatrixK8sVersion string) (*installer.CephInstaller, *utils.K8sHelper) { + k8shelper, err := utils.CreateK8sHelper(t) require.NoError(t(), err) - checkIfShouldRunForMinimalTestMatrix(t, kh, cluster.minimalMatrixK8sVersion) + checkIfShouldRunForMinimalTestMatrix(t, k8shelper, minimalMatrixK8sVersion) - cluster.installer = installer.NewCephInstaller(t, kh.Clientset, cluster.useHelm, cluster.clusterName, cluster.rookVersion, cluster.cephVersion, cluster.rookCephCleanup) - cluster.kh = kh - cluster.helper = nil - cluster.T = t - - if cluster.rookVersion != installer.VersionMaster { - // make sure we have the images from a previous release locally so the test doesn't hit a timeout - assert.NoError(t(), kh.GetDockerImage("rook/ceph:"+cluster.rookVersion)) - } - - assert.NoError(t(), kh.GetDockerImage(cluster.cephVersion.Image)) - - cluster.Setup() - return cluster, kh -} - -// Setup is a wrapper for setting up rook -func (op *TestCluster) Setup() { // Turn on DEBUG logging capnslog.SetGlobalLogLevel(capnslog.DEBUG) - isRookInstalled, err := op.installer.InstallRook(op.namespace, op.storeType, op.usePVC, op.storageClassName, - cephv1.MonSpec{Count: op.mons, AllowMultiplePerNode: true}, false /* startWithAllNodes */, op.rbdMirrorWorkers, op.skipOSDCreation, op.rookVersion) + + installer := installer.NewCephInstaller(t, k8shelper.Clientset, settings) + isRookInstalled, err := installer.InstallRook() if !isRookInstalled || err != nil { logger.Errorf("Rook was not installed successfully: %v", err) - if !op.installer.T().Failed() { - op.installer.GatherAllRookLogs(op.installer.T().Name(), op.namespace, installer.SystemNamespace(op.namespace)) + if !installer.T().Failed() { + installer.GatherAllRookLogs(t().Name(), settings.Namespace, settings.OperatorNamespace) } - op.T().Fail() - op.Teardown() - op.T().FailNow() + t().Fail() + installer.UninstallRook() + t().FailNow() } -} -// SetInstallData updates the installer helper based on the version of Rook desired -func (op *TestCluster) SetInstallData(version string) {} - -// Teardown is a wrapper for tearDown after Suite -func (op *TestCluster) Teardown() { - op.installer.UninstallRook(op.namespace) + return installer, k8shelper } diff --git a/tests/integration/ceph_base_file_test.go b/tests/integration/ceph_base_file_test.go index dad175090..d7b3fd07a 100644 --- a/tests/integration/ceph_base_file_test.go +++ b/tests/integration/ceph_base_file_test.go @@ -239,83 +239,83 @@ func fileSystemCSISnapshotTest(helper *clients.TestClient, k8sh *utils.K8sHelper // Smoke Test for File System Storage - Test check the following operations on Filesystem Storage in order // Create,Mount,Write,Read,Unmount and Delete. -func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName string, useCSI bool, preserveFilesystemOnDelete bool) { - if useCSI { +func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string, preserveFilesystemOnDelete bool) { + if settings.UseCSI { checkSkipCSITest(s.T(), k8sh) } - defer fileTestDataCleanUp(helper, k8sh, s, filePodName, namespace, filesystemName) - logger.Infof("Running on Rook Cluster %s", namespace) + defer fileTestDataCleanUp(helper, k8sh, s, filePodName, settings.Namespace, filesystemName) + logger.Infof("Running on Rook Cluster %s", settings.Namespace) logger.Infof("File Storage End To End Integration Test - create, mount, write to, read from, and unmount") activeCount := 2 - createFilesystem(helper, k8sh, s, namespace, filesystemName, activeCount) + createFilesystem(helper, k8sh, s, settings, filesystemName, activeCount) if preserveFilesystemOnDelete { - _, err := k8sh.Kubectl("-n", namespace, "patch", "CephFilesystem", filesystemName, "--type=merge", "-p", `{"spec": {"preserveFilesystemOnDelete": true}}`) + _, err := k8sh.Kubectl("-n", settings.Namespace, "patch", "CephFilesystem", filesystemName, "--type=merge", "-p", `{"spec": {"preserveFilesystemOnDelete": true}}`) assert.NoError(s.T(), err) } // Create a test pod where CephFS is consumed without user creds storageClassName := "cephfs-storageclass" - err := helper.FSClient.CreateStorageClass(filesystemName, installer.SystemNamespace(namespace), namespace, storageClassName) + err := helper.FSClient.CreateStorageClass(filesystemName, settings.OperatorNamespace, settings.Namespace, storageClassName) assert.NoError(s.T(), err) - createFilesystemConsumerPod(helper, k8sh, s, namespace, filesystemName, storageClassName, useCSI) + createFilesystemConsumerPod(helper, k8sh, s, settings, filesystemName, storageClassName) // Test reading and writing to the first pod - err = writeAndReadToFilesystem(helper, k8sh, s, namespace, filePodName, "test_file") + err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, filePodName, "test_file") assert.NoError(s.T(), err) // TODO: Also mount with user credentials with the CSI driver - if !useCSI { + if !settings.UseCSI { // Create a test pod where CephFS is consumed with a mountUser and mountSecret specified. - createFilesystemMountCephCredentials(helper, k8sh, s, namespace, filesystemName) - createFilesystemMountUserConsumerPod(helper, k8sh, s, namespace, filesystemName, storageClassName) + createFilesystemMountCephCredentials(helper, k8sh, s, settings, filesystemName) + createFilesystemMountUserConsumerPod(helper, k8sh, s, settings, filesystemName, storageClassName) // Test reading and writing to the second pod - err = writeAndReadToFilesystem(helper, k8sh, s, namespace, fileMountUserPodName, "canttouchthis") + err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, fileMountUserPodName, "canttouchthis") assert.Error(s.T(), err, "we should not be able to write to file canttouchthis on CephFS `/`") - err = writeAndReadToFilesystem(helper, k8sh, s, namespace, fileMountUserPodName, "foo/test_file") + err = writeAndReadToFilesystem(helper, k8sh, s, settings.Namespace, fileMountUserPodName, "foo/test_file") assert.NoError(s.T(), err, "we should be able to write to the `/foo` directory on CephFS") - cleanupFilesystemConsumer(helper, k8sh, s, namespace, fileMountUserPodName) + cleanupFilesystemConsumer(helper, k8sh, s, settings.Namespace, fileMountUserPodName) assert.NoError(s.T(), err) } // Start the NFS daemons - testNFSDaemons(helper, k8sh, s, namespace, filesystemName) + testNFSDaemons(helper, k8sh, s, settings, filesystemName) // Cleanup the filesystem and its clients - cleanupFilesystemConsumer(helper, k8sh, s, namespace, filePodName) + cleanupFilesystemConsumer(helper, k8sh, s, settings.Namespace, filePodName) assert.NoError(s.T(), err) - downscaleMetadataServers(helper, k8sh, s, namespace, filesystemName) - cleanupFilesystem(helper, k8sh, s, namespace, filesystemName) + downscaleMetadataServers(helper, k8sh, s, settings.Namespace, filesystemName) + cleanupFilesystem(helper, k8sh, s, settings.Namespace, filesystemName) err = helper.FSClient.DeleteStorageClass(storageClassName) assertNoErrorUnlessNotFound(s, err) if preserveFilesystemOnDelete { - fses, err := helper.FSClient.List(namespace) + fses, err := helper.FSClient.List(settings.Namespace) assert.NoError(s.T(), err) assert.Len(s.T(), fses, 1) assert.Equal(s.T(), fses[0].Name, filesystemName) - err = helper.FSClient.Delete(filesystemName, namespace) + err = helper.FSClient.Delete(filesystemName, settings.Namespace) assert.NoError(s.T(), err) } } -func testNFSDaemons(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) { +func testNFSDaemons(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) { name := "my-nfs" - err := helper.NFSClient.Create(namespace, name, filesystemName+"-data0", 2) + err := helper.NFSClient.Create(settings.Namespace, name, filesystemName+"-data0", 2) require.Nil(s.T(), err) - err = helper.NFSClient.Delete(namespace, name) + err = helper.NFSClient.Delete(settings.Namespace, name) assert.Nil(s.T(), err) } -func createFilesystemConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName, storageClassName string, useCSI bool) { - err := createPodWithFilesystem(k8sh, s, filePodName, namespace, filesystemName, storageClassName, false, useCSI) +func createFilesystemConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName, storageClassName string) { + err := createPodWithFilesystem(k8sh, s, settings, filePodName, filesystemName, storageClassName, false) require.NoError(s.T(), err) - filePodRunning := k8sh.IsPodRunning(filePodName, namespace) + filePodRunning := k8sh.IsPodRunning(filePodName, settings.Namespace) require.True(s.T(), filePodRunning, "make sure file-test pod is in running state") logger.Infof("File system mounted successfully") } @@ -358,36 +358,36 @@ func cleanupFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suit } // Test File System Creation on Rook that was installed on a custom namespace i.e. Namespace != "rook" and delete it again -func runFileE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) { +func runFileE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) { checkSkipCSITest(s.T(), k8sh) logger.Infof("File Storage End to End Integration Test - create Filesystem and make sure mds pod is running") - logger.Infof("Running on Rook Cluster %s", namespace) + logger.Infof("Running on Rook Cluster %s", settings.Namespace) activeCount := 1 - createFilesystem(helper, k8sh, s, namespace, filesystemName, activeCount) + createFilesystem(helper, k8sh, s, settings, filesystemName, activeCount) // Create a test pod where CephFS is consumed without user creds storageClassName := "cephfs-storageclass" - err := helper.FSClient.CreateStorageClass(filesystemName, namespace, namespace, storageClassName) + err := helper.FSClient.CreateStorageClass(filesystemName, settings.OperatorNamespace, settings.Namespace, storageClassName) assert.NoError(s.T(), err) assert.NoError(s.T(), err) if !skipSnapshotTest(k8sh) { - fileSystemCSISnapshotTest(helper, k8sh, s, storageClassName, namespace) + fileSystemCSISnapshotTest(helper, k8sh, s, storageClassName, settings.Namespace) } if !skipCloneTest(k8sh) { - fileSystemCSICloneTest(helper, k8sh, s, storageClassName, namespace) + fileSystemCSICloneTest(helper, k8sh, s, storageClassName, settings.Namespace) } - cleanupFilesystem(helper, k8sh, s, namespace, filesystemName) + cleanupFilesystem(helper, k8sh, s, settings.Namespace, filesystemName) err = helper.FSClient.DeleteStorageClass(storageClassName) assertNoErrorUnlessNotFound(s, err) } -func createFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName string, activeCount int) { +func createFilesystem(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string, activeCount int) { logger.Infof("Create file System") - fscErr := helper.FSClient.Create(filesystemName, namespace, activeCount) + fscErr := helper.FSClient.Create(filesystemName, settings.Namespace, activeCount) require.Nil(s.T(), fscErr) logger.Infof("File system %s created", filesystemName) - filesystemList, _ := helper.FSClient.List(namespace) + filesystemList, _ := helper.FSClient.List(settings.Namespace) require.Equal(s.T(), 1, len(filesystemList), "There should be one shared file system present") } @@ -399,13 +399,12 @@ func fileTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s su assert.NoError(s.T(), err) } -func createPodWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podName, namespace, filesystemName, storageClassName string, mountUser, useCSI bool) error { +func createPodWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, podName, filesystemName, storageClassName string, mountUser bool) error { var testPodManifest string - if useCSI { - testPodManifest = getFilesystemCSITestPod(podName, namespace, storageClassName) + if settings.UseCSI { + testPodManifest = getFilesystemCSITestPod(settings, podName, storageClassName) } else { - driverName := installer.SystemNamespace(namespace) - testPodManifest = getFilesystemFlexTestPod(podName, namespace, filesystemName, driverName, mountUser) + testPodManifest = getFilesystemFlexTestPod(settings, podName, filesystemName, mountUser) } if err := k8sh.ResourceOperation("create", testPodManifest); err != nil { return fmt.Errorf("failed to create pod -- %s. %+v", testPodManifest, err) @@ -413,7 +412,7 @@ func createPodWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podName, name return nil } -func getFilesystemFlexTestPod(podName, namespace, filesystemName, driverName string, mountUser bool) string { +func getFilesystemFlexTestPod(settings *installer.TestCephSettings, podName, filesystemName string, mountUser bool) string { mountUserInsert := "" if mountUser { mountUserInsert = ` @@ -425,7 +424,7 @@ func getFilesystemFlexTestPod(podName, namespace, filesystemName, driverName str kind: Pod metadata: name: ` + podName + ` - namespace: ` + namespace + ` + namespace: ` + settings.Namespace + ` spec: containers: - name: ` + podName + ` @@ -444,23 +443,23 @@ spec: volumes: - name: ` + filesystemName + ` flexVolume: - driver: ceph.rook.io/` + driverName + ` + driver: ceph.rook.io/` + settings.OperatorNamespace + ` fsType: ceph options: fsName: ` + filesystemName + ` - clusterNamespace: ` + namespace + mountUserInsert + ` + clusterNamespace: ` + settings.Namespace + mountUserInsert + ` restartPolicy: Always ` } -func getFilesystemCSITestPod(podName, namespace, storageClassName string) string { +func getFilesystemCSITestPod(settings *installer.TestCephSettings, podName, storageClassName string) string { claimName := podName return ` apiVersion: v1 kind: PersistentVolumeClaim metadata: name: ` + claimName + ` - namespace: ` + namespace + ` + namespace: ` + settings.Namespace + ` spec: accessModes: - ReadWriteOnce @@ -473,7 +472,7 @@ apiVersion: v1 kind: Pod metadata: name: ` + podName + ` - namespace: ` + namespace + ` + namespace: ` + settings.Namespace + ` spec: containers: - name: ` + podName + ` @@ -496,20 +495,20 @@ spec: ` } -func createFilesystemMountCephCredentials(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, filesystemName string) { +func createFilesystemMountCephCredentials(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) { ctx := context.TODO() // Create agent binding for access to Secrets - err := k8sh.ResourceOperation("apply", getFilesystemAgentMountSecretsBinding(namespace)) + err := k8sh.ResourceOperation("apply", getFilesystemAgentMountSecretsBinding(settings.Namespace)) require.Nil(s.T(), err) // Mount CephFS in toolbox and create /foo directory on it logger.Info("Creating /foo directory on CephFS") - _, err = k8sh.Exec(namespace, "rook-ceph-tools", "mkdir", []string{"-p", utils.TestMountPath}) + _, err = k8sh.Exec(settings.Namespace, client.RunAllCephCommandsInToolboxPod, "mkdir", []string{"-p", utils.TestMountPath}) require.Nil(s.T(), err) - _, err = k8sh.ExecWithRetry(10, namespace, "rook-ceph-tools", "bash", []string{"-c", fmt.Sprintf("mount -t ceph -o mds_namespace=%s,name=admin,secret=$(grep key /etc/ceph/keyring | awk '{print $3}') $(grep mon_host /etc/ceph/ceph.conf | awk '{print $3}'):/ %s", filesystemName, utils.TestMountPath)}) + _, err = k8sh.ExecWithRetry(10, settings.Namespace, client.RunAllCephCommandsInToolboxPod, "bash", []string{"-c", fmt.Sprintf("mount -t ceph -o mds_namespace=%s,name=admin,secret=$(grep key /etc/ceph/keyring | awk '{print $3}') $(grep mon_host /etc/ceph/ceph.conf | awk '{print $3}'):/ %s", filesystemName, utils.TestMountPath)}) require.Nil(s.T(), err) - _, err = k8sh.Exec(namespace, "rook-ceph-tools", "mkdir", []string{"-p", fmt.Sprintf("%s/foo", utils.TestMountPath)}) + _, err = k8sh.Exec(settings.Namespace, client.RunAllCephCommandsInToolboxPod, "mkdir", []string{"-p", fmt.Sprintf("%s/foo", utils.TestMountPath)}) require.Nil(s.T(), err) - _, err = k8sh.Exec(namespace, "rook-ceph-tools", "umount", []string{utils.TestMountPath}) + _, err = k8sh.Exec(settings.Namespace, client.RunAllCephCommandsInToolboxPod, "umount", []string{utils.TestMountPath}) require.Nil(s.T(), err) logger.Info("Created /foo directory on CephFS") @@ -523,15 +522,15 @@ func createFilesystemMountCephCredentials(helper *clients.TestClient, k8sh *util ), } logger.Infof("ceph credentials command args: %s", commandArgs[1]) - result, err := k8sh.Exec(namespace, "rook-ceph-tools", "bash", commandArgs) + result, err := k8sh.Exec(settings.Namespace, client.RunAllCephCommandsInToolboxPod, "bash", commandArgs) logger.Infof("Ceph filesystem credentials output: %s", result) logger.Info("Created Ceph credentials") require.Nil(s.T(), err) // Save Ceph credentials to Kubernetes - _, err = k8sh.Clientset.CoreV1().Secrets(namespace).Create(ctx, &v1.Secret{ + _, err = k8sh.Clientset.CoreV1().Secrets(settings.Namespace).Create(ctx, &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: fileMountSecret, - Namespace: namespace, + Namespace: settings.Namespace, }, Data: map[string][]byte{ "mykey": []byte(result), @@ -541,12 +540,11 @@ func createFilesystemMountCephCredentials(helper *clients.TestClient, k8sh *util logger.Info("Created Ceph credentials Secret in Kubernetes") } -func createFilesystemMountUserConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace, filesystemName, storageClassName string) { +func createFilesystemMountUserConsumerPod(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName, storageClassName string) { // TODO: Mount with user credentials for the CSI driver - useCSI := false - mtfsErr := createPodWithFilesystem(k8sh, s, fileMountUserPodName, namespace, filesystemName, storageClassName, true, useCSI) + mtfsErr := createPodWithFilesystem(k8sh, s, settings, fileMountUserPodName, filesystemName, storageClassName, true) require.Nil(s.T(), mtfsErr) - filePodRunning := k8sh.IsPodRunning(fileMountUserPodName, namespace) + filePodRunning := k8sh.IsPodRunning(fileMountUserPodName, settings.Namespace) require.True(s.T(), filePodRunning, "make sure file-mountuser-test pod is in running state") logger.Infof("File system mounted successfully") } diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go index 3cbb6c19a..4cb263334 100644 --- a/tests/integration/ceph_base_object_test.go +++ b/tests/integration/ceph_base_object_test.go @@ -24,9 +24,9 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" rgw "github.com/rook/rook/pkg/operator/ceph/object" "github.com/rook/rook/tests/framework/clients" + "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -59,11 +59,11 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite logger.Infof("Step 0 : Create Object Store User") cosuErr := helper.ObjectUserClient.Create(namespace, userid, userdisplayname, storeName) - require.Nil(s.T(), cosuErr) + assert.Nil(s.T(), cosuErr) logger.Infof("Step 1 : Create Object Store") cobsErr := helper.ObjectClient.Create(namespace, storeName, 3) - require.Nil(s.T(), cobsErr) + assert.Nil(s.T(), cobsErr) // check that ObjectStore is created logger.Infof("Check that RGW pods are Running") @@ -86,7 +86,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite assert.True(s.T(), helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid)) userInfo, err := helper.ObjectUserClient.GetUser(namespace, storeName, userid) - require.NoError(s.T(), err) + assert.NoError(s.T(), err) assert.Equal(s.T(), userid, userInfo.UserID) assert.Equal(s.T(), userdisplayname, *userInfo.DisplayName) logger.Infof("Done creating object store user") @@ -107,11 +107,11 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite assert.NotEmpty(s.T(), objectStore.Status.Info["endpoint"]) break } - require.NotEqual(s.T(), i, 4) + assert.NotEqual(s.T(), i, 4) logger.Infof("Step 2 : Test Deleting User") dosuErr := helper.ObjectUserClient.Delete(namespace, userid) - require.Nil(s.T(), dosuErr) + assert.Nil(s.T(), dosuErr) logger.Infof("Object store user deleted successfully") logger.Infof("Checking to see if the user secret has been deleted") for i = 0; i < 4 && helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid) == true; i++ { @@ -119,7 +119,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite time.Sleep(5 * time.Second) } assert.False(s.T(), helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid)) - require.NotEqual(s.T(), i, 4) + assert.NotEqual(s.T(), i, 4) logger.Infof("Check that MGRs are not in a crashloop") assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mgr", namespace, 1, "Running")) @@ -129,14 +129,14 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite logger.Infof("Step 3 : Create Object Bucket Claim with reclaim policy delete") bucketStorageClassName := "rook-smoke-delete-bucket" cobErr := helper.BucketClient.CreateBucketStorageClass(namespace, storeName, bucketStorageClassName, "Delete", region) - require.Nil(s.T(), cobErr) + assert.Nil(s.T(), cobErr) cobcErr := helper.BucketClient.CreateObc(obcName, bucketStorageClassName, bucketname, maxObject, true) - require.Nil(s.T(), cobcErr) + assert.Nil(s.T(), cobcErr) created := utils.Retry(12, 2*time.Second, "OBC is created", func() bool { return helper.BucketClient.CheckOBC(obcName, "bound") }) - require.True(s.T(), created) + assert.True(s.T(), created) logger.Infof("Check if bucket was created") context := k8sh.MakeContext() @@ -153,7 +153,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite time.Sleep(5 * time.Second) } assert.NotEqual(s.T(), i, 4) - require.Equal(s.T(), bucketname, bkt.Name) + assert.Equal(s.T(), bucketname, bkt.Name) logger.Infof("OBC, Secret and ConfigMap created") logger.Infof("Step 4 : Create s3 client") @@ -161,37 +161,37 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName) s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName) s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true) - require.Nil(s.T(), err) + assert.Nil(s.T(), err) logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey) logger.Infof("Step 5 : Put Object on bucket") _, poErr := s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey1, contentType) - require.Nil(s.T(), poErr) + assert.Nil(s.T(), poErr) logger.Infof("Step 6 : Get Object from bucket") read, err := s3client.GetObjectInBucket(bucketname, ObjectKey1) - require.Nil(s.T(), err) - require.Equal(s.T(), ObjBody, read) + assert.Nil(s.T(), err) + assert.Equal(s.T(), ObjBody, read) logger.Infof("Object Created and Retrieved on bucket successfully") logger.Infof("Step 7 : Testing Quota for the OBC") logger.Infof("Adding one more object to the bucket") _, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey2, contentType) - require.Nil(s.T(), poErr) + assert.Nil(s.T(), poErr) logger.Infof("Testing the max object limit") _, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey3, contentType) - require.Error(s.T(), poErr) + assert.Error(s.T(), poErr) logger.Infof("Step 8 : Delete objects on bucket") _, delobjErr := s3client.DeleteObjectInBucket(bucketname, ObjectKey1) - require.Nil(s.T(), delobjErr) + assert.Nil(s.T(), delobjErr) _, delobjErr = s3client.DeleteObjectInBucket(bucketname, ObjectKey2) - require.Nil(s.T(), delobjErr) + assert.Nil(s.T(), delobjErr) logger.Infof("Objects deleted on bucket successfully") logger.Infof("Step 9 : Delete Object Bucket Claim") dobcErr := helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketname, maxObject, true) - require.Nil(s.T(), dobcErr) + assert.Nil(s.T(), dobcErr) logger.Infof("Checking to see if the obc, secret and cm have all been deleted") for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "deleted"); i++ { logger.Infof("(%d) obc deleted check, sleeping for 5 seconds ...", i) @@ -220,27 +220,27 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite } // Test Object StoreCreation on Rook that was installed via helm -func runObjectE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, name string, replicaSize int, deleteStore bool) { +func runObjectE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, name string, replicaSize int, deleteStore bool) { logger.Infof("Object Storage End To End Integration Test - Create Object Store and check if rgw service is Running") - logger.Infof("Running on Rook Cluster %s", namespace) + logger.Infof("Running on Rook Cluster %s", settings.Namespace) logger.Infof("Step 1 : Create Object Store") - err := helper.ObjectClient.Create(namespace, name, int32(replicaSize)) - require.Nil(s.T(), err) + err := helper.ObjectClient.Create(settings.Namespace, name, int32(replicaSize)) + assert.Nil(s.T(), err) logger.Infof("Step 2 : check rook-ceph-rgw service status and count") - require.True(s.T(), k8sh.IsPodInExpectedState("rook-ceph-rgw", namespace, "Running"), + assert.True(s.T(), k8sh.IsPodInExpectedState("rook-ceph-rgw", settings.Namespace, "Running"), "Make sure rook-ceph-rgw is in running state") - assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-rgw", namespace, replicaSize, "Running"), + assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-rgw", settings.Namespace, replicaSize, "Running"), "Make sure all rook-ceph-rgw pods are in Running state") - require.True(s.T(), k8sh.IsServiceUp("rook-ceph-rgw-"+name, namespace)) + assert.True(s.T(), k8sh.IsServiceUp("rook-ceph-rgw-"+name, settings.Namespace)) if deleteStore { logger.Infof("Delete Object Store") - err = helper.ObjectClient.Delete(namespace, name) - require.Nil(s.T(), err) + err = helper.ObjectClient.Delete(settings.Namespace, name) + assert.Nil(s.T(), err) logger.Infof("Done deleting object store") } } diff --git a/tests/integration/ceph_flex_test.go b/tests/integration/ceph_flex_test.go index e00618e70..0937e090e 100644 --- a/tests/integration/ceph_flex_test.go +++ b/tests/integration/ceph_flex_test.go @@ -61,7 +61,7 @@ func TestCephFlexSuite(t *testing.T) { s := new(CephFlexDriverSuite) defer func(s *CephFlexDriverSuite) { - HandlePanics(recover(), s.op, s.T) + HandlePanics(recover(), s.TearDownSuite, s.T) }(s) suite.Run(t, s) } @@ -72,47 +72,45 @@ type CephFlexDriverSuite struct { clusterInfo *client.ClusterInfo bc *clients.BlockOperation kh *utils.K8sHelper - namespace string + installer *installer.CephInstaller + settings *installer.TestCephSettings pvcNameRWO string pvcNameRWX string - op *TestCluster } func (s *CephFlexDriverSuite) SetupSuite() { - s.namespace = "flex-ns" + namespace := "flex-ns" s.pvcNameRWO = "block-persistent-rwo" s.pvcNameRWX = "block-persistent-rwx" - - flexTestCluster := TestCluster{ - namespace: s.namespace, - storeType: "bluestore", - storageClassName: "", - useHelm: false, - usePVC: false, - mons: 1, - rbdMirrorWorkers: 1, - rookCephCleanup: true, - skipOSDCreation: false, - minimalMatrixK8sVersion: flexDriverMinimalTestVersion, - rookVersion: installer.VersionMaster, - cephVersion: installer.OctopusVersion, + s.settings = &installer.TestCephSettings{ + ClusterName: "flex-test", + Namespace: namespace, + OperatorNamespace: installer.SystemNamespace(namespace), + StorageClassName: "", + UseHelm: false, + UsePVC: false, + Mons: 1, + SkipOSDCreation: false, + UseCSI: false, + DirectMountToolbox: true, + RookVersion: installer.VersionMaster, + CephVersion: installer.OctopusVersion, } - s.clusterInfo = client.AdminClusterInfo(s.namespace) - s.op, s.kh = StartTestCluster(s.T, &flexTestCluster) - s.testClient = clients.CreateTestClient(s.kh, s.op.installer.Manifests) + s.clusterInfo = client.AdminClusterInfo(namespace) + s.installer, s.kh = StartTestCluster(s.T, s.settings, flexDriverMinimalTestVersion) + s.testClient = clients.CreateTestClient(s.kh, s.installer.Manifests) s.bc = s.testClient.BlockClient } func (s *CephFlexDriverSuite) AfterTest(suiteName, testName string) { - s.op.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(s.namespace)) + s.installer.CollectOperatorLog(suiteName, testName) } func (s *CephFlexDriverSuite) TestFileSystem() { - useCSI := false preserveFilesystemOnDelete := false - runFileE2ETest(s.testClient, s.kh, s.Suite, s.namespace, "smoke-test-fs", useCSI, preserveFilesystemOnDelete) + runFileE2ETest(s.testClient, s.kh, s.Suite, s.settings, "flex-fs", preserveFilesystemOnDelete) } func (s *CephFlexDriverSuite) TestBlockStorageMountUnMountForStatefulSets() { @@ -127,9 +125,9 @@ func (s *CephFlexDriverSuite) TestBlockStorageMountUnMountForStatefulSets() { logger.Infof("Test case when block persistent volumes are scaled up and down along with StatefulSet") logger.Info("Step 1: Create pool and storageClass") - err := s.testClient.PoolClient.Create(poolName, s.namespace, 1) + err := s.testClient.PoolClient.Create(poolName, s.settings.Namespace, 1) assert.Nil(s.T(), err) - err = s.testClient.BlockClient.CreateStorageClass(false, poolName, storageClassName, reclaimPolicy, s.namespace) + err = s.testClient.BlockClient.CreateStorageClass(false, poolName, storageClassName, reclaimPolicy, s.settings.Namespace) assert.Nil(s.T(), err) logger.Info("Step 2 : Deploy statefulSet with 1X replication") service, statefulset := getBlockStatefulSetAndServiceDefinition(defaultNamespace, statefulSetName, statefulPodsName, storageClassName) @@ -190,11 +188,9 @@ func (s *CephFlexDriverSuite) setupPVCs() { logger.Infof("creating the test PVCs") poolNameRWO := "block-pool-rwo" storageClassNameRWO := "rook-ceph-block-rwo" - systemNamespace := installer.SystemNamespace(s.namespace) // Create PVCs - useCSI := false - err := s.testClient.BlockClient.CreateStorageClassAndPVC(useCSI, defaultNamespace, s.namespace, systemNamespace, poolNameRWO, storageClassNameRWO, "Delete", s.pvcNameRWO, "ReadWriteOnce") + err := s.testClient.BlockClient.CreateStorageClassAndPVC(defaultNamespace, poolNameRWO, storageClassNameRWO, "Delete", s.pvcNameRWO, "ReadWriteOnce") require.Nil(s.T(), err) require.True(s.T(), s.kh.WaitUntilPVCIsBound(defaultNamespace, s.pvcNameRWO), "Make sure PVC is Bound") @@ -207,13 +203,13 @@ func (s *CephFlexDriverSuite) setupPVCs() { require.Nil(s.T(), err) crdName, err := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWO) require.Nil(s.T(), err) - s.kh.IsVolumeResourcePresent(systemNamespace, crdName) + s.kh.IsVolumeResourcePresent(s.settings.OperatorNamespace, crdName) err = s.bc.CreateClientPod(getFlexBlockPodDefinition("setup-block-rwx", s.pvcNameRWX, false)) require.Nil(s.T(), err) crdName, err = s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWX) require.Nil(s.T(), err) - s.kh.IsVolumeResourcePresent(systemNamespace, crdName) + s.kh.IsVolumeResourcePresent(s.settings.OperatorNamespace, crdName) require.True(s.T(), s.kh.IsPodRunning("setup-block-rwo", defaultNamespace), "make sure setup-block-rwo pod is in running state") require.True(s.T(), s.kh.IsPodRunning("setup-block-rwx", defaultNamespace), "make sure setup-block-rwx pod is in running state") @@ -239,9 +235,9 @@ func (s *CephFlexDriverSuite) TearDownSuite() { "setup-block-rwo", "setup-block-rwx", "rwo-block-rw-one", "rwo-block-rw-two", "rwo-block-ro-one", "rwo-block-ro-two", "rwx-block-rw-one", "rwx-block-rw-two", "rwx-block-ro-one", "rwx-block-ro-two") assert.NoError(s.T(), err) - err = s.testClient.BlockClient.DeletePVC(s.namespace, s.pvcNameRWO) + err = s.testClient.BlockClient.DeletePVC(s.settings.Namespace, s.pvcNameRWO) assertNoErrorUnlessNotFound(s.Suite, err) - err = s.testClient.BlockClient.DeletePVC(s.namespace, s.pvcNameRWX) + err = s.testClient.BlockClient.DeletePVC(s.settings.Namespace, s.pvcNameRWX) assertNoErrorUnlessNotFound(s.Suite, err) err = s.testClient.BlockClient.DeleteStorageClass("rook-ceph-block-rwo") assert.NoError(s.T(), err) @@ -251,7 +247,7 @@ func (s *CephFlexDriverSuite) TearDownSuite() { assert.NoError(s.T(), err) err = s.testClient.PoolClient.DeletePool(s.testClient.BlockClient, s.clusterInfo, "block-pool-rwx") assert.NoError(s.T(), err) - s.op.Teardown() + s.installer.UninstallRook() } func (s *CephFlexDriverSuite) TestBlockStorageMountUnMountForDifferentAccessModes() { @@ -267,12 +263,12 @@ func (s *CephFlexDriverSuite) TestBlockStorageMountUnMountForDifferentAccessMode require.Nil(s.T(), err) crdName, err := s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWO) assert.Nil(s.T(), err) - assert.True(s.T(), s.kh.IsVolumeResourcePresent(installer.SystemNamespace(s.namespace), crdName), fmt.Sprintf("make sure Volume %s is created", crdName)) + assert.True(s.T(), s.kh.IsVolumeResourcePresent(s.settings.OperatorNamespace, crdName), fmt.Sprintf("make sure Volume %s is created", crdName)) assert.True(s.T(), s.kh.IsPodRunning("rwo-block-rw-one", defaultNamespace), "make sure block-rw-one pod is in running state") crdName, err = s.kh.GetVolumeResourceName(defaultNamespace, s.pvcNameRWX) assert.Nil(s.T(), err) - assert.True(s.T(), s.kh.IsVolumeResourcePresent(installer.SystemNamespace(s.namespace), crdName), fmt.Sprintf("make sure Volume %s is created", crdName)) + assert.True(s.T(), s.kh.IsVolumeResourcePresent(s.settings.OperatorNamespace, crdName), fmt.Sprintf("make sure Volume %s is created", crdName)) assert.True(s.T(), s.kh.IsPodRunning("rwx-block-rw-one", defaultNamespace), "make sure rwx-block-rw-one pod is in running state") logger.Infof("Step 2: Check if previously persisted data is readable from ReadWriteOnce and ReadWriteMany PVC") diff --git a/tests/integration/ceph_helm_test.go b/tests/integration/ceph_helm_test.go index a50420cdb..028a88d50 100644 --- a/tests/integration/ceph_helm_test.go +++ b/tests/integration/ceph_helm_test.go @@ -47,7 +47,7 @@ func TestCephHelmSuite(t *testing.T) { s := new(HelmSuite) defer func(s *HelmSuite) { - HandlePanics(recover(), s.op, s.T) + HandlePanics(recover(), s.TearDownSuite, s.T) }(s) suite.Run(t, s) } @@ -55,56 +55,56 @@ func TestCephHelmSuite(t *testing.T) { type HelmSuite struct { suite.Suite helper *clients.TestClient - kh *utils.K8sHelper - op *TestCluster - namespace string + installer *installer.CephInstaller + settings *installer.TestCephSettings + k8shelper *utils.K8sHelper } -func (hs *HelmSuite) SetupSuite() { - hs.namespace = "helm-ns" - helmTestCluster := TestCluster{ - namespace: hs.namespace, - storeType: "bluestore", - storageClassName: "", - useHelm: true, - usePVC: false, - mons: 1, - rbdMirrorWorkers: 1, - rookCephCleanup: true, - skipOSDCreation: false, - minimalMatrixK8sVersion: helmMinimalTestVersion, - rookVersion: installer.VersionMaster, - cephVersion: installer.NautilusVersion, +func (h *HelmSuite) SetupSuite() { + namespace := "helm-ns" + h.settings = &installer.TestCephSettings{ + Namespace: namespace, + OperatorNamespace: namespace, + StorageClassName: "", + UseHelm: true, + UsePVC: false, + Mons: 1, + UseCSI: true, + SkipOSDCreation: false, + EnableAdmissionController: true, + EnableDiscovery: true, + RookVersion: installer.VersionMaster, + CephVersion: installer.OctopusVersion, } - hs.op, hs.kh = StartTestCluster(hs.T, &helmTestCluster) - hs.helper = clients.CreateTestClient(hs.kh, hs.op.installer.Manifests) + h.installer, h.k8shelper = StartTestCluster(h.T, h.settings, helmMinimalTestVersion) + h.helper = clients.CreateTestClient(h.k8shelper, h.installer.Manifests) } -func (hs *HelmSuite) TearDownSuite() { - hs.op.Teardown() +func (h *HelmSuite) TearDownSuite() { + h.installer.UninstallRook() } -func (hs *HelmSuite) AfterTest(suiteName, testName string) { - hs.op.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(hs.namespace)) +func (h *HelmSuite) AfterTest(suiteName, testName string) { + h.installer.CollectOperatorLog(suiteName, testName) } // Test to make sure all rook components are installed and Running -func (hs *HelmSuite) TestARookInstallViaHelm() { - checkIfRookClusterIsInstalled(hs.Suite, hs.kh, hs.namespace, hs.namespace, 1) +func (h *HelmSuite) TestARookInstallViaHelm() { + checkIfRookClusterIsInstalled(h.Suite, h.k8shelper, h.settings.Namespace, h.settings.Namespace, 1) } // Test BlockCreation on Rook that was installed via Helm -func (hs *HelmSuite) TestBlockStoreOnRookInstalledViaHelm() { - runBlockCSITestLite(hs.helper, hs.kh, hs.Suite, hs.namespace, hs.namespace, hs.op.installer.CephVersion) +func (h *HelmSuite) TestBlockStoreOnRookInstalledViaHelm() { + runBlockCSITestLite(h.helper, h.k8shelper, h.Suite, h.settings) } // Test File System Creation on Rook that was installed via helm -func (hs *HelmSuite) TestFileStoreOnRookInstalledViaHelm() { - runFileE2ETestLite(hs.helper, hs.kh, hs.Suite, hs.namespace, "testfs") +func (h *HelmSuite) TestFileStoreOnRookInstalledViaHelm() { + runFileE2ETestLite(h.helper, h.k8shelper, h.Suite, h.settings, "testfs") } // Test Object StoreCreation on Rook that was installed via helm -func (hs *HelmSuite) TestObjectStoreOnRookInstalledViaHelm() { - runObjectE2ETestLite(hs.helper, hs.kh, hs.Suite, hs.namespace, "default", 3, true) +func (h *HelmSuite) TestObjectStoreOnRookInstalledViaHelm() { + runObjectE2ETestLite(h.helper, h.k8shelper, h.Suite, h.settings, "default", 3, true) } diff --git a/tests/integration/ceph_mgr_test.go b/tests/integration/ceph_mgr_test.go index 35b849357..cc5abd077 100644 --- a/tests/integration/ceph_mgr_test.go +++ b/tests/integration/ceph_mgr_test.go @@ -55,15 +55,16 @@ func TestCephMgrSuite(t *testing.T) { s := new(CephMgrSuite) defer func(s *CephMgrSuite) { - HandlePanics(recover(), s.cluster, s.T) + HandlePanics(recover(), s.TearDownSuite, s.T) }(s) suite.Run(t, s) } type CephMgrSuite struct { suite.Suite - cluster *TestCluster + settings *installer.TestCephSettings k8sh *utils.K8sHelper + installer *installer.CephInstaller namespace string } @@ -87,46 +88,44 @@ type service struct { Status serviceStatus } -func (suite *CephMgrSuite) SetupSuite() { - suite.namespace = "mgr-ns" +func (s *CephMgrSuite) SetupSuite() { + s.namespace = "mgr-ns" - mgrTestCluster := TestCluster{ - clusterName: suite.namespace, - namespace: suite.namespace, - storeType: "bluestore", - storageClassName: "", - useHelm: false, - usePVC: false, - mons: 1, - rbdMirrorWorkers: 0, - rookCephCleanup: true, - skipOSDCreation: true, - minimalMatrixK8sVersion: cephMasterSuiteMinimalTestVersion, - rookVersion: installer.VersionMaster, - cephVersion: installer.MasterVersion, + s.settings = &installer.TestCephSettings{ + ClusterName: s.namespace, + OperatorNamespace: installer.SystemNamespace(s.namespace), + Namespace: s.namespace, + StorageClassName: "", + UseHelm: false, + UsePVC: false, + Mons: 1, + UseCSI: true, + SkipOSDCreation: true, + RookVersion: installer.VersionMaster, + CephVersion: installer.MasterVersion, } - suite.cluster, suite.k8sh = StartTestCluster(suite.T, &mgrTestCluster) - suite.waitForOrchestrationModule() + s.installer, s.k8sh = StartTestCluster(s.T, s.settings, cephMasterSuiteMinimalTestVersion) + s.waitForOrchestrationModule() } -func (suite *CephMgrSuite) AfterTest(suiteName, testName string) { - suite.cluster.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(suite.namespace)) +func (s *CephMgrSuite) AfterTest(suiteName, testName string) { + s.installer.CollectOperatorLog(suiteName, testName) } -func (suite *CephMgrSuite) TearDownSuite() { - suite.cluster.Teardown() +func (s *CephMgrSuite) TearDownSuite() { + s.installer.UninstallRook() } -func (suite *CephMgrSuite) execute(command []string) (error, string) { +func (s *CephMgrSuite) execute(command []string) (error, string) { orchCommand := append([]string{"orch"}, command...) - return suite.cluster.installer.Execute("ceph", orchCommand, suite.namespace) + return s.installer.Execute("ceph", orchCommand, s.namespace) } -func (suite *CephMgrSuite) waitForOrchestrationModule() { +func (s *CephMgrSuite) waitForOrchestrationModule() { var err error for timeout := 0; timeout < 30; timeout++ { - err, output := suite.execute([]string{"status"}) + err, output := s.execute([]string{"status"}) logger.Infof("%s", output) if err == nil { logger.Info("Rook Toolbox ready to execute commands") @@ -135,30 +134,30 @@ func (suite *CephMgrSuite) waitForOrchestrationModule() { time.Sleep(2 * time.Second) } logger.Error("Giving up waiting for Rook Toolbox to be ready") - assert.Nil(suite.T(), err) + assert.Nil(s.T(), err) } -func (suite *CephMgrSuite) TestDeviceLs() { +func (s *CephMgrSuite) TestDeviceLs() { logger.Info("Testing .... ") - err, device_list := suite.execute([]string{"device", "ls"}) - assert.Nil(suite.T(), err) + err, device_list := s.execute([]string{"device", "ls"}) + assert.Nil(s.T(), err) logger.Infof("output = %s", device_list) } -func (suite *CephMgrSuite) TestStatus() { +func (s *CephMgrSuite) TestStatus() { logger.Info("Testing .... ") - err, status := suite.execute([]string{"status"}) - assert.Nil(suite.T(), err) + err, status := s.execute([]string{"status"}) + assert.Nil(s.T(), err) logger.Infof("output = %s", status) - assert.Equal(suite.T(), status, "Backend: rook\nAvailable: True") + assert.Equal(s.T(), status, "Backend: rook\nAvailable: True") } -func (suite *CephMgrSuite) TestHostLs() { +func (s *CephMgrSuite) TestHostLs() { logger.Info("Testing .... ") // Get the orchestrator hosts - err, output := suite.execute([]string{"host", "ls", "json"}) - assert.Nil(suite.T(), err) + err, output := s.execute([]string{"host", "ls", "json"}) + assert.Nil(s.T(), err) logger.Infof("output = %s", output) hosts := []byte(output) @@ -166,7 +165,7 @@ func (suite *CephMgrSuite) TestHostLs() { err = json.Unmarshal(hosts, &hostsList) if err != nil { - assert.Nil(suite.T(), err) + assert.Nil(s.T(), err) } var hostOutput []string @@ -176,8 +175,8 @@ func (suite *CephMgrSuite) TestHostLs() { sort.Strings(hostOutput) // get the k8s nodes - nodes, err := k8sutil.GetNodeHostNames(suite.k8sh.Clientset) - assert.Nil(suite.T(), err) + nodes, err := k8sutil.GetNodeHostNames(s.k8sh.Clientset) + assert.Nil(s.T(), err) k8sNodes := make([]string, 0, len(nodes)) for k := range nodes { @@ -186,21 +185,21 @@ func (suite *CephMgrSuite) TestHostLs() { sort.Strings(k8sNodes) // nodes and hosts must be the same - assert.Equal(suite.T(), hostOutput, k8sNodes) + assert.Equal(s.T(), hostOutput, k8sNodes) } -func (suite *CephMgrSuite) TestCreateOSD() { +func (s *CephMgrSuite) TestCreateOSD() { logger.Info("Testing .... ") // Get the first available device - err, deviceList := suite.execute([]string{"device", "ls", "--format", "json"}) - assert.Nil(suite.T(), err) + err, deviceList := s.execute([]string{"device", "ls", "--format", "json"}) + assert.Nil(s.T(), err) logger.Infof("output = %s", deviceList) inventory := make([]map[string]interface{}, 0) err = json.Unmarshal([]byte(deviceList), &inventory) - assert.Nil(suite.T(), err) + assert.Nil(s.T(), err) selectedNode := "" selectedDevice := "" @@ -216,40 +215,40 @@ func (suite *CephMgrSuite) TestCreateOSD() { break } } - assert.NotEqual(suite.T(), "", selectedDevice, "No devices available to create test OSD") - assert.NotEqual(suite.T(), "", selectedNode, "No nodes available to create test OSD") + assert.NotEqual(s.T(), "", selectedDevice, "No devices available to create test OSD") + assert.NotEqual(s.T(), "", selectedNode, "No nodes available to create test OSD") if selectedDevice == "" || selectedNode == "" { return } // Create the OSD - err, output := suite.execute([]string{"daemon", "add", "osd", fmt.Sprintf("%s:%s", selectedNode, selectedDevice)}) + err, output := s.execute([]string{"daemon", "add", "osd", fmt.Sprintf("%s:%s", selectedNode, selectedDevice)}) - assert.Nil(suite.T(), err) + assert.Nil(s.T(), err) logger.Infof("output = %s", output) - err = suite.k8sh.WaitForPodCount("app=rook-ceph-osd", suite.namespace, 1) - assert.Nil(suite.T(), err) + err = s.k8sh.WaitForPodCount("app=rook-ceph-osd", s.namespace, 1) + assert.Nil(s.T(), err) } -func (suite *CephMgrSuite) TestServiceLs() { +func (s *CephMgrSuite) TestServiceLs() { logger.Info("Testing .... ") - err, output := suite.execute([]string{"ls", "--format", "json"}) - assert.Nil(suite.T(), err) + err, output := s.execute([]string{"ls", "--format", "json"}) + assert.Nil(s.T(), err) logger.Infof("output = %s", output) services := []byte(output) var servicesList []service err = json.Unmarshal(services, &servicesList) - assert.Nil(suite.T(), err) + assert.Nil(s.T(), err) for _, svc := range servicesList { labelFilter := fmt.Sprintf("app=rook-ceph-%s", svc.ServiceName) - k8sPods, err := k8sutil.PodsRunningWithLabel(suite.k8sh.Clientset, suite.namespace, labelFilter) + k8sPods, err := k8sutil.PodsRunningWithLabel(s.k8sh.Clientset, s.namespace, labelFilter) logger.Infof("Service: %+v", svc) logger.Infof("k8s pods for svc %q: %d", svc.ServiceName, k8sPods) - assert.Nil(suite.T(), err) - assert.Equal(suite.T(), svc.Status.Running, k8sPods, fmt.Sprintf("Wrong number of pods for kind of service <%s>", svc.ServiceName)) + assert.Nil(s.T(), err) + assert.Equal(s.T(), svc.Status.Running, k8sPods, fmt.Sprintf("Wrong number of pods for kind of service <%s>", svc.ServiceName)) } } diff --git a/tests/integration/ceph_multi_cluster_test.go b/tests/integration/ceph_multi_cluster_test.go index 7e5e2af2f..bd552453a 100644 --- a/tests/integration/ceph_multi_cluster_test.go +++ b/tests/integration/ceph_multi_cluster_test.go @@ -17,16 +17,14 @@ limitations under the License. package integration import ( - "fmt" "path/filepath" "testing" - "time" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -58,180 +56,128 @@ func TestCephMultiClusterDeploySuite(t *testing.T) { s := new(MultiClusterDeploySuite) defer func(s *MultiClusterDeploySuite) { - HandlePanics(recover(), s.op, s.T) + HandlePanics(recover(), s.TearDownSuite, s.T) }(s) suite.Run(t, s) } type MultiClusterDeploySuite struct { suite.Suite - testClient *clients.TestClient - k8sh *utils.K8sHelper - namespace1 string - namespace2 string - op *MCTestOperations - poolName string + testClient *clients.TestClient + k8sh *utils.K8sHelper + settings *installer.TestCephSettings + externalManifests installer.CephManifests + installer *installer.CephInstaller + coreToolbox string + externalToolbox string + poolName string } // Deploy Multiple Rook clusters -func (mrc *MultiClusterDeploySuite) SetupSuite() { - mrc.poolName = "multi-cluster-pool1" - mrc.namespace1 = "mrc-n1" - mrc.namespace2 = "mrc-n2" - - mrc.op, mrc.k8sh = NewMCTestOperations(mrc.T, mrc.namespace1, mrc.namespace2) - mrc.testClient = clients.CreateTestClient(mrc.k8sh, mrc.op.installer.Manifests) - mrc.createPools() -} - -func (mrc *MultiClusterDeploySuite) AfterTest(suiteName, testName string) { - mrc.op.installer.CollectOperatorLog(suiteName, testName, mrc.op.systemNamespace) -} - -func (mrc *MultiClusterDeploySuite) createPools() { - // create a test pool in each cluster so that we get some PGs - logger.Infof("Creating pool %s", mrc.poolName) - err := mrc.testClient.PoolClient.Create(mrc.poolName, mrc.namespace1, 1) - require.Nil(mrc.T(), err) -} - -func (mrc *MultiClusterDeploySuite) deletePools() { - // create a test pool in each cluster so that we get some PGs - logger.Infof("Deleting pool %s", mrc.poolName) - clusterInfo := client.AdminClusterInfo(mrc.namespace1) - if err := mrc.testClient.PoolClient.DeletePool(mrc.testClient.BlockClient, clusterInfo, mrc.poolName); err != nil { - logger.Errorf("failed to delete pool %q. %v", mrc.poolName, err) - } else { - logger.Infof("deleted pool %q", mrc.poolName) +func (s *MultiClusterDeploySuite) SetupSuite() { + s.poolName = "multi-cluster-pool1" + coreNamespace := "multi-core" + s.settings = &installer.TestCephSettings{ + ClusterName: "multi-cluster", + Namespace: coreNamespace, + OperatorNamespace: installer.SystemNamespace(coreNamespace), + StorageClassName: "manual", + UsePVC: installer.UsePVC(), + Mons: 1, + UseCSI: true, + MultipleMgrs: true, + EnableAdmissionController: true, + RookVersion: installer.VersionMaster, + CephVersion: installer.NautilusVersion, } -} - -func (mrc *MultiClusterDeploySuite) TearDownSuite() { - mrc.deletePools() - mrc.op.Teardown() -} - -// Test to make sure all rook components are installed and Running -func (mrc *MultiClusterDeploySuite) TestInstallingMultipleRookClusters() { - // Check if Rook cluster 1 is deployed successfully - checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace1, 1) - checkIfRookClusterIsHealthy(mrc.Suite, mrc.testClient, mrc.namespace1) - - // Check if Rook external cluster is deployed successfully - // Checking health status is enough to validate the connection - checkIfRookClusterIsHealthy(mrc.Suite, mrc.testClient, mrc.namespace2) -} - -// MCTestOperations struct for handling panic and test suite tear down -type MCTestOperations struct { - installer *installer.CephInstaller - kh *utils.K8sHelper - T func() *testing.T - namespace1 string - namespace2 string - systemNamespace string - storageClassName string - testOverPVC bool -} - -// NewMCTestOperations creates new instance of TestCluster struct -func NewMCTestOperations(t func() *testing.T, namespace1 string, namespace2 string) (*MCTestOperations, *utils.K8sHelper) { - - kh, err := utils.CreateK8sHelper(t) - require.NoError(t(), err) - checkIfShouldRunForMinimalTestMatrix(t, kh, multiClusterMinimalTestVersion) - - cleanupHost := false - i := installer.NewCephInstaller(t, kh.Clientset, false, "", installer.VersionMaster, installer.NautilusVersion, cleanupHost) - - op := &MCTestOperations{i, kh, t, namespace1, namespace2, installer.SystemNamespace(namespace1), "", false} - if kh.VersionAtLeast("v1.13.0") { - op.testOverPVC = true - op.storageClassName = "manual" + externalSettings := &installer.TestCephSettings{ + IsExternal: true, + ClusterName: "test-external", + Namespace: "multi-external", + OperatorNamespace: s.settings.OperatorNamespace, + RookVersion: s.settings.RookVersion, + UseCSI: true, } - op.Setup() - return op, kh -} + s.externalManifests = installer.NewCephManifests(externalSettings) -// Setup is wrapper for setting up multiple rook clusters. -func (o MCTestOperations) Setup() { - var err error - if o.testOverPVC { - root, err := utils.FindRookRoot() - require.NoError(o.T(), err, "failed to get rook root") - cmdArgs := utils.CommandArgs{Command: filepath.Join(root, localPathPVCmd), - CmdArgs: []string{installer.TestScratchDevice()}} - cmdOut := utils.ExecuteCommand(cmdArgs) - require.NoError(o.T(), cmdOut.Err) + k8sh, err := utils.CreateK8sHelper(s.T) + assert.NoError(s.T(), err) + if !k8sh.VersionAtLeast("v1.16.0") { + s.T().Skip("requires at least k8s 1.16, no need to run on older versions") } - err = o.installer.CreateCephOperator(installer.SystemNamespace(o.namespace1)) - require.NoError(o.T(), err) - - require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-operator", o.systemNamespace, "Running"), - "Make sure rook-operator is in running state") - - require.True(o.T(), o.kh.IsPodInExpectedState("rook-discover", o.systemNamespace, "Running"), - "Make sure rook-discover is in running state") - - time.Sleep(10 * time.Second) - - // start the two clusters in parallel - logger.Infof("starting two clusters, one traditional and one external") - err = o.startCluster(o.namespace1, "bluestore") - require.NoError(o.T(), err) - // Wait for the monitors to be up so that we can fetch the configmap and secrets - require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-mon", o.namespace1, "Running"), - "Make sure rook-ceph-mon is in running state") + // Start the core storage cluster + s.setupMultiClusterCore() + s.createPools() + // Start the external cluster that will connect to the core cluster // create an external cluster - err = o.startExternalCluster(o.namespace2) - require.NoError(o.T(), err) + s.startExternalCluster() logger.Infof("finished starting clusters") } -// Teardown is a wrapper for tearDown after suite -func (o MCTestOperations) Teardown() { - o.installer.UninstallRookFromMultipleNS(installer.SystemNamespace(o.namespace1), o.namespace1, o.namespace2) +func (s *MultiClusterDeploySuite) AfterTest(suiteName, testName string) { + s.installer.CollectOperatorLog(suiteName, testName) } -func (o MCTestOperations) startCluster(namespace, store string) error { - logger.Infof("starting cluster %s", namespace) - multipleManagers := true - err := o.installer.CreateRookCluster(namespace, o.systemNamespace, store, o.testOverPVC, o.storageClassName, - cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, multipleManagers, false, false, installer.NautilusVersion) +func (s *MultiClusterDeploySuite) createPools() { + // create a test pool in each cluster so that we get some PGs + logger.Infof("Creating pool %s", s.poolName) + err := s.testClient.PoolClient.Create(s.poolName, s.settings.Namespace, 1) + require.Nil(s.T(), err) +} + +func (s *MultiClusterDeploySuite) deletePools() { + // create a test pool in each cluster so that we get some PGs + clusterInfo := client.AdminClusterInfo(s.settings.Namespace) + if err := s.testClient.PoolClient.DeletePool(s.testClient.BlockClient, clusterInfo, s.poolName); err != nil { + logger.Errorf("failed to delete pool %q. %v", s.poolName, err) + } else { + logger.Infof("deleted pool %q", s.poolName) + } +} + +func (s *MultiClusterDeploySuite) TearDownSuite() { + s.deletePools() + s.installer.UninstallRookFromMultipleNS(s.installer.Manifests, s.externalManifests) +} + +// Test to make sure all rook components are installed and Running +func (s *MultiClusterDeploySuite) TestInstallingMultipleRookClusters() { + // Check if Rook cluster 1 is deployed successfully + client.RunAllCephCommandsInToolboxPod = s.coreToolbox + checkIfRookClusterIsInstalled(s.Suite, s.k8sh, s.settings.OperatorNamespace, s.settings.Namespace, 1) + checkIfRookClusterIsHealthy(s.Suite, s.testClient, s.settings.Namespace) + + // Check if Rook external cluster is deployed successfully + // Checking health status is enough to validate the connection + client.RunAllCephCommandsInToolboxPod = s.externalToolbox + checkIfRookClusterIsHealthy(s.Suite, s.testClient, s.externalManifests.Settings().Namespace) +} + +// Setup is wrapper for setting up multiple rook clusters. +func (s *MultiClusterDeploySuite) setupMultiClusterCore() { + root, err := utils.FindRookRoot() + require.NoError(s.T(), err, "failed to get rook root") + cmdArgs := utils.CommandArgs{Command: filepath.Join(root, localPathPVCmd), + CmdArgs: []string{installer.TestScratchDevice()}} + cmdOut := utils.ExecuteCommand(cmdArgs) + require.NoError(s.T(), cmdOut.Err) + + s.installer, s.k8sh = StartTestCluster(s.T, s.settings, multiClusterMinimalTestVersion) + s.testClient = clients.CreateTestClient(s.k8sh, s.installer.Manifests) + s.coreToolbox = client.RunAllCephCommandsInToolboxPod +} + +func (s *MultiClusterDeploySuite) startExternalCluster() { + err := s.installer.CreateRookExternalCluster(s.externalManifests) if err != nil { - o.T().Fail() - o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace) - return fmt.Errorf("failed to create cluster %s. %+v", namespace, err) + s.T().Fail() + s.installer.GatherAllRookLogs(s.T().Name(), s.externalManifests.Settings().Namespace) + require.NoError(s.T(), err) } - if err := o.installer.CreateRookToolbox(namespace); err != nil { - o.T().Fail() - o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace) - return fmt.Errorf("failed to create toolbox for %s. %+v", namespace, err) - } - logger.Infof("succeeded starting cluster %s", namespace) - return nil -} - -func (o MCTestOperations) startExternalCluster(namespace string) error { - logger.Infof("starting external cluster %q", namespace) - err := o.installer.CreateRookExternalCluster(namespace, o.namespace1) - if err != nil { - o.T().Fail() - o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace) - return fmt.Errorf("failed to create external cluster %s. %+v", namespace, err) - } - - logger.Infof("running toolbox on namespace %q", namespace) - if err := o.installer.CreateRookToolbox(namespace); err != nil { - o.T().Fail() - o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace) - return fmt.Errorf("failed to create toolbox for %s. %+v", namespace, err) - } - - logger.Infof("succeeded starting external cluster %s", namespace) - return nil + s.externalToolbox = client.RunAllCephCommandsInToolboxPod + logger.Infof("succeeded starting external cluster %s", s.externalManifests.Settings().Namespace) } diff --git a/tests/integration/ceph_smoke_test.go b/tests/integration/ceph_smoke_test.go index fa6981343..98c7a985c 100644 --- a/tests/integration/ceph_smoke_test.go +++ b/tests/integration/ceph_smoke_test.go @@ -72,7 +72,7 @@ func TestCephSmokeSuite(t *testing.T) { s := new(SmokeSuite) defer func(s *SmokeSuite) { - HandlePanics(recover(), s.op, s.T) + HandlePanics(recover(), s.TearDownSuite, s.T) }(s) suite.Run(t, s) } @@ -80,83 +80,84 @@ func TestCephSmokeSuite(t *testing.T) { type SmokeSuite struct { suite.Suite helper *clients.TestClient - op *TestCluster + settings *installer.TestCephSettings + installer *installer.CephInstaller k8sh *utils.K8sHelper - namespace string } -func (suite *SmokeSuite) SetupSuite() { - suite.namespace = "smoke-ns" - smokeTestCluster := TestCluster{ - namespace: suite.namespace, - storeType: "bluestore", - storageClassName: installer.StorageClassName(), - useHelm: false, - usePVC: installer.UsePVC(), - mons: 3, - rbdMirrorWorkers: 1, - rookCephCleanup: true, - skipOSDCreation: false, - minimalMatrixK8sVersion: smokeSuiteMinimalTestVersion, - rookVersion: installer.VersionMaster, - cephVersion: installer.OctopusVersion, +func (s *SmokeSuite) SetupSuite() { + namespace := "smoke-ns" + s.settings = &installer.TestCephSettings{ + ClusterName: "smoke-cluster", + Namespace: namespace, + OperatorNamespace: installer.SystemNamespace(namespace), + StorageClassName: installer.StorageClassName(), + UseHelm: false, + UsePVC: installer.UsePVC(), + Mons: 3, + SkipOSDCreation: false, + UseCSI: true, + EnableAdmissionController: true, + UseCrashPruner: true, + RookVersion: installer.VersionMaster, + CephVersion: installer.OctopusVersion, } - suite.op, suite.k8sh = StartTestCluster(suite.T, &smokeTestCluster) - suite.helper = clients.CreateTestClient(suite.k8sh, suite.op.installer.Manifests) + s.installer, s.k8sh = StartTestCluster(s.T, s.settings, smokeSuiteMinimalTestVersion) + s.helper = clients.CreateTestClient(s.k8sh, s.installer.Manifests) } -func (suite *SmokeSuite) AfterTest(suiteName, testName string) { - suite.op.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(suite.namespace)) +func (s *SmokeSuite) AfterTest(suiteName, testName string) { + s.installer.CollectOperatorLog(suiteName, testName) } -func (suite *SmokeSuite) TearDownSuite() { - suite.op.Teardown() +func (s *SmokeSuite) TearDownSuite() { + s.installer.UninstallRook() } -func (suite *SmokeSuite) TestBlockStorage_SmokeTest() { - runBlockCSITest(suite.helper, suite.k8sh, suite.Suite, suite.namespace) +func (s *SmokeSuite) TestBlockStorage_SmokeTest() { + runBlockCSITest(s.helper, s.k8sh, s.Suite, s.settings.Namespace) } -func (suite *SmokeSuite) TestFileStorage_SmokeTest() { - useCSI := true +func (s *SmokeSuite) TestFileStorage_SmokeTest() { preserveFilesystemOnDelete := true - runFileE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace, "smoke-test-fs", useCSI, preserveFilesystemOnDelete) + runFileE2ETest(s.helper, s.k8sh, s.Suite, s.settings, "smoke-test-fs", preserveFilesystemOnDelete) } -func (suite *SmokeSuite) TestObjectStorage_SmokeTest() { - if !utils.IsPlatformOpenShift() { - runObjectE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace) +func (s *SmokeSuite) TestObjectStorage_SmokeTest() { + if utils.IsPlatformOpenShift() { + s.T().Skip("object store tests skipped on openshift") } + runObjectE2ETest(s.helper, s.k8sh, s.Suite, s.settings.Namespace) } // Test to make sure all rook components are installed and Running -func (suite *SmokeSuite) TestARookClusterInstallation_SmokeTest() { - checkIfRookClusterIsInstalled(suite.Suite, suite.k8sh, installer.SystemNamespace(suite.namespace), suite.namespace, 3) +func (s *SmokeSuite) TestARookClusterInstallation_SmokeTest() { + checkIfRookClusterIsInstalled(s.Suite, s.k8sh, s.settings.OperatorNamespace, s.settings.Namespace, 3) } // Smoke Test for Mon failover - Test check the following operations for the Mon failover in order // Delete mon pod, Wait for new mon pod -func (suite *SmokeSuite) TestMonFailover() { +func (s *SmokeSuite) TestMonFailover() { ctx := context.TODO() logger.Infof("Mon Failover Smoke Test") - deployments, err := suite.getNonCanaryMonDeployments() - require.Nil(suite.T(), err) - require.Equal(suite.T(), 3, len(deployments)) + deployments, err := s.getNonCanaryMonDeployments() + require.NoError(s.T(), err) + require.Equal(s.T(), 3, len(deployments)) monToKill := deployments[0].Name logger.Infof("Killing mon %s", monToKill) propagation := metav1.DeletePropagationForeground delOptions := &metav1.DeleteOptions{PropagationPolicy: &propagation} - err = suite.k8sh.Clientset.AppsV1().Deployments(suite.namespace).Delete(ctx, monToKill, *delOptions) - require.Nil(suite.T(), err) + err = s.k8sh.Clientset.AppsV1().Deployments(s.settings.Namespace).Delete(ctx, monToKill, *delOptions) + require.NoError(s.T(), err) // Wait for the health check to start a new monitor originalMonDeleted := false for i := 0; i < 30; i++ { - deployments, err := suite.getNonCanaryMonDeployments() - require.Nil(suite.T(), err) + deployments, err := s.getNonCanaryMonDeployments() + require.NoError(s.T(), err) // Make sure the old mon is not still alive foundOldMon := false @@ -188,31 +189,31 @@ func (suite *SmokeSuite) TestMonFailover() { return } - assert.Equal(suite.T(), 2, len(deployments)) + assert.Equal(s.T(), 2, len(deployments)) } time.Sleep(5 * time.Second) } - require.Fail(suite.T(), "giving up waiting for a new monitor") + require.Fail(s.T(), "giving up waiting for a new monitor") } // Smoke Test for pool Resizing -func (suite *SmokeSuite) TestPoolResize() { +func (s *SmokeSuite) TestPoolResize() { ctx := context.TODO() logger.Infof("Pool Resize Smoke Test") poolName := "testpool" - err := suite.helper.PoolClient.Create(poolName, suite.namespace, 1) - require.Nil(suite.T(), err) + err := s.helper.PoolClient.Create(poolName, s.settings.Namespace, 1) + require.NoError(s.T(), err) poolFound := false - clusterInfo := client.AdminClusterInfo(suite.namespace) + clusterInfo := client.AdminClusterInfo(s.settings.Namespace) // Wait for pool to appear for i := 0; i < 10; i++ { - pools, err := suite.helper.PoolClient.ListCephPools(clusterInfo) - require.Nil(suite.T(), err) + pools, err := s.helper.PoolClient.ListCephPools(clusterInfo) + require.NoError(s.T(), err) for _, p := range pools { if p.Name != poolName { continue @@ -227,24 +228,24 @@ func (suite *SmokeSuite) TestPoolResize() { time.Sleep(2 * time.Second) } - require.Equal(suite.T(), true, poolFound, "pool not found") + require.Equal(s.T(), true, poolFound, "pool not found") - err = suite.helper.PoolClient.Update(poolName, suite.namespace, 2) - require.Nil(suite.T(), err) + err = s.helper.PoolClient.Update(poolName, s.settings.Namespace, 2) + require.NoError(s.T(), err) poolResized := false // Wait for pool resize to happen for i := 0; i < 10; i++ { - details, err := suite.helper.PoolClient.GetCephPoolDetails(clusterInfo, poolName) - require.Nil(suite.T(), err) + details, err := s.helper.PoolClient.GetCephPoolDetails(clusterInfo, poolName) + require.NoError(s.T(), err) if details.Size > 1 { logger.Infof("pool %s size was updated", poolName) - require.Equal(suite.T(), 2, int(details.Size)) + require.Equal(s.T(), 2, int(details.Size)) poolResized = true // resize the pool back to 1 to avoid hangs around not having enough OSDs to satisfy rbd - err = suite.helper.PoolClient.Update(poolName, suite.namespace, 1) - require.Nil(suite.T(), err) + err = s.helper.PoolClient.Update(poolName, s.settings.Namespace, 1) + require.NoError(s.T(), err) } else if poolResized && details.Size == 1 { logger.Infof("pool resized back to 1") break @@ -255,45 +256,45 @@ func (suite *SmokeSuite) TestPoolResize() { time.Sleep(2 * time.Second) } - require.Equal(suite.T(), true, poolResized, fmt.Sprintf("pool %s not found", poolName)) + require.Equal(s.T(), true, poolResized, fmt.Sprintf("pool %s not found", poolName)) // Verify the Kubernetes Secret has been created (bootstrap peer token) - pool, err := suite.k8sh.RookClientset.CephV1().CephBlockPools(suite.namespace).Get(ctx, poolName, metav1.GetOptions{}) - assert.NoError(suite.T(), err) + pool, err := s.k8sh.RookClientset.CephV1().CephBlockPools(s.settings.Namespace).Get(ctx, poolName, metav1.GetOptions{}) + assert.NoError(s.T(), err) if pool.Spec.Mirroring.Enabled { secretName := pool.Status.Info[oppool.RBDMirrorBootstrapPeerSecretName] - assert.NotEmpty(suite.T(), secretName) + assert.NotEmpty(s.T(), secretName) // now fetch the secret which contains the bootstrap peer token - s, err := suite.k8sh.Clientset.CoreV1().Secrets(suite.namespace).Get(ctx, secretName, metav1.GetOptions{}) - require.Nil(suite.T(), err) - assert.NotEmpty(suite.T(), s.Data["token"]) + secret, err := s.k8sh.Clientset.CoreV1().Secrets(s.settings.Namespace).Get(ctx, secretName, metav1.GetOptions{}) + require.NoError(s.T(), err) + assert.NotEmpty(s.T(), secret.Data["token"]) // Once we have a scenario with another Ceph cluster - needs to be added in the MultiCluster suite // We would need to add a bootstrap peer token following the below procedure // bootstrapSecretName := "bootstrap-peer-token" // token := "eyJmc2lkIjoiYzZiMDg3ZjItNzgyOS00ZGJiLWJjZmMtNTNkYzM0ZTBiMzVkIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBV1lsWmZVQ1Q2RGhBQVBtVnAwbGtubDA5YVZWS3lyRVV1NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMTExLjEwOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTA6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjEyOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTI6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjExOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTE6Njc4OV0ifQ==" - // s = oppool.GenerateBootstrapPeerSecret(bootstrapSecretName, suite.namespace, string(pool.GetUID()), []byte(token)) - // s, err = suite.k8sh.Clientset.CoreV1().Secrets(suite.namespace).Create(s) - // require.Nil(suite.T(), err, err.Error()) + // s = oppool.GenerateBootstrapPeerSecret(bootstrapSecretName, s.settings.Namespace, string(pool.GetUID()), []byte(token)) + // s, err = s.k8sh.Clientset.CoreV1().Secrets(s.settings.Namespace).Create(s) + // require.Nil(s.T(), err, err.Error()) // // update the ceph block pool cr // pool.Spec.Mirrored.PeersSecretNames = append(pool.Spec.Mirrored.PeersSecretNames, bootstrapSecretName) - // _, err = suite.k8sh.RookClientset.CephV1().CephBlockPools(suite.namespace).Update(pool) - // require.Nil(suite.T(), err, err.Error()) + // _, err = s.k8sh.RookClientset.CephV1().CephBlockPools(s.settings.Namespace).Update(pool) + // require.Nil(s.T(), err, err.Error()) - // mirrorInfo, err := client.PrintPoolMirroringInfo(suite.k8sh.MakeContext(), clusterInfo, poolName) - // require.Nil(suite.T(), err, err.Error()) - // assert.Equal(suite.T(), "image", mirrorInfo.Mode) - // assert.Equal(suite.T(), 1, len(mirrorInfo.Peers)) + // mirrorInfo, err := client.PrintPoolMirroringInfo(s.k8sh.MakeContext(), clusterInfo, poolName) + // require.Nil(s.T(), err, err.Error()) + // assert.Equal(s.T(), "image", mirrorInfo.Mode) + // assert.Equal(s.T(), 1, len(mirrorInfo.Peers)) } // clean up the pool - err = suite.helper.PoolClient.DeletePool(suite.helper.BlockClient, clusterInfo, poolName) - assert.NoError(suite.T(), err) + err = s.helper.PoolClient.DeletePool(s.helper.BlockClient, clusterInfo, poolName) + assert.NoError(s.T(), err) } // Smoke Test for Client CRD -func (suite *SmokeSuite) TestCreateClient() { +func (s *SmokeSuite) TestCreateClient() { logger.Infof("Create Client Smoke Test") clientName := "client1" @@ -302,14 +303,14 @@ func (suite *SmokeSuite) TestCreateClient() { "mgr": "allow rwx", "osd": "allow rwx", } - clusterInfo := client.AdminClusterInfo(suite.namespace) - err := suite.helper.UserClient.Create(clientName, suite.namespace, caps) - require.Nil(suite.T(), err) + clusterInfo := client.AdminClusterInfo(s.settings.Namespace) + err := s.helper.UserClient.Create(clientName, s.settings.Namespace, caps) + require.NoError(s.T(), err) clientFound := false for i := 0; i < 30; i++ { - clients, _ := suite.helper.UserClient.Get(clusterInfo, "client."+clientName) + clients, _ := s.helper.UserClient.Get(clusterInfo, "client."+clientName) if clients != "" { clientFound = true } @@ -322,7 +323,7 @@ func (suite *SmokeSuite) TestCreateClient() { time.Sleep(2 * time.Second) } - require.Equal(suite.T(), true, clientFound, "client not found") + assert.Equal(s.T(), true, clientFound, "client not found") logger.Infof("Update Client Smoke Test") newcaps := map[string]string{ @@ -330,30 +331,33 @@ func (suite *SmokeSuite) TestCreateClient() { "mgr": "allow rw", "osd": "allow *", } - caps, _ = suite.helper.UserClient.Update(clusterInfo, clientName, newcaps) + caps, _ = s.helper.UserClient.Update(clusterInfo, clientName, newcaps) - require.Equal(suite.T(), "allow r", caps["mon"], "wrong caps") - require.Equal(suite.T(), "allow rw", caps["mgr"], "wrong caps") - require.Equal(suite.T(), "allow *", caps["osd"], "wrong caps") + assert.Equal(s.T(), "allow r", caps["mon"], "wrong caps") + assert.Equal(s.T(), "allow rw", caps["mgr"], "wrong caps") + assert.Equal(s.T(), "allow *", caps["osd"], "wrong caps") + + err = s.helper.UserClient.Delete(clientName, s.settings.Namespace) + require.NoError(s.T(), err) } // Smoke Test for RBD Mirror CRD -func (suite *SmokeSuite) TestCreateRBDMirrorClient() { +func (s *SmokeSuite) TestCreateRBDMirrorClient() { logger.Infof("Create rbd-mirror Smoke Test") rbdMirrorName := "my-rbd-mirror" - err := suite.helper.RBDMirrorClient.Create(suite.namespace, rbdMirrorName, 1) - require.Nil(suite.T(), err) + err := s.helper.RBDMirrorClient.Create(s.settings.Namespace, rbdMirrorName, 1) + require.NoError(s.T(), err) - err = suite.helper.RBDMirrorClient.Delete(suite.namespace, rbdMirrorName) - require.Nil(suite.T(), err) + err = s.helper.RBDMirrorClient.Delete(s.settings.Namespace, rbdMirrorName) + require.NoError(s.T(), err) } -func (suite *SmokeSuite) getNonCanaryMonDeployments() ([]appsv1.Deployment, error) { +func (s *SmokeSuite) getNonCanaryMonDeployments() ([]appsv1.Deployment, error) { ctx := context.TODO() opts := metav1.ListOptions{LabelSelector: "app=rook-ceph-mon"} - deployments, err := suite.k8sh.Clientset.AppsV1().Deployments(suite.namespace).List(ctx, opts) + deployments, err := s.k8sh.Clientset.AppsV1().Deployments(s.settings.Namespace).List(ctx, opts) if err != nil { return nil, err } diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index 45d88f93f..4ebf92714 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -58,7 +58,7 @@ func TestCephUpgradeSuite(t *testing.T) { s := new(UpgradeSuite) defer func(s *UpgradeSuite) { - HandlePanics(recover(), s.op, s.T) + HandlePanics(recover(), s.TearDownSuite, s.T) }(s) suite.Run(t, s) } @@ -66,40 +66,37 @@ func TestCephUpgradeSuite(t *testing.T) { type UpgradeSuite struct { suite.Suite helper *clients.TestClient - op *TestCluster k8sh *utils.K8sHelper + settings *installer.TestCephSettings + installer *installer.CephInstaller namespace string } func (s *UpgradeSuite) SetupSuite() { s.namespace = "upgrade-ns" - upgradeTestCluster := TestCluster{ - clusterName: s.namespace, - namespace: s.namespace, - storeType: "", - storageClassName: "", - useHelm: false, - usePVC: false, - mons: 1, - rbdMirrorWorkers: 0, - rookCephCleanup: false, - skipOSDCreation: false, - minimalMatrixK8sVersion: upgradeMinimalTestVersion, - rookVersion: installer.Version1_5, - cephVersion: installer.NautilusVersion, + s.settings = &installer.TestCephSettings{ + ClusterName: s.namespace, + Namespace: s.namespace, + OperatorNamespace: installer.SystemNamespace(s.namespace), + StorageClassName: "", + UseHelm: false, + UsePVC: false, + Mons: 1, + UseCSI: true, + SkipOSDCreation: false, + RookVersion: installer.Version1_5, + CephVersion: installer.NautilusVersion, } - s.op, s.k8sh = StartTestCluster(s.T, &upgradeTestCluster) - s.helper = clients.CreateTestClient(s.k8sh, s.op.installer.Manifests) + s.installer, s.k8sh = StartTestCluster(s.T, s.settings, upgradeMinimalTestVersion) + s.helper = clients.CreateTestClient(s.k8sh, s.installer.Manifests) } func (s *UpgradeSuite) TearDownSuite() { - s.op.Teardown() + s.installer.UninstallRook() } func (s *UpgradeSuite) TestUpgradeToMaster() { - systemNamespace := installer.SystemNamespace(s.namespace) - // // Create block, object, and file storage before the upgrade // @@ -108,7 +105,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { blockName := "block-claim-upgrade" logger.Infof("Initializing block before the upgrade") clusterInfo := client.AdminClusterInfo(s.namespace) - setupBlockLite(s.helper, s.k8sh, s.Suite, clusterInfo, systemNamespace, poolName, storageClassName, blockName, rbdPodName, s.op.installer.CephVersion) + setupBlockLite(s.helper, s.k8sh, s.Suite, clusterInfo, poolName, storageClassName, blockName, rbdPodName) createPodWithBlock(s.helper, s.k8sh, s.Suite, s.namespace, storageClassName, rbdPodName, blockName) @@ -121,13 +118,12 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { logger.Infof("Initializing file before the upgrade") filesystemName := "upgrade-test-fs" activeCount := 1 - createFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, activeCount) + createFilesystem(s.helper, s.k8sh, s.Suite, s.settings, filesystemName, activeCount) // Start the file test client fsStorageClass := "file-upgrade" - assert.NoError(s.T(), s.helper.FSClient.CreateStorageClass(filesystemName, systemNamespace, s.namespace, fsStorageClass)) - useCSI := true - createFilesystemConsumerPod(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, fsStorageClass, useCSI) + assert.NoError(s.T(), s.helper.FSClient.CreateStorageClass(filesystemName, s.settings.OperatorNamespace, s.namespace, fsStorageClass)) + createFilesystemConsumerPod(s.helper, s.k8sh, s.Suite, s.settings, filesystemName, fsStorageClass) defer func() { cleanupFilesystemConsumer(s.helper, s.k8sh, s.Suite, s.namespace, filePodName) cleanupFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName) @@ -135,7 +131,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { logger.Infof("Initializing object before the upgrade") objectStoreName := "upgraded-object" - runObjectE2ETestLite(s.helper, s.k8sh, s.Suite, s.namespace, objectStoreName, 1, false) + runObjectE2ETestLite(s.helper, s.k8sh, s.Suite, s.settings, objectStoreName, 1, false) logger.Info("Initializing object bucket claim before the upgrade") bucketStorageClassName := "rook-smoke-delete-bucket" @@ -147,6 +143,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { defer func() { _ = s.helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketPrefix, maxObject, false) _ = s.helper.BucketClient.DeleteBucketStorageClass(s.namespace, objectStoreName, bucketStorageClassName, "Delete", region) + objectStoreCleanUp(s.Suite, s.helper, s.k8sh, s.settings.Namespace, objectStoreName) }() created := utils.Retry(12, 2*time.Second, "OBC is created", func() bool { @@ -178,7 +175,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { // Upgrade Rook from v1.5 to master // logger.Infof("*** UPGRADING ROOK FROM %s to master ***", installer.Version1_5) - s.gatherLogs(systemNamespace, "_before_master_upgrade") + s.gatherLogs(s.settings.OperatorNamespace, "_before_master_upgrade") s.upgradeToMaster() s.verifyOperatorImage(installer.VersionMaster) @@ -199,7 +196,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { // Upgrade from nautilus to octopus // logger.Infof("*** UPGRADING CEPH FROM Nautilus TO Octopus ***") - s.gatherLogs(systemNamespace, "_before_octopus_upgrade") + s.gatherLogs(s.settings.OperatorNamespace, "_before_octopus_upgrade") s.upgradeCephVersion(installer.OctopusVersion.Image, numOSDs) // Verify reading and writing to the test clients newFile = "post-octopus-upgrade-file" @@ -213,7 +210,7 @@ func (s *UpgradeSuite) gatherLogs(systemNamespace, testSuffix string) { s.k8sh.PrintPodDescribe(s.namespace) } n := strings.Replace(s.T().Name(), "/", "_", -1) + testSuffix - s.op.installer.GatherAllRookLogs(n, systemNamespace, s.namespace) + s.installer.GatherAllRookLogs(n, systemNamespace, s.namespace) } func (s *UpgradeSuite) upgradeCephVersion(newCephImage string, numOSDs int) { @@ -241,7 +238,7 @@ func (s *UpgradeSuite) verifyRookUpgrade(numOSDs int) { // Get some info about the currently deployed mons to determine later if they are all updated monDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-mon") require.NoError(s.T(), err) - require.Equal(s.T(), s.op.mons, len(monDepList.Items), monDepList.Items) + require.Equal(s.T(), s.settings.Mons, len(monDepList.Items), monDepList.Items) // Get some info about the currently deployed mgr to determine later if it is updated mgrDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-mgr") @@ -263,7 +260,7 @@ func (s *UpgradeSuite) verifyRookUpgrade(numOSDs int) { func (s *UpgradeSuite) waitForUpgradedDaemons(previousVersion, versionLabel string, numOSDs int, waitForMDS bool) { // wait for the mon(s) to be updated monsNotOldVersion := fmt.Sprintf("app=rook-ceph-mon,%s!=%s", versionLabel, previousVersion) - err := s.k8sh.WaitForDeploymentCount(monsNotOldVersion, s.namespace, s.op.mons) + err := s.k8sh.WaitForDeploymentCount(monsNotOldVersion, s.namespace, s.settings.Mons) require.NoError(s.T(), err, "mon(s) didn't update") err = s.k8sh.WaitForLabeledDeploymentsToBeReady(monsNotOldVersion, s.namespace) require.NoError(s.T(), err) @@ -336,19 +333,13 @@ func (s *UpgradeSuite) verifyFilesAfterUpgrade(fsName, newFileToWrite, messageFo // UpgradeToMaster performs the steps necessary to upgrade a Rook v1.4 cluster to master. It does not // verify the upgrade but merely starts the upgrade process. func (s *UpgradeSuite) upgradeToMaster() { - v1ExtensionsSupported := false - if s.k8sh.VersionAtLeast("v1.16.0") { - v1ExtensionsSupported = true - } - require.NoError(s.T(), s.k8sh.ResourceOperation("apply", upgradeManifestToMaster(s.namespace, v1ExtensionsSupported))) + // Apply the CRDs for the latest master + s.settings.RookVersion = installer.VersionMaster + m := installer.NewCephManifests(s.settings) + require.NoError(s.T(), s.k8sh.ResourceOperation("apply", m.GetCRDs(s.k8sh))) + + require.NoError(s.T(), s.k8sh.ResourceOperation("apply", m.GetCommon())) require.NoError(s.T(), - s.k8sh.SetDeploymentVersion(installer.SystemNamespace(s.namespace), operatorContainer, operatorContainer, installer.VersionMaster)) -} - -func upgradeManifestToMaster(namespace string, useV1Extensions bool) string { - // Always apply CRDs for the latest master - m := installer.NewCephManifests(installer.VersionMaster) - manifests := m.GetRookCRDs(useV1Extensions) - return manifests + s.k8sh.SetDeploymentVersion(s.settings.OperatorNamespace, operatorContainer, operatorContainer, installer.VersionMaster)) } diff --git a/tests/integration/nfs_test.go b/tests/integration/nfs_test.go index 6c4a2b60a..e84659532 100644 --- a/tests/integration/nfs_test.go +++ b/tests/integration/nfs_test.go @@ -45,7 +45,7 @@ func TestNfsSuite(t *testing.T) { s := new(NfsSuite) defer func(s *NfsSuite) { - HandlePanics(recover(), s, s.T) + HandlePanics(recover(), s.Teardown, s.T) }(s) suite.Run(t, s) } @@ -60,87 +60,87 @@ type NfsSuite struct { instanceCount int } -func (suite *NfsSuite) SetupSuite() { - suite.Setup() +func (s *NfsSuite) SetupSuite() { + s.Setup() } -func (suite *NfsSuite) TearDownSuite() { - suite.Teardown() +func (s *NfsSuite) TearDownSuite() { + s.Teardown() } -func (suite *NfsSuite) Setup() { - suite.namespace = "rook-nfs" - suite.systemNamespace = installer.SystemNamespace(suite.namespace) - suite.instanceCount = 1 +func (s *NfsSuite) Setup() { + s.namespace = "rook-nfs" + s.systemNamespace = installer.SystemNamespace(s.namespace) + s.instanceCount = 1 - k8shelper, err := utils.CreateK8sHelper(suite.T) + k8shelper, err := utils.CreateK8sHelper(s.T) v := version.MustParseSemantic(k8shelper.GetK8sServerVersion()) if !v.AtLeast(version.MustParseSemantic("1.14.0")) { logger.Info("Skipping NFS tests when not at least K8s v1.14") - suite.T().Skip() + s.T().Skip() } - require.NoError(suite.T(), err) - suite.k8shelper = k8shelper + require.NoError(s.T(), err) + s.k8shelper = k8shelper - k8sversion := suite.k8shelper.GetK8sServerVersion() + k8sversion := s.k8shelper.GetK8sServerVersion() logger.Infof("Installing nfs server on k8s %s", k8sversion) - suite.installer = installer.NewNFSInstaller(suite.k8shelper, suite.T) + s.installer = installer.NewNFSInstaller(s.k8shelper, s.T) - suite.rwClient = clients.CreateReadWriteOperation(suite.k8shelper) + s.rwClient = clients.CreateReadWriteOperation(s.k8shelper) - err = suite.installer.InstallNFSServer(suite.systemNamespace, suite.namespace, suite.instanceCount) + err = s.installer.InstallNFSServer(s.systemNamespace, s.namespace, s.instanceCount) if err != nil { logger.Errorf("nfs server installation failed: %+v", err) - suite.T().Fail() - suite.Teardown() - suite.T().FailNow() + s.T().Fail() + s.Teardown() + s.T().FailNow() } } -func (suite *NfsSuite) Teardown() { - suite.installer.GatherAllNFSServerLogs(suite.systemNamespace, suite.namespace, suite.T().Name()) - suite.installer.UninstallNFSServer(suite.systemNamespace, suite.namespace) +func (s *NfsSuite) Teardown() { + s.installer.GatherAllNFSServerLogs(s.systemNamespace, s.namespace, s.T().Name()) + s.installer.UninstallNFSServer(s.systemNamespace, s.namespace) } -func (suite *NfsSuite) TestNfsServerInstallation() { - logger.Infof("Verifying that nfs server pod %s is running", suite.namespace) +func (s *NfsSuite) TestNfsServerInstallation() { + logger.Infof("Verifying that nfs server pod %s is running", s.namespace) // verify nfs server operator is running OK - assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-nfs-operator", suite.systemNamespace, 1, "Running"), + assert.True(s.T(), s.k8shelper.CheckPodCountAndState("rook-nfs-operator", s.systemNamespace, 1, "Running"), "1 rook-nfs-operator must be in Running state") // verify nfs server instances are running OK - assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState(suite.namespace, suite.namespace, suite.instanceCount, "Running"), - fmt.Sprintf("%d rook-nfs pods must be in Running state", suite.instanceCount)) + assert.True(s.T(), s.k8shelper.CheckPodCountAndState(s.namespace, s.namespace, s.instanceCount, "Running"), + fmt.Sprintf("%d rook-nfs pods must be in Running state", s.instanceCount)) // verify bigger export is running OK - assert.True(suite.T(), true, suite.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim-bigger")) + assert.True(s.T(), true, s.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim-bigger")) - podList, err := suite.rwClient.CreateWriteClient("nfs-pv-claim-bigger") - require.NoError(suite.T(), err) - assert.True(suite.T(), true, suite.checkReadData(podList)) - err = suite.rwClient.Delete() - assert.NoError(suite.T(), err) + podList, err := s.rwClient.CreateWriteClient("nfs-pv-claim-bigger") + require.NoError(s.T(), err) + assert.True(s.T(), true, s.checkReadData(podList)) + err = s.rwClient.Delete() + assert.NoError(s.T(), err) // verify another smaller export is running OK - assert.True(suite.T(), true, suite.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim")) + assert.True(s.T(), true, s.k8shelper.WaitUntilPVCIsBound("default", "nfs-pv-claim")) - defer suite.rwClient.Delete() //nolint, delete a nfs consuming pod in rook - podList, err = suite.rwClient.CreateWriteClient("nfs-pv-claim") - require.NoError(suite.T(), err) - assert.True(suite.T(), true, suite.checkReadData(podList)) + defer s.rwClient.Delete() //nolint, delete a nfs consuming pod in rook + podList, err = s.rwClient.CreateWriteClient("nfs-pv-claim") + require.NoError(s.T(), err) + assert.True(s.T(), true, s.checkReadData(podList)) } -func (suite *NfsSuite) checkReadData(podList []string) bool { +func (s *NfsSuite) checkReadData(podList []string) bool { var result string var err error // the following for loop retries to read data from the first pod in the pod list for i := 0; i < utils.RetryLoop; i++ { // the nfs volume is mounted on "/mnt" and the data(hostname of the pod) is written in "/mnt/data" of the pod // results stores the hostname of either one of the pod which is same as the pod name, which is read from "/mnt/data" - result, err = suite.rwClient.Read(podList[0]) + result, err = s.rwClient.Read(podList[0]) logger.Infof("nfs volume read exited, err: %+v. result: %s", err, result) if err == nil { break @@ -148,7 +148,7 @@ func (suite *NfsSuite) checkReadData(podList []string) bool { logger.Warning("nfs volume read failed, will try again") time.Sleep(utils.RetryInterval * time.Second) } - require.NoError(suite.T(), err) + require.NoError(s.T(), err) // the value of result must be same as the name of pod. if result == podList[0] || result == podList[1] { return true diff --git a/tests/scripts/deploy_admission_controller.sh b/tests/scripts/deploy_admission_controller.sh index 440cb1900..6c0be04c7 100755 --- a/tests/scripts/deploy_admission_controller.sh +++ b/tests/scripts/deploy_admission_controller.sh @@ -17,7 +17,7 @@ trap cleanup SIGINT ERR SERVER_VERSION=$(kubectl version --short | awk -F "." '/Server Version/ {print $2}') MINIMUM_VERSION=16 -if [ ${SERVER_VERSION} -le ${MINIMUM_VERSION} ]; then +if [ ${SERVER_VERSION} -lt ${MINIMUM_VERSION} ]; then echo "required minimum kubernetes version 1.$MINIMUM_VERSION.0" exit fi From bdbf264e68c5409896070db76e5a11b930430c2b Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Mon, 15 Mar 2021 20:47:38 -0600 Subject: [PATCH 5/6] ceph: integration tests in github actions skip cleanup The integration tests run independently in the github actions so there is no need to cleanup from every test. The cleanup is still needed in the Jenkins tests where all the suites run serially. Signed-off-by: Travis Nielsen --- .../integration-test-flex-suite.yaml | 2 +- .../integration-test-helm-suite.yaml | 4 +- .../integration-test-multi-cluster-suite.yaml | 2 +- .../integration-test-smoke-suite.yaml | 4 +- .../integration-test-upgrade-suite.yaml | 2 +- Jenkinsfile | 2 + tests/framework/installer/ceph_installer.go | 90 +++++++++++-------- tests/framework/installer/ceph_manifests.go | 7 +- .../installer/ceph_manifests_v1.5.go | 7 +- tests/framework/installer/ceph_settings.go | 16 ++++ tests/framework/utils/k8s_helper.go | 10 +++ tests/integration/ceph_flex_test.go | 2 +- tests/integration/ceph_helm_test.go | 2 +- tests/integration/ceph_mgr_test.go | 2 +- tests/integration/ceph_multi_cluster_test.go | 2 + tests/integration/ceph_smoke_test.go | 1 + 16 files changed, 104 insertions(+), 51 deletions(-) diff --git a/.github/workflows/integration-test-flex-suite.yaml b/.github/workflows/integration-test-flex-suite.yaml index 8f32e6ec1..868f537e7 100644 --- a/.github/workflows/integration-test-flex-suite.yaml +++ b/.github/workflows/integration-test-flex-suite.yaml @@ -54,4 +54,4 @@ jobs: - name: setup tmate session for debugging if: failure() && contains(github.event.pull_request.labels.*.name, 'debug-flex-suite') - uses: mxschmitt/action-tmate@v3 \ No newline at end of file + uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/integration-test-helm-suite.yaml b/.github/workflows/integration-test-helm-suite.yaml index 313977802..ea8d66c5f 100644 --- a/.github/workflows/integration-test-helm-suite.yaml +++ b/.github/workflows/integration-test-helm-suite.yaml @@ -45,7 +45,7 @@ jobs: tests/scripts/minikube.sh helm tests/scripts/helm.sh up export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) - go test -v -timeout 1800s -run CephHelmSuite github.com/rook/rook/tests/integration + SKIP_TEST_CLEANUP=false SKIP_CLEANUP_POLICY=false go test -v -timeout 1800s -run CephHelmSuite github.com/rook/rook/tests/integration - name: Artifact uses: actions/upload-artifact@v2 @@ -56,4 +56,4 @@ jobs: - name: setup tmate session for debugging if: failure() && contains(github.event.pull_request.labels.*.name, 'debug-helm-suite') - uses: mxschmitt/action-tmate@v3 \ No newline at end of file + uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/integration-test-multi-cluster-suite.yaml b/.github/workflows/integration-test-multi-cluster-suite.yaml index e2b82bcbf..d799a35ab 100644 --- a/.github/workflows/integration-test-multi-cluster-suite.yaml +++ b/.github/workflows/integration-test-multi-cluster-suite.yaml @@ -55,4 +55,4 @@ jobs: - name: setup tmate session for debugging if: failure() && contains(github.event.pull_request.labels.*.name, 'debug-multi-cluster-suite') - uses: mxschmitt/action-tmate@v3 \ No newline at end of file + uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/integration-test-smoke-suite.yaml b/.github/workflows/integration-test-smoke-suite.yaml index 1c93d9143..b70b67320 100644 --- a/.github/workflows/integration-test-smoke-suite.yaml +++ b/.github/workflows/integration-test-smoke-suite.yaml @@ -43,7 +43,7 @@ jobs: - name: TestCephSmokeSuite run: | export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) - go test -v -timeout 1800s -run CephSmokeSuite github.com/rook/rook/tests/integration + SKIP_CLEANUP_POLICY=false go test -v -timeout 1800s -run CephSmokeSuite github.com/rook/rook/tests/integration - name: Artifact uses: actions/upload-artifact@v2 @@ -54,4 +54,4 @@ jobs: - name: setup tmate session for debugging if: failure() && contains(github.event.pull_request.labels.*.name, 'debug-smoke-suite') - uses: mxschmitt/action-tmate@v3 \ No newline at end of file + uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/integration-test-upgrade-suite.yaml b/.github/workflows/integration-test-upgrade-suite.yaml index 0e5964803..7f9b46459 100644 --- a/.github/workflows/integration-test-upgrade-suite.yaml +++ b/.github/workflows/integration-test-upgrade-suite.yaml @@ -54,4 +54,4 @@ jobs: - name: setup tmate session for debugging if: failure() && contains(github.event.pull_request.labels.*.name, 'debug-upgrade-suite') - uses: mxschmitt/action-tmate@v3 \ No newline at end of file + uses: mxschmitt/action-tmate@v3 diff --git a/Jenkinsfile b/Jenkinsfile index 45d911121..8fbe2c099 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -232,6 +232,8 @@ def RunIntegrationTest(k, v) { sh '''#!/bin/bash set -o pipefail export KUBECONFIG=$HOME/admin.conf \ + SKIP_TEST_CLEANUP=false \ + SKIP_CLEANUP_POLICY=false \ TEST_ENV_NAME='''+"${k}"+''' \ TEST_BASE_DIR="WORKING_DIR" \ TEST_LOG_COLLECTION_LEVEL='''+"${env.getLogs}"+''' \ diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 1e65cc2a2..9fab7efc4 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -97,17 +97,17 @@ func (h *CephInstaller) CreateCephOperator() (err error) { // Create the namespace and RBAC before starting the operator _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetCommon(), createFromStdinArgs...) if err != nil { - return fmt.Errorf("Failed to create rook-operator pod: %v ", err) + return errors.Errorf("Failed to create rook-operator pod: %v ", err) } err = h.startAdmissionController() if err != nil { - return fmt.Errorf("Failed to start admission controllers: %v", err) + return errors.Errorf("Failed to start admission controllers: %v", err) } _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetOperator(), createFromStdinArgs...) if err != nil { - return fmt.Errorf("Failed to create rook-operator pod: %v", err) + return errors.Errorf("Failed to create rook-operator pod: %v", err) } logger.Infof("Rook operator started") @@ -123,7 +123,6 @@ func (h *CephInstaller) startAdmissionController() error { logger.Info("skipping admission controller for this test suite") return nil } - // disable admission controller for Kubernetes version older than v1.16.0 if utils.IsPlatformOpenShift() { logger.Info("skipping the admission controller on OpenShift") return nil @@ -131,11 +130,11 @@ func (h *CephInstaller) startAdmissionController() error { rootPath, err := utils.FindRookRoot() if err != nil { - return fmt.Errorf("failed to find rook root. %v", err) + return errors.Errorf("failed to find rook root. %v", err) } userHome, err := os.UserHomeDir() if err != nil { - return fmt.Errorf("failed to find user home directory. %v", err) + return errors.Errorf("failed to find user home directory. %v", err) } scriptPath := path.Join(rootPath, "tests/scripts/deploy_admission_controller_test.sh") err = h.k8shelper.MakeContext().Executor.ExecuteCommandWithEnv([]string{fmt.Sprintf("NAMESPACE=%s", h.settings.OperatorNamespace), fmt.Sprintf("HOME=%s", userHome)}, "bash", scriptPath) @@ -153,17 +152,17 @@ func (h *CephInstaller) CreateRookOperatorViaHelm(chartSettings string) error { // create the operator namespace before the admission controller is created if err := h.k8shelper.CreateNamespace(h.settings.OperatorNamespace); err != nil { - return fmt.Errorf("failed to create namespace %s. %v", h.settings.Namespace, err) + return errors.Errorf("failed to create namespace %s. %v", h.settings.Namespace, err) } if err := h.startAdmissionController(); err != nil { - return fmt.Errorf("failed to start admission controllers: %v", err) + return errors.Errorf("failed to start admission controllers: %v", err) } if err := h.helmHelper.InstallLocalRookHelmChart(h.settings.Namespace, chartSettings); err != nil { - return fmt.Errorf("failed to install rook operator via helm, err : %v", err) + return errors.Errorf("failed to install rook operator via helm, err : %v", err) } // create the cluster namespace since it wasn't created by common.yaml if err := h.k8shelper.CreateNamespace(h.settings.Namespace); err != nil { - return fmt.Errorf("failed to create namespace %s. %v", h.settings.Namespace, err) + return errors.Errorf("failed to create namespace %s. %v", h.settings.Namespace, err) } return nil @@ -214,7 +213,7 @@ func (h *CephInstaller) CreateCephCluster() error { var err error h.settings.DataDirHostPath, err = h.initTestDir(h.settings.Namespace) if err != nil { - return fmt.Errorf("failed to create test dir. %+v", err) + return errors.Errorf("failed to create test dir. %+v", err) } logger.Infof("Creating cluster with settings: %+v", h.settings) @@ -232,10 +231,10 @@ osd_pool_default_size = 1 Data: customSettings, } if _, err := h.k8shelper.Clientset.CoreV1().ConfigMaps(h.settings.Namespace).Create(ctx, customCM, metav1.CreateOptions{}); err != nil { - return fmt.Errorf("failed to create custom ceph.conf. %+v", err) + return errors.Errorf("failed to create custom ceph.conf. %+v", err) } - logger.Info("Starting Rook Cluster with yaml") + logger.Info("Starting Rook Cluster") rookCluster := h.Manifests.GetCephCluster() logger.Info(rookCluster) maxTry := 10 @@ -245,7 +244,7 @@ osd_pool_default_size = 1 break } if i == maxTry-1 { - return fmt.Errorf("failed to create rook cluster. %v", err) + return errors.Errorf("failed to create rook cluster. %v", err) } logger.Infof("failed to create rook cluster, trying again... %v", err) time.Sleep(5 * time.Second) @@ -286,7 +285,7 @@ func (h *CephInstaller) CreateRookExternalCluster(externalManifests CephManifest externalSettings := externalManifests.Settings() externalSettings.DataDirHostPath, err = h.initTestDir(externalSettings.Namespace) if err != nil { - return fmt.Errorf("failed to create test dir. %+v", err) + return errors.Errorf("failed to create test dir. %+v", err) } logger.Infof("Creating external cluster %q with core storage namespace %q", externalSettings.Namespace, h.settings.Namespace) @@ -326,25 +325,25 @@ func (h *CephInstaller) injectRookExternalClusterInfo(externalSettings *TestCeph // get config map cm, err := h.GetRookExternalClusterMonConfigMap() if err != nil { - return fmt.Errorf("failed to get configmap. %v", err) + return errors.Errorf("failed to get configmap. %v", err) } // create config map _, err = h.k8shelper.Clientset.CoreV1().ConfigMaps(externalSettings.Namespace).Create(ctx, cm, metav1.CreateOptions{}) if err != nil { - return fmt.Errorf("failed to create configmap. %v", err) + return errors.Errorf("failed to create configmap. %v", err) } // get secret secret, err := h.GetRookExternalClusterMonSecret() if err != nil { - return fmt.Errorf("failed to get secret. %v", err) + return errors.Errorf("failed to get secret. %v", err) } // create secret _, err = h.k8shelper.Clientset.CoreV1().Secrets(externalSettings.Namespace).Create(ctx, secret, metav1.CreateOptions{}) if err != nil { - return fmt.Errorf("failed to create secret. %v", err) + return errors.Errorf("failed to create secret. %v", err) } return nil @@ -356,7 +355,7 @@ func (h *CephInstaller) GetRookExternalClusterMonConfigMap() (*v1.ConfigMap, err configMapName := "rook-ceph-mon-endpoints" externalCM, err := h.k8shelper.Clientset.CoreV1().ConfigMaps(h.settings.Namespace).Get(ctx, configMapName, metav1.GetOptions{}) if err != nil { - return nil, fmt.Errorf("failed to get secret. %v", err) + return nil, errors.Errorf("failed to get secret. %v", err) } newCM := &v1.ConfigMap{} newCM.Name = externalCM.Name @@ -372,7 +371,7 @@ func (h *CephInstaller) GetRookExternalClusterMonSecret() (*v1.Secret, error) { externalSecret, err := h.k8shelper.Clientset.CoreV1().Secrets(h.settings.Namespace).Get(ctx, secretName, metav1.GetOptions{}) if err != nil { - return nil, fmt.Errorf("failed to get secret. %v", err) + return nil, errors.Errorf("failed to get secret. %v", err) } newSecret := &v1.Secret{} newSecret.Name = externalSecret.Name @@ -414,7 +413,7 @@ func (h *CephInstaller) GetNodeHostnames() ([]string, error) { ctx := context.TODO() nodes, err := h.k8shelper.Clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) if err != nil { - return nil, fmt.Errorf("failed to get k8s nodes. %+v", err) + return nil, errors.Errorf("failed to get k8s nodes. %+v", err) } var names []string for _, node := range nodes.Items { @@ -446,14 +445,12 @@ func (h *CephInstaller) InstallRook() (bool, error) { startDiscovery = true err = h.CreateRookOperatorViaHelm("enableDiscoveryDaemon=true,image.tag=master") if err != nil { - logger.Errorf("helm install failed. %v", err) - return false, err + return false, errors.Wrap(err, "failed to configure helm") } } else { err := h.CreateCephOperator() if err != nil { - logger.Errorf("operator install failed. %v", err) - return false, err + return false, errors.Wrap(err, "failed to configure ceph operator") } } if !h.k8shelper.IsPodInExpectedState("rook-ceph-operator", h.settings.OperatorNamespace, "Running") { @@ -482,8 +479,7 @@ func (h *CephInstaller) InstallRook() (bool, error) { // Create rook client err = h.CreateRookToolbox(h.Manifests) if err != nil { - logger.Errorf("Rook toolbox in cluster %s not installed. %v", h.settings.Namespace, err) - return false, err + return false, errors.Wrapf(err, "failed to install toolbox in cluster %s", h.settings.Namespace) } logger.Infof("installed rook operator and cluster %s on k8s %s", h.settings.Namespace, h.k8sVersion) @@ -515,11 +511,18 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(manifests ...CephManifests) logger.Infof("Uninstalling Rook") var err error + skipOperatorCleanup := false for _, manifest := range manifests { namespace := manifest.Settings().Namespace clusterName := manifest.Settings().ClusterName + if manifest.Settings().SkipCleanupPolicy && manifest.Settings().SkipClusterCleanup { + logger.Infof("SKIPPING ALL CLEANUP for namespace %q", namespace) + skipOperatorCleanup = true + continue + } - if !manifest.Settings().IsExternal { + testCleanupPolicy := !manifest.Settings().IsExternal && !manifest.Settings().SkipCleanupPolicy + if testCleanupPolicy { // Add cleanup policy to the core ceph cluster err = h.addCleanupPolicy(namespace) if err != nil { @@ -562,7 +565,7 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(manifests ...CephManifests) err = h.k8shelper.WaitForCustomResourceDeletion(namespace, clusterName, crdCheckerFunc) checkError(h.T(), err, fmt.Sprintf("failed to wait for crd %s deletion", namespace)) - if !manifest.Settings().IsExternal { + if testCleanupPolicy { err = h.waitForCleanupJobs(namespace) if err != nil { assert.NoError(h.T(), err) @@ -581,12 +584,19 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(manifests ...CephManifests) continue } + // Skip the remainder of cluster cleanup if desired + if manifest.Settings().SkipClusterCleanup { + logger.Infof("SKIPPING CLUSTER CLEANUP") + skipOperatorCleanup = true + continue + } + // non-helm cleanup if manifest.Settings().IsExternal { logger.Infof("Deleting all the resources in the common external manifest") _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetCommonExternal(), deleteFromStdinArgs...) if err != nil { - logger.Warningf("failed to remove common external resources. %v", err) + logger.Errorf("failed to remove common external resources. %v", err) } else { logger.Infof("done deleting all the resources in the common external manifest") } @@ -610,13 +620,19 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(manifests ...CephManifests) logger.Infof("Deleting all the resources in the common manifest") _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetCommon(), deleteFromStdinArgs...) if err != nil { - logger.Warningf("failed to remove common manifest. %v", err) + logger.Errorf("failed to remove common manifest. %v", err) } else { logger.Infof("done deleting all the resources in the common manifest") } } } + // Skip the remainder of cluster cleanup if desired + if skipOperatorCleanup { + logger.Infof("SKIPPING OPERATOR CLEANUP") + return + } + if !h.settings.UseHelm { logger.Infof("Deleting all the resources in the operator manifest") _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetOperator(), deleteFromStdinArgs...) @@ -856,13 +872,13 @@ func (h *CephInstaller) addCleanupPolicy(namespace string) error { ctx := context.TODO() cluster, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(ctx, h.settings.ClusterName, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("failed to get ceph cluster. %+v", err) + return errors.Errorf("failed to get ceph cluster. %+v", err) } cluster.Spec.CleanupPolicy.Confirmation = cephv1.DeleteDataDirOnHostsConfirmation cluster.Spec.CleanupPolicy.AllowUninstallWithVolumes = true _, err = h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Update(ctx, cluster, metav1.UpdateOptions{}) if err != nil { - returnErr = fmt.Errorf("failed to add clean up policy to the cluster. %+v", err) + returnErr = errors.Errorf("failed to add clean up policy to the cluster. %+v", err) logger.Warningf("could not add cleanup policy, trying again... %v", err) } else { logger.Info("successfully added cleanup policy to the ceph cluster and skipping checks for existing volumes") @@ -878,7 +894,7 @@ func (h *CephInstaller) waitForCleanupJobs(namespace string) error { appLabelSelector := fmt.Sprintf("app=%s", cluster.CleanupAppName) cleanupJobs, err := h.k8shelper.Clientset.BatchV1().Jobs(namespace).List(ctx, metav1.ListOptions{LabelSelector: appLabelSelector}) if err != nil { - return false, fmt.Errorf("failed to get cleanup jobs. %+v", err) + return false, errors.Errorf("failed to get cleanup jobs. %+v", err) } // Clean up jobs might take some time to start if len(cleanupJobs.Items) == 0 { @@ -888,7 +904,7 @@ func (h *CephInstaller) waitForCleanupJobs(namespace string) error { for _, job := range cleanupJobs.Items { logger.Infof("job %q status: %+v", job.Name, job.Status) if job.Status.Failed > 0 { - return false, fmt.Errorf("job %s failed", job.Name) + return false, errors.Errorf("job %s failed", job.Name) } if job.Status.Succeeded == 1 { l, err := h.k8shelper.Kubectl("-n", namespace, "logs", fmt.Sprintf("job.batch/%s", job.Name)) @@ -909,7 +925,7 @@ func (h *CephInstaller) waitForCleanupJobs(namespace string) error { logger.Info("waiting for job(s) to cleanup the host...") err := wait.Poll(5*time.Second, 90*time.Second, allRookCephCleanupJobs) if err != nil { - return fmt.Errorf("failed to wait for clean up jobs to complete. %+v", err) + return errors.Errorf("failed to wait for clean up jobs to complete. %+v", err) } logger.Info("successfully executed all the ceph clean up jobs") diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 283072307..547ceda93 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -94,8 +94,11 @@ func (m *CephManifestsMaster) GetCommon() string { } func (m *CephManifestsMaster) GetToolbox() string { - manifest := strings.ReplaceAll(m.settings.readManifest("direct-mount.yaml"), "name: rook-direct-mount", "name: rook-ceph-tools") - return strings.ReplaceAll(manifest, "app: rook-direct-mount", "app: rook-ceph-tools") + if m.settings.DirectMountToolbox { + manifest := strings.ReplaceAll(m.settings.readManifest("direct-mount.yaml"), "name: rook-direct-mount", "name: rook-ceph-tools") + return strings.ReplaceAll(manifest, "app: rook-direct-mount", "app: rook-ceph-tools") + } + return m.settings.readManifest("toolbox.yaml") } //********************************************************************************** diff --git a/tests/framework/installer/ceph_manifests_v1.5.go b/tests/framework/installer/ceph_manifests_v1.5.go index 5376b0233..bd742dc6c 100644 --- a/tests/framework/installer/ceph_manifests_v1.5.go +++ b/tests/framework/installer/ceph_manifests_v1.5.go @@ -62,8 +62,11 @@ func (m *CephManifestsV1_5) GetCommon() string { // GetRookToolBox returns rook-toolbox manifest func (m *CephManifestsV1_5) GetToolbox() string { - manifest := strings.ReplaceAll(m.settings.readManifestFromGithub("direct-mount.yaml"), "name: rook-direct-mount", "name: rook-ceph-tools") - return strings.ReplaceAll(manifest, "app: rook-direct-mount", "app: rook-ceph-tools") + if m.settings.DirectMountToolbox { + manifest := strings.ReplaceAll(m.settings.readManifestFromGithub("direct-mount.yaml"), "name: rook-direct-mount", "name: rook-ceph-tools") + return strings.ReplaceAll(manifest, "app: rook-direct-mount", "app: rook-ceph-tools") + } + return m.settings.readManifestFromGithub("toolbox.yaml") } func (m *CephManifestsV1_5) GetCommonExternal() string { diff --git a/tests/framework/installer/ceph_settings.go b/tests/framework/installer/ceph_settings.go index e617f460d..5fdbd9fc6 100644 --- a/tests/framework/installer/ceph_settings.go +++ b/tests/framework/installer/ceph_settings.go @@ -20,6 +20,7 @@ import ( "fmt" "io/ioutil" "net/http" + "os" "path" "strings" @@ -45,10 +46,25 @@ type TestCephSettings struct { EnableDiscovery bool EnableAdmissionController bool IsExternal bool + SkipClusterCleanup bool + SkipCleanupPolicy bool + DirectMountToolbox bool RookVersion string CephVersion cephv1.CephVersionSpec } +func (s *TestCephSettings) ApplyEnvVars() { + // skip the cleanup by default + s.SkipClusterCleanup = true + if os.Getenv("SKIP_TEST_CLEANUP") == "false" { + s.SkipClusterCleanup = false + } + s.SkipCleanupPolicy = true + if os.Getenv("SKIP_CLEANUP_POLICY") == "false" { + s.SkipCleanupPolicy = false + } +} + func (s *TestCephSettings) readManifest(filename string) string { rootDir, err := utils.FindRookRoot() if err != nil { diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index 25cbbc94a..55a98c017 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -936,6 +936,16 @@ func (k8sh *K8sHelper) PrintPVCs(namespace string, detailed bool) { } } +func (k8sh *K8sHelper) PrintResources(namespace, name string) { + args := []string{"-n", namespace, "get", name, "-o", "yaml"} + result, err := k8sh.Kubectl(args...) + if err != nil { + logger.Warningf("failed to get resource %s. %v", name, err) + } else { + logger.Infof("%s\n", result) + } +} + func (k8sh *K8sHelper) PrintStorageClasses(detailed bool) { ctx := context.TODO() scs, err := k8sh.Clientset.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{}) diff --git a/tests/integration/ceph_flex_test.go b/tests/integration/ceph_flex_test.go index 0937e090e..93caa1e3f 100644 --- a/tests/integration/ceph_flex_test.go +++ b/tests/integration/ceph_flex_test.go @@ -97,7 +97,7 @@ func (s *CephFlexDriverSuite) SetupSuite() { RookVersion: installer.VersionMaster, CephVersion: installer.OctopusVersion, } - + s.settings.ApplyEnvVars() s.clusterInfo = client.AdminClusterInfo(namespace) s.installer, s.kh = StartTestCluster(s.T, s.settings, flexDriverMinimalTestVersion) s.testClient = clients.CreateTestClient(s.kh, s.installer.Manifests) diff --git a/tests/integration/ceph_helm_test.go b/tests/integration/ceph_helm_test.go index 028a88d50..3c03227e8 100644 --- a/tests/integration/ceph_helm_test.go +++ b/tests/integration/ceph_helm_test.go @@ -76,7 +76,7 @@ func (h *HelmSuite) SetupSuite() { RookVersion: installer.VersionMaster, CephVersion: installer.OctopusVersion, } - + h.settings.ApplyEnvVars() h.installer, h.k8shelper = StartTestCluster(h.T, h.settings, helmMinimalTestVersion) h.helper = clients.CreateTestClient(h.k8shelper, h.installer.Manifests) } diff --git a/tests/integration/ceph_mgr_test.go b/tests/integration/ceph_mgr_test.go index cc5abd077..3f03ebb02 100644 --- a/tests/integration/ceph_mgr_test.go +++ b/tests/integration/ceph_mgr_test.go @@ -104,7 +104,7 @@ func (s *CephMgrSuite) SetupSuite() { RookVersion: installer.VersionMaster, CephVersion: installer.MasterVersion, } - + s.settings.ApplyEnvVars() s.installer, s.k8sh = StartTestCluster(s.T, s.settings, cephMasterSuiteMinimalTestVersion) s.waitForOrchestrationModule() } diff --git a/tests/integration/ceph_multi_cluster_test.go b/tests/integration/ceph_multi_cluster_test.go index bd552453a..b426fa0a8 100644 --- a/tests/integration/ceph_multi_cluster_test.go +++ b/tests/integration/ceph_multi_cluster_test.go @@ -90,6 +90,7 @@ func (s *MultiClusterDeploySuite) SetupSuite() { RookVersion: installer.VersionMaster, CephVersion: installer.NautilusVersion, } + s.settings.ApplyEnvVars() externalSettings := &installer.TestCephSettings{ IsExternal: true, ClusterName: "test-external", @@ -98,6 +99,7 @@ func (s *MultiClusterDeploySuite) SetupSuite() { RookVersion: s.settings.RookVersion, UseCSI: true, } + externalSettings.ApplyEnvVars() s.externalManifests = installer.NewCephManifests(externalSettings) k8sh, err := utils.CreateK8sHelper(s.T) diff --git a/tests/integration/ceph_smoke_test.go b/tests/integration/ceph_smoke_test.go index 98c7a985c..251d42330 100644 --- a/tests/integration/ceph_smoke_test.go +++ b/tests/integration/ceph_smoke_test.go @@ -102,6 +102,7 @@ func (s *SmokeSuite) SetupSuite() { RookVersion: installer.VersionMaster, CephVersion: installer.OctopusVersion, } + s.settings.ApplyEnvVars() s.installer, s.k8sh = StartTestCluster(s.T, s.settings, smokeSuiteMinimalTestVersion) s.helper = clients.CreateTestClient(s.k8sh, s.installer.Manifests) From deb7c21f98a99303baf507c11ba5a9d2b7d68dc3 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Wed, 17 Mar 2021 12:00:59 -0600 Subject: [PATCH 6/6] ceph: install pre-1.16 crds with helm chart The helm chart was not installing the CRDs on pre-1.16 k8s due to a regression in the conditionals for the chart that was introduced with 7103. Signed-off-by: Travis Nielsen --- build/crds/build-crds.sh | 8 ++++---- cluster/charts/rook-ceph/templates/resources.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build/crds/build-crds.sh b/build/crds/build-crds.sh index 774fc540d..82a253fea 100755 --- a/build/crds/build-crds.sh +++ b/build/crds/build-crds.sh @@ -68,16 +68,16 @@ EOF build_helm_resources() { # Add helm annotations to all CRDS yq w -i -d'*' "$HELM_CRDS_FILE_PATH" "metadata.annotations[helm.sh/resource-policy]" keep - + # add header - sed -i '1s/^/{{- if .Values.crds.enabled }}\n/' "$HELM_CRDS_FILE_PATH" sed -i '1s/^/{{- if semverCompare ">=1.16.0" .Capabilities.KubeVersion.GitVersion }}\n/' "$HELM_CRDS_FILE_PATH" - + sed -i '1s/^/{{- if .Values.crds.enabled }}\n/' "$HELM_CRDS_FILE_PATH" + echo "Generating helm resources.yaml" { # add else echo "{{- else }}" - + # add footer cat "$CRDS_BEFORE_1_16_FILE_PATH" # DO NOT REMOVE the empty line, it is necessary diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index 02a78a949..3dd7891f9 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -1,5 +1,5 @@ -{{- if semverCompare ">=1.16.0" .Capabilities.KubeVersion.GitVersion }} {{- if .Values.crds.enabled }} +{{- if semverCompare ">=1.16.0" .Capabilities.KubeVersion.GitVersion }} apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: