Merge pull request #5452 from leseb/osd-wipe

ceph: enhancement disk sanitize options

Documentation/ceph-cluster-crd.md

@@ -149,30 +149,7 @@ For more details on the mons and when to choose a number other than `3`, see the
   * `manageMachineDisruptionBudgets`: if `true`, the operator will create and manage MachineDisruptionBudgets to ensure OSDs are only fenced when the cluster is healthy. Only available on OpenShift.
   * `machineDisruptionBudgetNamespace`: the namespace in which to watch the MachineDisruptionBudgets.
 * `removeOSDsIfOutAndSafeToRemove`: If `true` the operator will remove the OSDs that are down and whose data has been restored to other OSDs. In Ceph terms, the osds are `out` and `safe-to-destroy` when then would be removed.
-* `cleanupPolicy`: The section for confirming that cluster data should be forcibly deleted. The cleanupPolicy should only be added to the cluster when the cluster is about to be deleted. After any field of the cleanup policy is set, Rook will stop configuring the cluster as if the cluster is about to be destroyed in order to prevent these settings from being deployed unintentionally.
-  * `confirmation`: If `yes-really-destroy-data` the operator will automatically delete data on the hostpath of cluster nodes and clean devices with OSDs when a `delete cephcluster` command is issued. Only `yes-really-destroy-data` and an empty string are valid values for this field.
-* `healthCheck`: control period health status checks and livenessprobes, see the [health settings](#health-settings)
-
-To activate the cleanup, you can use the following command **AT YOUR OWN RISK**:
-
-```console
-kubectl -n rook-ceph patch cephcluster rook-ceph --type merge -p '{"spec":{"cleanupPolicy":{"confirmation":"yes-really-destroy-data"}}}'
-```
-
-When applied you will see the following from the Operator's logs:
-
-```text
-2020-05-27 13:24:04.267665 I | ceph-spec: CR has changed for "rook-ceph". diff=  v1.ClusterSpec{
-        ... // 16 identical fields
-        Mgr:                            v1.MgrSpec{},
-        RemoveOSDsIfOutAndSafeToRemove: false,
--       CleanupPolicy:                  v1.CleanupPolicySpec{},
-+       CleanupPolicy:                  v1.CleanupPolicySpec{Confirmation: "yes-really-destroy-data"},
-  }
-```
-
-Nothing will happen until the deletion of the CR is requested, so this can still be reverted.
-However, all new orchestration/reconciliation will be blocked with this cleanup policy enabled.
+* `cleanupPolicy`: [cleanup policy settings](#cleanup-policy)
 
 ### Ceph container images
 
@@ -1029,3 +1006,29 @@ spec:
   cephVersion:
     image: ceph/ceph:v15.2.4 # Should match external cluster version
 ```
+
+### Cleanup policy
+
+Rook has the ability to cleanup resources and data that were deployed.
+The policy represents the confirmation that cluster data should be forcibly deleted.
+The cleanupPolicy should only be added to the cluster when the cluster is about to be deleted.
+After the `confirmation` field of the cleanup policy is set, Rook will stop configuring the cluster as if the cluster is about to be destroyed in order to prevent these settings from being deployed unintentionally.
+The `cleanupPolicy` CR settings has different fields:
+
+* `confirmation`: If `yes-really-destroy-data` the operator will automatically delete data on the hostpath of cluster nodes and clean devices with OSDs when a `delete cephcluster` command is issued. Only `yes-really-destroy-data` and an empty string are valid values for this field.
+* `sanitizeDisks`: sanitizeDisks represents advanced settings that can be used to sanitize drives.
+By default if the `confirmation` is set, Rook will only remove Ceph's metadata, which allows a cluster re-installation if desired.
+However, the administrator might want to sanitize the drives in more depth with the following flags:
+  * `method`: indicates if the entire disk should be sanitized or simply ceph's metadata. Possible choices are 'quick' (default) or 'complete'
+  * `dataSource`: indicate where to get random bytes from to write on the disk. Possible choices are 'zero' (default) or 'random'.
+  Using random sources will consume entropy from the system and will take much more time then the zero source
+  * `iteration`: overwrite N times instead of the default (1). Takes an integer value
+
+To automate activation of the cleanup, you can use the following command. **WARNING: DATA WILL BE PERMANENTLY DELETED**:
+
+```console
+kubectl -n rook-ceph patch cephcluster rook-ceph --type merge -p '{"spec":{"cleanupPolicy":{"confirmation":"yes-really-destroy-data"}}}'
+```
+
+Nothing will happen until the deletion of the CR is requested, so this can still be reverted.
+However, all new configuration by the operator will be blocked with this cleanup policy enabled.

cluster/charts/rook-ceph/templates/resources.yaml

@@ -169,6 +169,17 @@ spec:
                 confirmation:
                   type: string
                   pattern: ^$|^yes-really-destroy-data$
+                sanitizeDisks:
+                  properties:
+                    method:
+                      type: string
+                      pattern: ^(complete|quick)$
+                    dataSource:
+                      type: string
+                      pattern: ^(zero|random)$
+                    iteration:
+                      type: integer
+                      format: int32
   additionalPrinterColumns:
     - name: DataDirHostPath
       type: string

cluster/examples/kubernetes/ceph/cluster.yaml

@@ -92,7 +92,19 @@ spec:
     # To signify that automatic deletion is desired, use the value "yes-really-destroy-data". Only this and an empty
     # string are valid values for this field.
     confirmation: ""
-
+    # sanitizeDisks represents settings for sanitizing OSD disks on cluster deletion
+    sanitizeDisks:
+      # method indicates if the entire disk should be sanitized or simply ceph's metadata
+      # in both case, re-install is possible
+      # possible choices are 'complete' or 'quick' (default)
+      method: quick
+      # dataSource indicate where to get random bytes from to write on the disk
+      # possible choices are 'zero' (default) or 'random'
+      # using random sources will consume entropy from the system and will take much more time then the zero source
+      dataSource: zero
+      # iteration overwrite N times instead of the default (1)
+      # takes an integer value
+      iteration: 1
   # To control where various services will be scheduled by kubernetes, use the placement configuration sections below.
   # The example under 'all' would have all services scheduled on kubernetes nodes labeled with 'role=storage-node' and
   # tolerate taints with a key of 'storage-node'.

cluster/examples/kubernetes/ceph/common.yaml

@@ -192,6 +192,17 @@ spec:
                 confirmation:
                   type: string
                   pattern: ^$|^yes-really-destroy-data$
+                sanitizeDisks:
+                  properties:
+                    method:
+                      type: string
+                      pattern: ^(complete|quick)$
+                    dataSource:
+                      type: string
+                      pattern: ^(zero|random)$
+                    iteration:
+                      type: integer
+                      format: int32
             placement: {}
             resources: {}
             healthCheck: {}

cmd/rook/ceph/cleanup.go

@@ -20,6 +20,7 @@ import (
 	"os"
 
 	"github.com/rook/rook/cmd/rook/rook"
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	cleanup "github.com/rook/rook/pkg/daemon/ceph/cleanup"
 	"github.com/rook/rook/pkg/daemon/ceph/client"
 	"github.com/rook/rook/pkg/operator/k8sutil"
@@ -28,10 +29,13 @@ import (
 )
 
 var (
-	dataDirHostPath string
-	namespaceDir    string
-	monSecret       string
-	clusterFSID     string
+	dataDirHostPath    string
+	namespaceDir       string
+	monSecret          string
+	clusterFSID        string
+	sanitizeMethod     string
+	sanitizeDataSource string
+	sanitizeIteration  int32
 )
 
 var cleanUpCmd = &cobra.Command{
@@ -44,6 +48,9 @@ func init() {
 	cleanUpCmd.Flags().StringVar(&namespaceDir, "namespace-dir", "", "dataDirHostPath on the node")
 	cleanUpCmd.Flags().StringVar(&monSecret, "mon-secret", "", "monitor secret from the keyring")
 	cleanUpCmd.Flags().StringVar(&clusterFSID, "cluster-fsid", "", "ceph cluster fsid")
+	cleanUpCmd.Flags().StringVar(&sanitizeMethod, "sanitize-method", string(cephv1.SanitizeMethodQuick), "sanitize method to use (metadata or data)")
+	cleanUpCmd.Flags().StringVar(&sanitizeDataSource, "sanitize-data-source", string(cephv1.SanitizeDataSourceZero), "data source to sanitize the disk (zero or random)")
+	cleanUpCmd.Flags().Int32Var(&sanitizeIteration, "sanitize-iteration", 1, "overwrite N times the disk")
 	flags.SetFlagsFromEnv(cleanUpCmd.Flags(), rook.RookEnvVarPrefix)
 	cleanUpCmd.RunE = startCleanUp
 }
@@ -64,7 +71,14 @@ func startCleanUp(cmd *cobra.Command, args []string) error {
 	clusterInfo.FSID = clusterFSID
 
 	// Build Sanitizer
-	s := cleanup.NewDiskSanitizer(createContext(), clusterInfo)
+	s := cleanup.NewDiskSanitizer(createContext(),
+		clusterInfo,
+		&cephv1.SanitizeDisksSpec{
+			Method:     cephv1.SanitizeMethodProperty(sanitizeMethod),
+			DataSource: cephv1.SanitizeDataSourceProperty(sanitizeDataSource),
+			Iteration:  sanitizeIteration,
+		},
+	)
 
 	// Start OSD wipe process
 	s.StartSanitizeDisks()

pkg/apis/ceph.rook.io/v1/cleanup.go

@@ -17,6 +17,18 @@ limitations under the License.
 package v1
 
 const (
+	// SanitizeDataSourceZero uses /dev/zero as sanitize source
+	SanitizeDataSourceZero SanitizeDataSourceProperty = "zero"
+
+	// SanitizeDataSourceRandom uses `shred's default entropy source
+	SanitizeDataSourceRandom SanitizeDataSourceProperty = "random"
+
+	// SanitizeMethodComplete will sanitize everything on the disk
+	SanitizeMethodComplete SanitizeMethodProperty = "complete"
+
+	// SanitizeMethodQuick will sanitize metdata only on the disk
+	SanitizeMethodQuick SanitizeMethodProperty = "quick"
+
 	// DeleteDataDirOnHostsConfirmation represents the validation to destry dataDirHostPath
 	DeleteDataDirOnHostsConfirmation CleanupConfirmationProperty = "yes-really-destroy-data"
 )
@@ -26,6 +38,10 @@ func (c *CleanupPolicySpec) HasDataDirCleanPolicy() bool {
 	return c.Confirmation == DeleteDataDirOnHostsConfirmation
 }
 
-func (c *CleanupConfirmationProperty) String() string {
+func (c *SanitizeMethodProperty) String() string {
+	return string(*c)
+}
+
+func (c *SanitizeDataSourceProperty) String() string {
 	return string(*c)
 }

pkg/apis/ceph.rook.io/v1/types.go

@@ -716,11 +716,22 @@ type ClientSpec struct {
 }
 
 type CleanupPolicySpec struct {
-	Confirmation CleanupConfirmationProperty `json:"confirmation,omitempty"`
+	Confirmation  CleanupConfirmationProperty `json:"confirmation,omitempty"`
+	SanitizeDisks SanitizeDisksSpec           `json:"sanitizeDisks"`
 }
 
 type CleanupConfirmationProperty string
 
+type SanitizeDataSourceProperty string
+
+type SanitizeMethodProperty string
+
+type SanitizeDisksSpec struct {
+	Method     SanitizeMethodProperty     `json:"method,omitempty"`
+	DataSource SanitizeDataSourceProperty `json:"dataSource,omitempty"`
+	Iteration  int32                      `json:"iteration,omitempty"`
+}
+
 // +genclient
 // +genclient:noStatus
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/daemon/ceph/cleanup/disk.go

@@ -23,6 +23,7 @@ import (
 	"sync"
 
 	"github.com/coreos/pkg/capnslog"
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/daemon/ceph/client"
 	"github.com/rook/rook/pkg/daemon/ceph/osd"
@@ -30,11 +31,8 @@ import (
 )
 
 const (
-	ddBS      = "1M"           // DD's block size
-	ddCount   = "10"           // DD runs over the first 10 offsets
-	ddFlags   = "direct,dsync" // DD's sync flags"
-	ddIf      = "/dev/zero"
-	ddUtility = "dd"
+	shredUtility = "shred"
+	shredBS      = "10M" // Shred's block size
 )
 
 var (
@@ -43,15 +41,17 @@ var (
 
 // DiskSanitizer is simple struct to old the context to execute the commands
 type DiskSanitizer struct {
-	context     *clusterd.Context
-	clusterInfo *client.ClusterInfo
+	context           *clusterd.Context
+	clusterInfo       *client.ClusterInfo
+	sanitizeDisksSpec *cephv1.SanitizeDisksSpec
 }
 
 // NewDiskSanitizer is function that returns a full filled DiskSanitizer object
-func NewDiskSanitizer(context *clusterd.Context, clusterInfo *client.ClusterInfo) *DiskSanitizer {
+func NewDiskSanitizer(context *clusterd.Context, clusterInfo *client.ClusterInfo, sanitizeDisksSpec *cephv1.SanitizeDisksSpec) *DiskSanitizer {
 	return &DiskSanitizer{
-		context:     context,
-		clusterInfo: clusterInfo,
+		context:           context,
+		clusterInfo:       clusterInfo,
+		sanitizeDisksSpec: sanitizeDisksSpec,
 	}
 }
 
@@ -144,23 +144,42 @@ func (s *DiskSanitizer) returnPVDevice(disk string) []string {
 	return strings.Split(output, ":")
 }
 
-func (s *DiskSanitizer) buildDDArgs(disk string) []string {
-	ddArgs := []string{
-		fmt.Sprintf("if=%s", ddIf),
-		fmt.Sprintf("of=%s", disk),
-		fmt.Sprintf("bs=%s", ddBS),
-		fmt.Sprintf("count=%s", ddCount),
-		fmt.Sprintf("oflag=%s", ddFlags),
+func (s *DiskSanitizer) buildDataSource() string {
+	return fmt.Sprintf("/dev/%s", s.sanitizeDisksSpec.DataSource.String())
+}
+
+func (s *DiskSanitizer) buildShredArgs(disk string) []string {
+	var shredArgs []string
+
+	// If data source is not zero, then let's add zeros at the end of the pass
+	if s.sanitizeDisksSpec.DataSource != cephv1.SanitizeDataSourceZero {
+		shredArgs = append(shredArgs, "--zero")
 	}
 
-	return ddArgs
+	// If this is a quick pass let's just overwrite the first 10MB
+	if s.sanitizeDisksSpec.Method == cephv1.SanitizeMethodQuick {
+		shredArgs = append(shredArgs, fmt.Sprintf("--size=%s", shredBS))
+	}
+
+	// If the data source for randomness is zero
+	if s.sanitizeDisksSpec.DataSource == cephv1.SanitizeDataSourceZero {
+		shredArgs = append(shredArgs, fmt.Sprintf("--random-source=%s", s.buildDataSource()))
+	}
+
+	shredArgs = append(shredArgs, []string{
+		"--force",
+		"--verbose",
+		fmt.Sprintf("--iterations=%s", strconv.Itoa(int(s.sanitizeDisksSpec.Iteration))),
+		disk}...)
+
+	return shredArgs
 }
 
 func (s *DiskSanitizer) executeSanitizeCommand(disk string, wg *sync.WaitGroup) {
 	// On return, notify the WaitGroup that we’re done
 	defer wg.Done()
 
-	output, err := s.context.Executor.ExecuteCommandWithCombinedOutput(ddUtility, s.buildDDArgs(disk)...)
+	output, err := s.context.Executor.ExecuteCommandWithCombinedOutput(shredUtility, s.buildShredArgs(disk)...)
 	if err != nil {
 		logger.Errorf("failed to sanitize osd disk %q. %s. %v", disk, output, err)
 	}

pkg/daemon/ceph/cleanup/disk_test.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2020 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cleanup
+
+import (
+	"reflect"
+	"testing"
+
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	"github.com/rook/rook/pkg/clusterd"
+	"github.com/rook/rook/pkg/daemon/ceph/client"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBuildDataSource(t *testing.T) {
+	s := NewDiskSanitizer(&clusterd.Context{}, &client.ClusterInfo{}, &cephv1.SanitizeDisksSpec{})
+	s.sanitizeDisksSpec.DataSource = cephv1.SanitizeDataSourceZero
+
+	assert.Equal(t, "/dev/zero", s.buildDataSource())
+}
+
+func TestBuildShredArgs(t *testing.T) {
+	var i int32 = 1
+	c := &clusterd.Context{}
+	disk := "/dev/sda"
+	type fields struct {
+		context           *clusterd.Context
+		clusterInfo       *client.ClusterInfo
+		sanitizeDisksSpec *cephv1.SanitizeDisksSpec
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		disk   string
+		want   []string
+	}{
+		{"quick-zero", fields{c, &client.ClusterInfo{}, &cephv1.SanitizeDisksSpec{Method: cephv1.SanitizeMethodQuick, Iteration: i, DataSource: cephv1.SanitizeDataSourceZero}}, disk, []string{"--size=10M", "--random-source=/dev/zero", "--force", "--verbose", "--iterations=1", "/dev/sda"}},
+		{"quick-random", fields{c, &client.ClusterInfo{}, &cephv1.SanitizeDisksSpec{Method: cephv1.SanitizeMethodQuick, Iteration: i, DataSource: cephv1.SanitizeDataSourceRandom}}, disk, []string{"--zero", "--size=10M", "--force", "--verbose", "--iterations=1", "/dev/sda"}},
+		{"complete-zero", fields{c, &client.ClusterInfo{}, &cephv1.SanitizeDisksSpec{Method: cephv1.SanitizeMethodComplete, Iteration: i, DataSource: cephv1.SanitizeDataSourceZero}}, disk, []string{"--random-source=/dev/zero", "--force", "--verbose", "--iterations=1", "/dev/sda"}},
+		{"complete-random", fields{c, &client.ClusterInfo{}, &cephv1.SanitizeDisksSpec{Method: cephv1.SanitizeMethodComplete, Iteration: i, DataSource: cephv1.SanitizeDataSourceRandom}}, disk, []string{"--zero", "--force", "--verbose", "--iterations=1", "/dev/sda"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := &DiskSanitizer{
+				context:           tt.fields.context,
+				clusterInfo:       tt.fields.clusterInfo,
+				sanitizeDisksSpec: tt.fields.sanitizeDisksSpec,
+			}
+			if got := s.buildShredArgs(tt.disk); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("DiskSanitizer.buildShredArgs() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/daemon/ceph/osd/daemon.go

@@ -55,7 +55,7 @@ func StartOSD(context *clusterd.Context, osdType, osdID, osdUUID, lvPath string,
 	}
 
 	// Update LVM config at runtime
-	if err := updateLVMConfig(context, pvcBackedOSD, lvBackedPV); err != nil {
+	if err := UpdateLVMConfig(context, pvcBackedOSD, lvBackedPV); err != nil {
 		return errors.Wrap(err, "failed to update lvm configuration file") // fail return here as validation provided by ceph-volume
 	}
 

pkg/daemon/ceph/osd/volume.go

@@ -179,7 +179,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device
 	//
 	// Or keep doing this if the PV is backend by an LV already
 	if !a.clusterInfo.CephVersion.IsAtLeast(cephVolumeRawModeMinCephVersion) || lvBackedPV {
-		if err := updateLVMConfig(context, a.pvcBacked, lvBackedPV); err != nil {
+		if err := UpdateLVMConfig(context, a.pvcBacked, lvBackedPV); err != nil {
 			return nil, errors.Wrap(err, "failed to update lvm configuration file")
 		}
 	}
@@ -343,7 +343,8 @@ func getLVPath(op string) string {
 	return ""
 }
 
-func updateLVMConfig(context *clusterd.Context, onPVC, lvBackedPV bool) error {
+// UpdateLVMConfig updates the lvm.conf file
+func UpdateLVMConfig(context *clusterd.Context, onPVC, lvBackedPV bool) error {
 
 	input, err := ioutil.ReadFile(lvmConfPath)
 	if err != nil {

pkg/operator/ceph/cluster/cleanup.go

@@ -18,6 +18,7 @@ package cluster
 
 import (
 	"fmt"
+	"strconv"
 	"time"
 
 	"github.com/pkg/errors"
@@ -44,11 +45,16 @@ const (
 )
 
 var (
-	volumeName      = "cleanup-volume"
-	dataDirHostPath = "ROOK_DATA_DIR_HOST_PATH"
-	namespaceDir    = "ROOK_NAMESPACE_DIR"
-	monitorSecret   = "ROOK_MON_SECRET"
-	clusterFSID     = "ROOK_CLUSTER_FSID"
+	volumeName                     = "cleanup-volume"
+	dataDirHostPath                = "ROOK_DATA_DIR_HOST_PATH"
+	namespaceDir                   = "ROOK_NAMESPACE_DIR"
+	monitorSecret                  = "ROOK_MON_SECRET"
+	clusterFSID                    = "ROOK_CLUSTER_FSID"
+	sanitizeMethod                 = "ROOK_SANITIZE_METHOD"
+	sanitizeDataSource             = "ROOK_SANITIZE_DATA_SOURCE"
+	sanitizeLayer                  = "ROOK_SANITIZE_LAYER"
+	sanitizeIteration              = "ROOK_SANITIZE_ITERATION"
+	sanitizeIterationDefault int32 = 1
 )
 
 func (c *ClusterController) startClusterCleanUp(stopCleanupCh chan struct{}, cluster *cephv1.CephCluster, cephHosts []string, monSecret, clusterFSID string) {
@@ -94,6 +100,10 @@ func (c *ClusterController) cleanUpJobContainer(cluster *cephv1.CephCluster, mon
 	volumeMounts := []v1.VolumeMount{}
 	envVars := []v1.EnvVar{}
 	if cluster.Spec.DataDirHostPath != "" {
+		if cluster.Spec.CleanupPolicy.SanitizeDisks.Iteration == 0 {
+			cluster.Spec.CleanupPolicy.SanitizeDisks.Iteration = sanitizeIterationDefault
+		}
+
 		hostPathVolumeMount := v1.VolumeMount{Name: volumeName, MountPath: cluster.Spec.DataDirHostPath}
 		devMount := v1.VolumeMount{Name: "devices", MountPath: "/dev"}
 		volumeMounts = append(volumeMounts, hostPathVolumeMount)
@@ -105,6 +115,9 @@ func (c *ClusterController) cleanUpJobContainer(cluster *cephv1.CephCluster, mon
 			{Name: clusterFSID, Value: cephFSID},
 			{Name: "ROOK_LOG_LEVEL", Value: "DEBUG"},
 			mon.PodNamespaceEnvVar(cluster.Namespace),
+			{Name: sanitizeMethod, Value: cluster.Spec.CleanupPolicy.SanitizeDisks.Method.String()},
+			{Name: sanitizeDataSource, Value: cluster.Spec.CleanupPolicy.SanitizeDisks.DataSource.String()},
+			{Name: sanitizeIteration, Value: strconv.Itoa(int(cluster.Spec.CleanupPolicy.SanitizeDisks.Iteration))},
 		}...)
 	}
 

pkg/operator/ceph/object/controller.go

@@ -75,13 +75,14 @@ var controllerTypeMeta = metav1.TypeMeta{
 
 // ReconcileCephObjectStore reconciles a cephObjectStore object
 type ReconcileCephObjectStore struct {
-	client              client.Client
-	bktclient           bktclient.Interface
-	scheme              *runtime.Scheme
-	context             *clusterd.Context
-	cephClusterSpec     *cephv1.ClusterSpec
-	clusterInfo         *cephclient.ClusterInfo
-	objectStoreChannels map[string]*objectStoreHealth
+	client                 client.Client
+	bktclient              bktclient.Interface
+	scheme                 *runtime.Scheme
+	context                *clusterd.Context
+	cephClusterSpec        *cephv1.ClusterSpec
+	clusterInfo            *cephclient.ClusterInfo
+	objectStoreChannels    map[string]*objectStoreHealth
+	clusterResourceDeleted bool
 }
 
 type objectStoreHealth struct {
@@ -221,35 +222,40 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
 	if !cephObjectStore.GetDeletionTimestamp().IsZero() {
 		logger.Debugf("deleting store %q", cephObjectStore.Name)
 
-		response, okToDelete := r.verifyObjectBucketCleanup(cephObjectStore)
-		if !okToDelete {
-			// If the object store cannot be deleted, requeue the request for deletion to see if the conditions
-			// will eventually be satisfied such as the object buckets being removed
-			return response, nil
-		}
+		if !r.clusterResourceDeleted {
+			response, okToDelete := r.verifyObjectBucketCleanup(cephObjectStore)
+			if !okToDelete {
+				// If the object store cannot be deleted, requeue the request for deletion to see if the conditions
+				// will eventually be satisfied such as the object buckets being removed
+				return response, nil
+			}
 
-		response, okToDelete = r.verifyObjectUserCleanup(cephObjectStore)
-		if !okToDelete {
-			// If the object store cannot be deleted, requeue the request for deletion to see if the conditions
-			// will eventually be satisfied such as the object users being removed
-			return response, nil
-		}
+			response, okToDelete = r.verifyObjectUserCleanup(cephObjectStore)
+			if !okToDelete {
+				// If the object store cannot be deleted, requeue the request for deletion to see if the conditions
+				// will eventually be satisfied such as the object users being removed
+				return response, nil
+			}
 
-		// Close the channel to stop the healthcheck of the endpoint
-		close(r.objectStoreChannels[cephObjectStore.Name].stopChan)
+			// Close the channel to stop the healthcheck of the endpoint
+			close(r.objectStoreChannels[cephObjectStore.Name].stopChan)
 
-		// Remove object store from the map
-		delete(r.objectStoreChannels, cephObjectStore.Name)
+			// Remove object store from the map
+			delete(r.objectStoreChannels, cephObjectStore.Name)
 
-		cfg := clusterConfig{
-			context:     r.context,
-			store:       cephObjectStore,
-			clusterSpec: r.cephClusterSpec,
-			clusterInfo: r.clusterInfo,
-		}
-		err = cfg.deleteStore()
-		if err != nil {
-			return reconcile.Result{}, errors.Wrapf(err, "failed to delete store %q", cephObjectStore.Name)
+			cfg := clusterConfig{
+				context:     r.context,
+				store:       cephObjectStore,
+				clusterSpec: r.cephClusterSpec,
+				clusterInfo: r.clusterInfo,
+			}
+			err = cfg.deleteStore()
+			if err != nil {
+				return reconcile.Result{}, errors.Wrapf(err, "failed to delete store %q", cephObjectStore.Name)
+			}
+
+			// Mark the ressource deletion as complete
+			r.clusterResourceDeleted = true
 		}
 
 		// Remove finalizer

tests/framework/installer/ceph_installer.go

@@ -97,10 +97,6 @@ func (h *CephInstaller) CreateCephOperator(namespace string) (err error) {
 	if err != nil {
 		return fmt.Errorf("Failed to start admission controllers: %v", err)
 	}
-
-	if err = h.WipeDisks(namespace, 2); err != nil {
-		return err
-	}
 	rookOperator := h.Manifests.GetRookOperator(namespace)
 	_, err = h.k8shelper.KubectlWithStdin(rookOperator, createFromStdinArgs...)
 	if err != nil {
@@ -190,9 +186,6 @@ func (h *CephInstaller) CreateRookOperatorViaHelm(namespace, chartSettings strin
 
 	}
 
-	if err = h.WipeDisks(namespace, 2); err != nil {
-		return err
-	}
 	return nil
 }
 
@@ -527,8 +520,11 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, name
 	for clusterNum, namespace := range namespaces {
 		if h.cleanupHost {
 			//Add cleanup policy to the ceph cluster
-			err = h.addCleanupPolicy(namespace)
-			assert.NoError(h.T(), err)
+			// Add cleanup policy to the ceph cluster but NOT the external one
+			if clusterNum == 0 {
+				err = h.addCleanupPolicy(namespace)
+				assert.NoError(h.T(), err)
+			}
 		}
 
 		if !h.T().Failed() {
@@ -550,17 +546,20 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, name
 			h.GatherAllRookLogs(h.T().Name()+"poolcheck", systemNamespace)
 		}
 
-		roles := h.Manifests.GetClusterRoles(namespace, systemNamespace)
-		_, err = h.k8shelper.KubectlWithStdin(roles, deleteFromStdinArgs...)
-
 		err = h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cephcluster", h.clusterName)
 		checkError(h.T(), err, fmt.Sprintf("cannot remove cluster %s", namespace))
 
 		if h.cleanupHost {
-			err = h.waitForCleanupJobs(namespace)
-			assert.NoError(h.T(), err)
+			// The second cluster is external so the cleanup pod will never exist!
+			if clusterNum == 0 {
+				err = h.waitForCleanupJobs(namespace)
+				assert.NoError(h.T(), err)
+			}
 		}
 
+		roles := h.Manifests.GetClusterRoles(namespace, systemNamespace)
+		_, err = h.k8shelper.KubectlWithStdin(roles, deleteFromStdinArgs...)
+
 		crdCheckerFunc := func() error {
 			_, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(h.clusterName, metav1.GetOptions{})
 			// ensure the finalizer(s) are removed
@@ -805,158 +804,6 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, useH
 	return h
 }
 
-// Wipe cluster disks
-func (h *CephInstaller) WipeDisks(namespace string, retries int) (err error) {
-	for i := 1; i <= retries; i++ {
-		if err = h.wipeClusterDisks(namespace); err == nil {
-			logger.Info("disks wiped successfully")
-			return nil
-		} else {
-			logger.Warningf("failed to wipe cluster disks. %+v. Attemp %d/%d ...", err, i, retries)
-		}
-	}
-	return fmt.Errorf("failed to wipe cluster disks. %+v", err)
-}
-
-// wipeClusterDisks runs a disk wipe job on all nodes in the k8s cluster.
-func (h *CephInstaller) wipeClusterDisks(namespace string) error {
-	// Wipe clean disks on all nodes
-	nodes, err := h.GetNodeHostnames()
-	if err != nil {
-		return fmt.Errorf("failed to get node hostnames. %+v", err)
-	}
-	var jobNames []string
-	for _, node := range nodes {
-		// Start the new job
-		globalTestJobIndex++
-		jobName := fmt.Sprintf("rook-ceph-disk-wipe-%d", globalTestJobIndex)
-		jobNames = append(jobNames, jobName)
-		job := h.GetDiskWipeJob(node, jobName, namespace)
-		_, err := h.k8shelper.KubectlWithStdin(job, createFromStdinArgs...)
-		if err != nil {
-			return fmt.Errorf("failed to create disk wipe job for host %s. %+v", node, err)
-		}
-	}
-
-	allJobsAreComplete := func() (done bool, err error) {
-		for _, jobName := range jobNames {
-			j, err := h.k8shelper.Clientset.BatchV1().Jobs(namespace).Get(jobName, metav1.GetOptions{})
-			if err != nil {
-				return false, nil
-			}
-			if j.Status.Failed > 0 {
-				return false, fmt.Errorf("job %s failed", jobName)
-			}
-			if j.Status.Succeeded == 0 {
-				return false, nil
-			}
-		}
-		return true, nil
-	}
-
-	// return the error below after cleaning up the jobs
-	err = wait.Poll(5*time.Second, 90*time.Second, allJobsAreComplete)
-	if err != nil {
-		return fmt.Errorf("failed to wait for wipe jobs to complete. %+v", err)
-	}
-	return nil
-}
-
-// GetDiskWipeJob returns a YAML manifest string for a job which will wipe clean the extra
-// (non-boot) disks on a node, allowing Ceph to use the disk(s) during its testing.
-func (h *CephInstaller) GetDiskWipeJob(nodeName, jobName, namespace string) string {
-	// put the wipe job in the cluster namespace so that logs get picked up in failure conditions
-	return `apiVersion: batch/v1
-kind: Job
-metadata:
-  name: ` + jobName + `
-  namespace: ` + namespace + `
-spec:
-    template:
-      spec:
-          restartPolicy: OnFailure
-          containers:
-              - name: disk-wipe
-                image: ` + h.CephVersion.Image + `
-                securityContext:
-                    privileged: true
-                volumeMounts:
-                    - name: dev
-                      mountPath: /dev
-                    - name: run-udev
-                      mountPath: /run/udev
-                command:
-                    - "sh"
-                    - "-c"
-                    - |
-                      set -xEeuo pipefail
-                      # Wipe LVM data from disks
-                      #
-                      udevadm trigger || true
-                      vgimport -a || true
-                      # We should prepopulate the PVs corresponding to Ceph VGs here.
-                      # It's because getting the relationship of the PVs and the VGs
-                      # is impossible after removing the VGs.
-                      TMP=$(pvs --noheadings --separator=',' -o pv_name,vg_name)
-                      PVS=
-                      for line in $TMP ; do
-                        if [[ $line =~ ,ceph- ]]; then
-                          PVS="$PVS ${line%,*}"
-                        fi
-                      done
-                      # Wipe VGs
-                      for vg in $(vgs --noheadings --readonly --separator=' ' -o vg_name); do
-                        if [[ $vg =~ ^ceph- ]]; then
-                          vgremove --yes --force "$vg"
-                        fi
-                      done
-                      # Wipe PVs
-                      for pv in $PVS; do
-                        pvremove --yes --force "$pv"
-                        wipefs --all "$pv"
-                        dd if=/dev/zero of="$pv" bs=1M count=100 oflag=direct,dsync
-                        # do not fail the timeout command
-                        # the command seems to hang sometimes for no reason so let's retry
-                        set +Ee
-                        for i in $(seq 1 3); do
-                            timeout --preserve-status 5s sgdisk --zap-all "$pv"
-                            if [ "$?" -eq 0 ]; then
-                                break
-                            fi
-                            sleep 5
-                        done
-                        set -Ee
-                      done
-                      # Wipe the specific disk in the CI that was running in raw mode
-                      set +Ee
-                      block=` + TestScratchDevice() + `
-                      wipefs --all "$block"
-                      dd if=/dev/zero of="$block" bs=1M count=100 oflag=direct,dsync
-                      set -Ee
-                      # Useful debug commands
-                      lsblk
-                      blkid
-                      df /
-                      #
-                      # some devices might still be mapped that lock the disks
-                      # this can fail with long-gone vestigial devices, so just assume this is successful
-                      ls /dev/mapper/ceph-* | xargs -I% -- dmsetup remove --force % || true
-                      rm -rf /dev/mapper/ceph-*  # clutter
-                      #
-                      # ceph-volume setup also leaves ceph-UUID directories in /dev (just clutter)
-                      rm -rf /dev/ceph-*
-          nodeSelector:
-            kubernetes.io/hostname: ` + nodeName + `
-          volumes:
-              - name: dev
-                hostPath:
-                  path: /dev
-              - name: run-udev
-                hostPath:
-                  path: /run/udev
-`
-}
-
 // GetCleanupPod gets a cleanup Pod that cleans up the dataDirHostPath
 func (h *CephInstaller) GetCleanupPod(node, removalDir string) string {
 	return `apiVersion: batch/v1
@@ -1049,10 +896,18 @@ func (h *CephInstaller) waitForCleanupJobs(namespace string) error {
 			return false, nil
 		}
 		for _, job := range cleanupJobs.Items {
-			logger.Debugf("job %q status: %+v", job.Name, job.Status)
+			logger.Infof("job %q status: %+v", job.Name, job.Status)
 			if job.Status.Failed > 0 {
 				return false, fmt.Errorf("job %s failed", job.Name)
 			}
+			if job.Status.Succeeded == 1 {
+				l, err := h.k8shelper.Kubectl("-n", namespace, "logs", fmt.Sprintf("job.batch/%s", job.Name))
+				if err != nil {
+					logger.Errorf("cannot get logs for pod %s. %v", job.Name, err)
+				}
+				rawData := []byte(l)
+				logger.Infof("cleanup job %s done. logs: %s", job.Name, string(rawData))
+			}
 			if job.Status.Succeeded == 0 {
 				return false, nil
 			}

tests/framework/installer/ceph_manifests.go

@@ -254,6 +254,17 @@ spec:
                 confirmation:
                   type: string
                   pattern: ^$|^yes-really-destroy-data$
+                sanitizeDisks:
+                  properties:
+                    method:
+                      type: string
+                      pattern: ^(complete|quick)$
+                    dataSource:
+                      type: string
+                      pattern: ^(zero|random)$
+                    iteration:
+                      type: integer
+                      format: int32
             placement: {}
             resources: {}
   additionalPrinterColumns:

tests/integration/ceph_flex_test.go

@@ -92,7 +92,7 @@ func (s *CephFlexDriverSuite) SetupSuite() {
 		usePVC:                  false,
 		mons:                    1,
 		rbdMirrorWorkers:        1,
-		rookCephCleanup:         false,
+		rookCephCleanup:         true,
 		skipOSDCreation:         false,
 		minimalMatrixK8sVersion: flexDriverMinimalTestVersion,
 		rookVersion:             installer.VersionMaster,