From 10ed662bdb25c7ae22f3ecaba87374a089a07956 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Mon, 25 May 2020 16:33:09 +0200 Subject: [PATCH 1/5] ceph: enhancement disk sanitize options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now, we can not only cleanup monitor data, logs and crashes but the disks too. As part of the cleanupPolicy CR spec, we have a new setting called sanitizeDisk which holds more details: * method: indicates if the entire disk should be sanitized or simply ceph's metadata. Possible choices are 'complete' or 'quick' (default) * dataSource: indicate where to get random bytes from to write on the disk. Possible choices are 'zero' (default) or 'random' Using random sources will consume entropy from the system and will take much more time then the zero source * iteration: overwrite N times instead of the default (1). Takes an integer value See the documentation for more details. Signed-off-by: Sébastien Han --- Documentation/ceph-cluster-crd.md | 51 +++--- .../charts/rook-ceph/templates/resources.yaml | 11 ++ cluster/examples/kubernetes/ceph/cluster.yaml | 14 +- cluster/examples/kubernetes/ceph/common.yaml | 11 ++ cmd/rook/ceph/cleanup.go | 24 ++- pkg/apis/ceph.rook.io/v1/cleanup.go | 18 +- pkg/apis/ceph.rook.io/v1/types.go | 13 +- pkg/daemon/ceph/cleanup/disk.go | 57 ++++--- pkg/daemon/ceph/cleanup/disk_test.go | 68 ++++++++ pkg/daemon/ceph/osd/daemon.go | 2 +- pkg/daemon/ceph/osd/volume.go | 5 +- pkg/operator/ceph/cluster/cleanup.go | 23 ++- tests/framework/installer/ceph_installer.go | 160 +----------------- tests/framework/installer/ceph_manifests.go | 11 ++ 14 files changed, 250 insertions(+), 218 deletions(-) create mode 100644 pkg/daemon/ceph/cleanup/disk_test.go diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index a4d859b6f..fff65e70a 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -149,30 +149,7 @@ For more details on the mons and when to choose a number other than `3`, see the * `manageMachineDisruptionBudgets`: if `true`, the operator will create and manage MachineDisruptionBudgets to ensure OSDs are only fenced when the cluster is healthy. Only available on OpenShift. * `machineDisruptionBudgetNamespace`: the namespace in which to watch the MachineDisruptionBudgets. * `removeOSDsIfOutAndSafeToRemove`: If `true` the operator will remove the OSDs that are down and whose data has been restored to other OSDs. In Ceph terms, the osds are `out` and `safe-to-destroy` when then would be removed. -* `cleanupPolicy`: The section for confirming that cluster data should be forcibly deleted. The cleanupPolicy should only be added to the cluster when the cluster is about to be deleted. After any field of the cleanup policy is set, Rook will stop configuring the cluster as if the cluster is about to be destroyed in order to prevent these settings from being deployed unintentionally. - * `confirmation`: If `yes-really-destroy-data` the operator will automatically delete data on the hostpath of cluster nodes and clean devices with OSDs when a `delete cephcluster` command is issued. Only `yes-really-destroy-data` and an empty string are valid values for this field. -* `healthCheck`: control period health status checks and livenessprobes, see the [health settings](#health-settings) - -To activate the cleanup, you can use the following command **AT YOUR OWN RISK**: - -```console -kubectl -n rook-ceph patch cephcluster rook-ceph --type merge -p '{"spec":{"cleanupPolicy":{"confirmation":"yes-really-destroy-data"}}}' -``` - -When applied you will see the following from the Operator's logs: - -```text -2020-05-27 13:24:04.267665 I | ceph-spec: CR has changed for "rook-ceph". diff= v1.ClusterSpec{ - ... // 16 identical fields - Mgr: v1.MgrSpec{}, - RemoveOSDsIfOutAndSafeToRemove: false, -- CleanupPolicy: v1.CleanupPolicySpec{}, -+ CleanupPolicy: v1.CleanupPolicySpec{Confirmation: "yes-really-destroy-data"}, - } -``` - -Nothing will happen until the deletion of the CR is requested, so this can still be reverted. -However, all new orchestration/reconciliation will be blocked with this cleanup policy enabled. +* `cleanupPolicy`: [cleanup policy settings](#cleanup-policy) ### Ceph container images @@ -1029,3 +1006,29 @@ spec: cephVersion: image: ceph/ceph:v15.2.4 # Should match external cluster version ``` + +### Cleanup policy + +Rook has the ability to cleanup resources and data that were deployed. +The policy represents the confirmation that cluster data should be forcibly deleted. +The cleanupPolicy should only be added to the cluster when the cluster is about to be deleted. +After the `confirmation` field of the cleanup policy is set, Rook will stop configuring the cluster as if the cluster is about to be destroyed in order to prevent these settings from being deployed unintentionally. +The `cleanupPolicy` CR settings has different fields: + +* `confirmation`: If `yes-really-destroy-data` the operator will automatically delete data on the hostpath of cluster nodes and clean devices with OSDs when a `delete cephcluster` command is issued. Only `yes-really-destroy-data` and an empty string are valid values for this field. +* `sanitizeDisks`: sanitizeDisks represents advanced settings that can be used to sanitize drives. +By default if the `confirmation` is set, Rook will only remove Ceph's metadata, which allows a cluster re-installation if desired. +However, the administrator might want to sanitize the drives in more depth with the following flags: + * `method`: indicates if the entire disk should be sanitized or simply ceph's metadata. Possible choices are 'quick' (default) or 'complete' + * `dataSource`: indicate where to get random bytes from to write on the disk. Possible choices are 'zero' (default) or 'random'. + Using random sources will consume entropy from the system and will take much more time then the zero source + * `iteration`: overwrite N times instead of the default (1). Takes an integer value + +To automate activation of the cleanup, you can use the following command. **WARNING: DATA WILL BE PERMANENTLY DELETED**: + +```console +kubectl -n rook-ceph patch cephcluster rook-ceph --type merge -p '{"spec":{"cleanupPolicy":{"confirmation":"yes-really-destroy-data"}}}' +``` + +Nothing will happen until the deletion of the CR is requested, so this can still be reverted. +However, all new configuration by the operator will be blocked with this cleanup policy enabled. \ No newline at end of file diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index 06160bdb8..67cb2e0f9 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -169,6 +169,17 @@ spec: confirmation: type: string pattern: ^$|^yes-really-destroy-data$ + sanitizeDisks: + properties: + method: + type: string + pattern: ^(complete|quick)$ + dataSource: + type: string + pattern: ^(zero|random)$ + iteration: + type: integer + format: int32 additionalPrinterColumns: - name: DataDirHostPath type: string diff --git a/cluster/examples/kubernetes/ceph/cluster.yaml b/cluster/examples/kubernetes/ceph/cluster.yaml index 684ac6b46..ad391b8fc 100644 --- a/cluster/examples/kubernetes/ceph/cluster.yaml +++ b/cluster/examples/kubernetes/ceph/cluster.yaml @@ -92,7 +92,19 @@ spec: # To signify that automatic deletion is desired, use the value "yes-really-destroy-data". Only this and an empty # string are valid values for this field. confirmation: "" - + # sanitizeDisks represents settings for sanitizing OSD disks on cluster deletion + sanitizeDisks: + # method indicates if the entire disk should be sanitized or simply ceph's metadata + # in both case, re-install is possible + # possible choices are 'complete' or 'quick' (default) + method: quick + # dataSource indicate where to get random bytes from to write on the disk + # possible choices are 'zero' (default) or 'random' + # using random sources will consume entropy from the system and will take much more time then the zero source + dataSource: zero + # iteration overwrite N times instead of the default (1) + # takes an integer value + iteration: 1 # To control where various services will be scheduled by kubernetes, use the placement configuration sections below. # The example under 'all' would have all services scheduled on kubernetes nodes labeled with 'role=storage-node' and # tolerate taints with a key of 'storage-node'. diff --git a/cluster/examples/kubernetes/ceph/common.yaml b/cluster/examples/kubernetes/ceph/common.yaml index 8897eb66c..9ea5aca67 100644 --- a/cluster/examples/kubernetes/ceph/common.yaml +++ b/cluster/examples/kubernetes/ceph/common.yaml @@ -192,6 +192,17 @@ spec: confirmation: type: string pattern: ^$|^yes-really-destroy-data$ + sanitizeDisks: + properties: + method: + type: string + pattern: ^(complete|quick)$ + dataSource: + type: string + pattern: ^(zero|random)$ + iteration: + type: integer + format: int32 placement: {} resources: {} healthCheck: {} diff --git a/cmd/rook/ceph/cleanup.go b/cmd/rook/ceph/cleanup.go index 29951c358..fe16107cd 100644 --- a/cmd/rook/ceph/cleanup.go +++ b/cmd/rook/ceph/cleanup.go @@ -20,6 +20,7 @@ import ( "os" "github.com/rook/rook/cmd/rook/rook" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" cleanup "github.com/rook/rook/pkg/daemon/ceph/cleanup" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/k8sutil" @@ -28,10 +29,13 @@ import ( ) var ( - dataDirHostPath string - namespaceDir string - monSecret string - clusterFSID string + dataDirHostPath string + namespaceDir string + monSecret string + clusterFSID string + sanitizeMethod string + sanitizeDataSource string + sanitizeIteration int32 ) var cleanUpCmd = &cobra.Command{ @@ -44,6 +48,9 @@ func init() { cleanUpCmd.Flags().StringVar(&namespaceDir, "namespace-dir", "", "dataDirHostPath on the node") cleanUpCmd.Flags().StringVar(&monSecret, "mon-secret", "", "monitor secret from the keyring") cleanUpCmd.Flags().StringVar(&clusterFSID, "cluster-fsid", "", "ceph cluster fsid") + cleanUpCmd.Flags().StringVar(&sanitizeMethod, "sanitize-method", string(cephv1.SanitizeMethodQuick), "sanitize method to use (metadata or data)") + cleanUpCmd.Flags().StringVar(&sanitizeDataSource, "sanitize-data-source", string(cephv1.SanitizeDataSourceZero), "data source to sanitize the disk (zero or random)") + cleanUpCmd.Flags().Int32Var(&sanitizeIteration, "sanitize-iteration", 1, "overwrite N times the disk") flags.SetFlagsFromEnv(cleanUpCmd.Flags(), rook.RookEnvVarPrefix) cleanUpCmd.RunE = startCleanUp } @@ -64,7 +71,14 @@ func startCleanUp(cmd *cobra.Command, args []string) error { clusterInfo.FSID = clusterFSID // Build Sanitizer - s := cleanup.NewDiskSanitizer(createContext(), clusterInfo) + s := cleanup.NewDiskSanitizer(createContext(), + clusterInfo, + &cephv1.SanitizeDisksSpec{ + Method: cephv1.SanitizeMethodProperty(sanitizeMethod), + DataSource: cephv1.SanitizeDataSourceProperty(sanitizeDataSource), + Iteration: sanitizeIteration, + }, + ) // Start OSD wipe process s.StartSanitizeDisks() diff --git a/pkg/apis/ceph.rook.io/v1/cleanup.go b/pkg/apis/ceph.rook.io/v1/cleanup.go index ab52e3f1a..0fedf49f8 100644 --- a/pkg/apis/ceph.rook.io/v1/cleanup.go +++ b/pkg/apis/ceph.rook.io/v1/cleanup.go @@ -17,6 +17,18 @@ limitations under the License. package v1 const ( + // SanitizeDataSourceZero uses /dev/zero as sanitize source + SanitizeDataSourceZero SanitizeDataSourceProperty = "zero" + + // SanitizeDataSourceRandom uses `shred's default entropy source + SanitizeDataSourceRandom SanitizeDataSourceProperty = "random" + + // SanitizeMethodComplete will sanitize everything on the disk + SanitizeMethodComplete SanitizeMethodProperty = "complete" + + // SanitizeMethodQuick will sanitize metdata only on the disk + SanitizeMethodQuick SanitizeMethodProperty = "quick" + // DeleteDataDirOnHostsConfirmation represents the validation to destry dataDirHostPath DeleteDataDirOnHostsConfirmation CleanupConfirmationProperty = "yes-really-destroy-data" ) @@ -26,6 +38,10 @@ func (c *CleanupPolicySpec) HasDataDirCleanPolicy() bool { return c.Confirmation == DeleteDataDirOnHostsConfirmation } -func (c *CleanupConfirmationProperty) String() string { +func (c *SanitizeMethodProperty) String() string { + return string(*c) +} + +func (c *SanitizeDataSourceProperty) String() string { return string(*c) } diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 13c40d0ef..eb95d9948 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -705,11 +705,22 @@ type ClientSpec struct { } type CleanupPolicySpec struct { - Confirmation CleanupConfirmationProperty `json:"confirmation,omitempty"` + Confirmation CleanupConfirmationProperty `json:"confirmation,omitempty"` + SanitizeDisks SanitizeDisksSpec `json:"sanitizeDisks"` } type CleanupConfirmationProperty string +type SanitizeDataSourceProperty string + +type SanitizeMethodProperty string + +type SanitizeDisksSpec struct { + Method SanitizeMethodProperty `json:"method,omitempty"` + DataSource SanitizeDataSourceProperty `json:"dataSource,omitempty"` + Iteration int32 `json:"iteration,omitempty"` +} + // +genclient // +genclient:noStatus // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/daemon/ceph/cleanup/disk.go b/pkg/daemon/ceph/cleanup/disk.go index dcb589584..9c641f009 100644 --- a/pkg/daemon/ceph/cleanup/disk.go +++ b/pkg/daemon/ceph/cleanup/disk.go @@ -23,6 +23,7 @@ import ( "sync" "github.com/coreos/pkg/capnslog" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/daemon/ceph/osd" @@ -30,11 +31,8 @@ import ( ) const ( - ddBS = "1M" // DD's block size - ddCount = "10" // DD runs over the first 10 offsets - ddFlags = "direct,dsync" // DD's sync flags" - ddIf = "/dev/zero" - ddUtility = "dd" + shredUtility = "shred" + shredBS = "10M" // Shred's block size ) var ( @@ -43,15 +41,17 @@ var ( // DiskSanitizer is simple struct to old the context to execute the commands type DiskSanitizer struct { - context *clusterd.Context - clusterInfo *client.ClusterInfo + context *clusterd.Context + clusterInfo *client.ClusterInfo + sanitizeDisksSpec *cephv1.SanitizeDisksSpec } // NewDiskSanitizer is function that returns a full filled DiskSanitizer object -func NewDiskSanitizer(context *clusterd.Context, clusterInfo *client.ClusterInfo) *DiskSanitizer { +func NewDiskSanitizer(context *clusterd.Context, clusterInfo *client.ClusterInfo, sanitizeDisksSpec *cephv1.SanitizeDisksSpec) *DiskSanitizer { return &DiskSanitizer{ - context: context, - clusterInfo: clusterInfo, + context: context, + clusterInfo: clusterInfo, + sanitizeDisksSpec: sanitizeDisksSpec, } } @@ -144,23 +144,42 @@ func (s *DiskSanitizer) returnPVDevice(disk string) []string { return strings.Split(output, ":") } -func (s *DiskSanitizer) buildDDArgs(disk string) []string { - ddArgs := []string{ - fmt.Sprintf("if=%s", ddIf), - fmt.Sprintf("of=%s", disk), - fmt.Sprintf("bs=%s", ddBS), - fmt.Sprintf("count=%s", ddCount), - fmt.Sprintf("oflag=%s", ddFlags), +func (s *DiskSanitizer) buildDataSource() string { + return fmt.Sprintf("/dev/%s", s.sanitizeDisksSpec.DataSource.String()) +} + +func (s *DiskSanitizer) buildShredArgs(disk string) []string { + var shredArgs []string + + // If data source is not zero, then let's add zeros at the end of the pass + if s.sanitizeDisksSpec.DataSource != cephv1.SanitizeDataSourceZero { + shredArgs = append(shredArgs, "--zero") } - return ddArgs + // If this is a quick pass let's just overwrite the first 10MB + if s.sanitizeDisksSpec.Method == cephv1.SanitizeMethodQuick { + shredArgs = append(shredArgs, fmt.Sprintf("--size=%s", shredBS)) + } + + // If the data source for randomness is zero + if s.sanitizeDisksSpec.DataSource == cephv1.SanitizeDataSourceZero { + shredArgs = append(shredArgs, fmt.Sprintf("--random-source=%s", s.buildDataSource())) + } + + shredArgs = append(shredArgs, []string{ + "--force", + "--verbose", + fmt.Sprintf("--iterations=%s", strconv.Itoa(int(s.sanitizeDisksSpec.Iteration))), + disk}...) + + return shredArgs } func (s *DiskSanitizer) executeSanitizeCommand(disk string, wg *sync.WaitGroup) { // On return, notify the WaitGroup that we’re done defer wg.Done() - output, err := s.context.Executor.ExecuteCommandWithCombinedOutput(ddUtility, s.buildDDArgs(disk)...) + output, err := s.context.Executor.ExecuteCommandWithCombinedOutput(shredUtility, s.buildShredArgs(disk)...) if err != nil { logger.Errorf("failed to sanitize osd disk %q. %s. %v", disk, output, err) } diff --git a/pkg/daemon/ceph/cleanup/disk_test.go b/pkg/daemon/ceph/cleanup/disk_test.go new file mode 100644 index 000000000..73667bc23 --- /dev/null +++ b/pkg/daemon/ceph/cleanup/disk_test.go @@ -0,0 +1,68 @@ +/* +Copyright 2020 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cleanup + +import ( + "reflect" + "testing" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/stretchr/testify/assert" +) + +func TestBuildDataSource(t *testing.T) { + s := NewDiskSanitizer(&clusterd.Context{}, &client.ClusterInfo{}, &cephv1.SanitizeDisksSpec{}) + s.sanitizeDisksSpec.DataSource = cephv1.SanitizeDataSourceZero + + assert.Equal(t, "/dev/zero", s.buildDataSource()) +} + +func TestBuildShredArgs(t *testing.T) { + var i int32 = 1 + c := &clusterd.Context{} + disk := "/dev/sda" + type fields struct { + context *clusterd.Context + clusterInfo *client.ClusterInfo + sanitizeDisksSpec *cephv1.SanitizeDisksSpec + } + tests := []struct { + name string + fields fields + disk string + want []string + }{ + {"quick-zero", fields{c, &client.ClusterInfo{}, &cephv1.SanitizeDisksSpec{Method: cephv1.SanitizeMethodQuick, Iteration: i, DataSource: cephv1.SanitizeDataSourceZero}}, disk, []string{"--size=10M", "--random-source=/dev/zero", "--force", "--verbose", "--iterations=1", "/dev/sda"}}, + {"quick-random", fields{c, &client.ClusterInfo{}, &cephv1.SanitizeDisksSpec{Method: cephv1.SanitizeMethodQuick, Iteration: i, DataSource: cephv1.SanitizeDataSourceRandom}}, disk, []string{"--zero", "--size=10M", "--force", "--verbose", "--iterations=1", "/dev/sda"}}, + {"complete-zero", fields{c, &client.ClusterInfo{}, &cephv1.SanitizeDisksSpec{Method: cephv1.SanitizeMethodComplete, Iteration: i, DataSource: cephv1.SanitizeDataSourceZero}}, disk, []string{"--random-source=/dev/zero", "--force", "--verbose", "--iterations=1", "/dev/sda"}}, + {"complete-random", fields{c, &client.ClusterInfo{}, &cephv1.SanitizeDisksSpec{Method: cephv1.SanitizeMethodComplete, Iteration: i, DataSource: cephv1.SanitizeDataSourceRandom}}, disk, []string{"--zero", "--force", "--verbose", "--iterations=1", "/dev/sda"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &DiskSanitizer{ + context: tt.fields.context, + clusterInfo: tt.fields.clusterInfo, + sanitizeDisksSpec: tt.fields.sanitizeDisksSpec, + } + if got := s.buildShredArgs(tt.disk); !reflect.DeepEqual(got, tt.want) { + t.Errorf("DiskSanitizer.buildShredArgs() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index 2e37d7130..ce63e2024 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -55,7 +55,7 @@ func StartOSD(context *clusterd.Context, osdType, osdID, osdUUID, lvPath string, } // Update LVM config at runtime - if err := updateLVMConfig(context, pvcBackedOSD, lvBackedPV); err != nil { + if err := UpdateLVMConfig(context, pvcBackedOSD, lvBackedPV); err != nil { return errors.Wrap(err, "failed to update lvm configuration file") // fail return here as validation provided by ceph-volume } diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index 8980d8ab8..c438b0790 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -179,7 +179,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device // // Or keep doing this if the PV is backend by an LV already if !a.clusterInfo.CephVersion.IsAtLeast(cephVolumeRawModeMinCephVersion) || lvBackedPV { - if err := updateLVMConfig(context, a.pvcBacked, lvBackedPV); err != nil { + if err := UpdateLVMConfig(context, a.pvcBacked, lvBackedPV); err != nil { return nil, errors.Wrap(err, "failed to update lvm configuration file") } } @@ -343,7 +343,8 @@ func getLVPath(op string) string { return "" } -func updateLVMConfig(context *clusterd.Context, onPVC, lvBackedPV bool) error { +// UpdateLVMConfig updates the lvm.conf file +func UpdateLVMConfig(context *clusterd.Context, onPVC, lvBackedPV bool) error { input, err := ioutil.ReadFile(lvmConfPath) if err != nil { diff --git a/pkg/operator/ceph/cluster/cleanup.go b/pkg/operator/ceph/cluster/cleanup.go index 005fb5114..826ec8ea0 100644 --- a/pkg/operator/ceph/cluster/cleanup.go +++ b/pkg/operator/ceph/cluster/cleanup.go @@ -18,6 +18,7 @@ package cluster import ( "fmt" + "strconv" "time" "github.com/pkg/errors" @@ -44,11 +45,16 @@ const ( ) var ( - volumeName = "cleanup-volume" - dataDirHostPath = "ROOK_DATA_DIR_HOST_PATH" - namespaceDir = "ROOK_NAMESPACE_DIR" - monitorSecret = "ROOK_MON_SECRET" - clusterFSID = "ROOK_CLUSTER_FSID" + volumeName = "cleanup-volume" + dataDirHostPath = "ROOK_DATA_DIR_HOST_PATH" + namespaceDir = "ROOK_NAMESPACE_DIR" + monitorSecret = "ROOK_MON_SECRET" + clusterFSID = "ROOK_CLUSTER_FSID" + sanitizeMethod = "ROOK_SANITIZE_METHOD" + sanitizeDataSource = "ROOK_SANITIZE_DATA_SOURCE" + sanitizeLayer = "ROOK_SANITIZE_LAYER" + sanitizeIteration = "ROOK_SANITIZE_ITERATION" + sanitizeIterationDefault int32 = 1 ) func (c *ClusterController) startClusterCleanUp(stopCleanupCh chan struct{}, cluster *cephv1.CephCluster, cephHosts []string, monSecret, clusterFSID string) { @@ -94,6 +100,10 @@ func (c *ClusterController) cleanUpJobContainer(cluster *cephv1.CephCluster, mon volumeMounts := []v1.VolumeMount{} envVars := []v1.EnvVar{} if cluster.Spec.DataDirHostPath != "" { + if cluster.Spec.CleanupPolicy.SanitizeDisks.Iteration == 0 { + cluster.Spec.CleanupPolicy.SanitizeDisks.Iteration = sanitizeIterationDefault + } + hostPathVolumeMount := v1.VolumeMount{Name: volumeName, MountPath: cluster.Spec.DataDirHostPath} devMount := v1.VolumeMount{Name: "devices", MountPath: "/dev"} volumeMounts = append(volumeMounts, hostPathVolumeMount) @@ -105,6 +115,9 @@ func (c *ClusterController) cleanUpJobContainer(cluster *cephv1.CephCluster, mon {Name: clusterFSID, Value: cephFSID}, {Name: "ROOK_LOG_LEVEL", Value: "DEBUG"}, mon.PodNamespaceEnvVar(cluster.Namespace), + {Name: sanitizeMethod, Value: cluster.Spec.CleanupPolicy.SanitizeDisks.Method.String()}, + {Name: sanitizeDataSource, Value: cluster.Spec.CleanupPolicy.SanitizeDisks.DataSource.String()}, + {Name: sanitizeIteration, Value: strconv.Itoa(int(cluster.Spec.CleanupPolicy.SanitizeDisks.Iteration))}, }...) } diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 54e9af97a..dc5987c59 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -97,10 +97,6 @@ func (h *CephInstaller) CreateCephOperator(namespace string) (err error) { if err != nil { return fmt.Errorf("Failed to start admission controllers: %v", err) } - - if err = h.WipeDisks(namespace, 2); err != nil { - return err - } rookOperator := h.Manifests.GetRookOperator(namespace) _, err = h.k8shelper.KubectlWithStdin(rookOperator, createFromStdinArgs...) if err != nil { @@ -190,9 +186,6 @@ func (h *CephInstaller) CreateRookOperatorViaHelm(namespace, chartSettings strin } - if err = h.WipeDisks(namespace, 2); err != nil { - return err - } return nil } @@ -805,158 +798,6 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, useH return h } -// Wipe cluster disks -func (h *CephInstaller) WipeDisks(namespace string, retries int) (err error) { - for i := 1; i <= retries; i++ { - if err = h.wipeClusterDisks(namespace); err == nil { - logger.Info("disks wiped successfully") - return nil - } else { - logger.Warningf("failed to wipe cluster disks. %+v. Attemp %d/%d ...", err, i, retries) - } - } - return fmt.Errorf("failed to wipe cluster disks. %+v", err) -} - -// wipeClusterDisks runs a disk wipe job on all nodes in the k8s cluster. -func (h *CephInstaller) wipeClusterDisks(namespace string) error { - // Wipe clean disks on all nodes - nodes, err := h.GetNodeHostnames() - if err != nil { - return fmt.Errorf("failed to get node hostnames. %+v", err) - } - var jobNames []string - for _, node := range nodes { - // Start the new job - globalTestJobIndex++ - jobName := fmt.Sprintf("rook-ceph-disk-wipe-%d", globalTestJobIndex) - jobNames = append(jobNames, jobName) - job := h.GetDiskWipeJob(node, jobName, namespace) - _, err := h.k8shelper.KubectlWithStdin(job, createFromStdinArgs...) - if err != nil { - return fmt.Errorf("failed to create disk wipe job for host %s. %+v", node, err) - } - } - - allJobsAreComplete := func() (done bool, err error) { - for _, jobName := range jobNames { - j, err := h.k8shelper.Clientset.BatchV1().Jobs(namespace).Get(jobName, metav1.GetOptions{}) - if err != nil { - return false, nil - } - if j.Status.Failed > 0 { - return false, fmt.Errorf("job %s failed", jobName) - } - if j.Status.Succeeded == 0 { - return false, nil - } - } - return true, nil - } - - // return the error below after cleaning up the jobs - err = wait.Poll(5*time.Second, 90*time.Second, allJobsAreComplete) - if err != nil { - return fmt.Errorf("failed to wait for wipe jobs to complete. %+v", err) - } - return nil -} - -// GetDiskWipeJob returns a YAML manifest string for a job which will wipe clean the extra -// (non-boot) disks on a node, allowing Ceph to use the disk(s) during its testing. -func (h *CephInstaller) GetDiskWipeJob(nodeName, jobName, namespace string) string { - // put the wipe job in the cluster namespace so that logs get picked up in failure conditions - return `apiVersion: batch/v1 -kind: Job -metadata: - name: ` + jobName + ` - namespace: ` + namespace + ` -spec: - template: - spec: - restartPolicy: OnFailure - containers: - - name: disk-wipe - image: ` + h.CephVersion.Image + ` - securityContext: - privileged: true - volumeMounts: - - name: dev - mountPath: /dev - - name: run-udev - mountPath: /run/udev - command: - - "sh" - - "-c" - - | - set -xEeuo pipefail - # Wipe LVM data from disks - # - udevadm trigger || true - vgimport -a || true - # We should prepopulate the PVs corresponding to Ceph VGs here. - # It's because getting the relationship of the PVs and the VGs - # is impossible after removing the VGs. - TMP=$(pvs --noheadings --separator=',' -o pv_name,vg_name) - PVS= - for line in $TMP ; do - if [[ $line =~ ,ceph- ]]; then - PVS="$PVS ${line%,*}" - fi - done - # Wipe VGs - for vg in $(vgs --noheadings --readonly --separator=' ' -o vg_name); do - if [[ $vg =~ ^ceph- ]]; then - vgremove --yes --force "$vg" - fi - done - # Wipe PVs - for pv in $PVS; do - pvremove --yes --force "$pv" - wipefs --all "$pv" - dd if=/dev/zero of="$pv" bs=1M count=100 oflag=direct,dsync - # do not fail the timeout command - # the command seems to hang sometimes for no reason so let's retry - set +Ee - for i in $(seq 1 3); do - timeout --preserve-status 5s sgdisk --zap-all "$pv" - if [ "$?" -eq 0 ]; then - break - fi - sleep 5 - done - set -Ee - done - # Wipe the specific disk in the CI that was running in raw mode - set +Ee - block=` + TestScratchDevice() + ` - wipefs --all "$block" - dd if=/dev/zero of="$block" bs=1M count=100 oflag=direct,dsync - set -Ee - # Useful debug commands - lsblk - blkid - df / - # - # some devices might still be mapped that lock the disks - # this can fail with long-gone vestigial devices, so just assume this is successful - ls /dev/mapper/ceph-* | xargs -I% -- dmsetup remove --force % || true - rm -rf /dev/mapper/ceph-* # clutter - # - # ceph-volume setup also leaves ceph-UUID directories in /dev (just clutter) - rm -rf /dev/ceph-* - nodeSelector: - kubernetes.io/hostname: ` + nodeName + ` - volumes: - - name: dev - hostPath: - path: /dev - - name: run-udev - hostPath: - path: /run/udev -` -} - // GetCleanupPod gets a cleanup Pod that cleans up the dataDirHostPath func (h *CephInstaller) GetCleanupPod(node, removalDir string) string { return `apiVersion: batch/v1 @@ -1028,6 +869,7 @@ func (h *CephInstaller) addCleanupPolicy(namespace string) error { return fmt.Errorf("failed to get ceph cluster. %+v", err) } cluster.Spec.CleanupPolicy.Confirmation = cephv1.DeleteDataDirOnHostsConfirmation + cluster.Spec.CleanupPolicy.SanitizeDisks.Method = cephv1.SanitizeMethodComplete _, err = h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Update(cluster) if err != nil { return fmt.Errorf("failed to add clean up policy to the cluster. %+v", err) diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index db68ec1b9..a8e610204 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -254,6 +254,17 @@ spec: confirmation: type: string pattern: ^$|^yes-really-destroy-data$ + sanitizeDisks: + properties: + method: + type: string + pattern: ^(complete|quick)$ + dataSource: + type: string + pattern: ^(zero|random)$ + iteration: + type: integer + format: int32 placement: {} resources: {} additionalPrinterColumns: From 6444675d4d06420800dc8c3abe8ed0c2b6097c1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Fri, 10 Jul 2020 11:08:24 +0200 Subject: [PATCH 2/5] ci: debug cleanup job logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add logs of the cleanup jobs on success. Signed-off-by: Sébastien Han --- tests/framework/installer/ceph_installer.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index dc5987c59..95fbad9a0 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -543,9 +543,6 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, name h.GatherAllRookLogs(h.T().Name()+"poolcheck", systemNamespace) } - roles := h.Manifests.GetClusterRoles(namespace, systemNamespace) - _, err = h.k8shelper.KubectlWithStdin(roles, deleteFromStdinArgs...) - err = h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cephcluster", h.clusterName) checkError(h.T(), err, fmt.Sprintf("cannot remove cluster %s", namespace)) @@ -554,6 +551,9 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, name assert.NoError(h.T(), err) } + roles := h.Manifests.GetClusterRoles(namespace, systemNamespace) + _, err = h.k8shelper.KubectlWithStdin(roles, deleteFromStdinArgs...) + crdCheckerFunc := func() error { _, err := h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Get(h.clusterName, metav1.GetOptions{}) // ensure the finalizer(s) are removed @@ -869,7 +869,6 @@ func (h *CephInstaller) addCleanupPolicy(namespace string) error { return fmt.Errorf("failed to get ceph cluster. %+v", err) } cluster.Spec.CleanupPolicy.Confirmation = cephv1.DeleteDataDirOnHostsConfirmation - cluster.Spec.CleanupPolicy.SanitizeDisks.Method = cephv1.SanitizeMethodComplete _, err = h.k8shelper.RookClientset.CephV1().CephClusters(namespace).Update(cluster) if err != nil { return fmt.Errorf("failed to add clean up policy to the cluster. %+v", err) @@ -891,10 +890,18 @@ func (h *CephInstaller) waitForCleanupJobs(namespace string) error { return false, nil } for _, job := range cleanupJobs.Items { - logger.Debugf("job %q status: %+v", job.Name, job.Status) + logger.Infof("job %q status: %+v", job.Name, job.Status) if job.Status.Failed > 0 { return false, fmt.Errorf("job %s failed", job.Name) } + if job.Status.Succeeded == 1 { + l, err := h.k8shelper.Kubectl("-n", namespace, "logs", fmt.Sprintf("job.batch/%s", job.Name)) + if err != nil { + logger.Errorf("cannot get logs for pod %s. %v", job.Name, err) + } + rawData := []byte(l) + logger.Infof("cleanup job %s done. logs: %s", job.Name, string(rawData)) + } if job.Status.Succeeded == 0 { return false, nil } From bc4875a5ebc097bbbad829ee5ce3e1a7cec81e69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Tue, 21 Jul 2020 08:57:59 +0200 Subject: [PATCH 3/5] ci: only apply cleanup policy on converged cluster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is no point of asking for a cleanup when the cluster is external, also the cleanup pod will never exist and the CI will fail waiting for it. Signed-off-by: Sébastien Han --- tests/framework/installer/ceph_installer.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 95fbad9a0..d4f13dc59 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -520,8 +520,11 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, name for clusterNum, namespace := range namespaces { if h.cleanupHost { //Add cleanup policy to the ceph cluster - err = h.addCleanupPolicy(namespace) - assert.NoError(h.T(), err) + // Add cleanup policy to the ceph cluster but NOT the external one + if clusterNum == 0 { + err = h.addCleanupPolicy(namespace) + assert.NoError(h.T(), err) + } } if !h.T().Failed() { @@ -547,8 +550,11 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, name checkError(h.T(), err, fmt.Sprintf("cannot remove cluster %s", namespace)) if h.cleanupHost { - err = h.waitForCleanupJobs(namespace) - assert.NoError(h.T(), err) + // The second cluster is external so the cleanup pod will never exist! + if clusterNum == 0 { + err = h.waitForCleanupJobs(namespace) + assert.NoError(h.T(), err) + } } roles := h.Manifests.GetClusterRoles(namespace, systemNamespace) From 698f46e97814b72da970310546579da4a48f90c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Tue, 21 Jul 2020 14:06:14 +0200 Subject: [PATCH 4/5] ci: purge after flexsuite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If we don't the mgr tests cannot verify the osd creation and will fail with: ``` --- FAIL: TestCephMgrSuite (169.32s) --- FAIL: TestCephMgrSuite/TestCreateOSD (1.65s) ceph_mgr_test.go:217: Error Trace: ceph_mgr_test.go:217 Error: Should not be: "" Test: TestCephMgrSuite/TestCreateOSD Messages: No devices available to create test OSD ceph_mgr_test.go:218: Error Trace: ceph_mgr_test.go:218 Error: Should not be: "" Test: TestCephMgrSuite/TestCreateOSD Messages: No nodes available to create test OSD ``` Signed-off-by: Sébastien Han --- tests/integration/ceph_flex_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/ceph_flex_test.go b/tests/integration/ceph_flex_test.go index d3d4e7053..a785d1e34 100644 --- a/tests/integration/ceph_flex_test.go +++ b/tests/integration/ceph_flex_test.go @@ -92,7 +92,7 @@ func (s *CephFlexDriverSuite) SetupSuite() { usePVC: false, mons: 1, rbdMirrorWorkers: 1, - rookCephCleanup: false, + rookCephCleanup: true, skipOSDCreation: false, minimalMatrixK8sVersion: flexDriverMinimalTestVersion, rookVersion: installer.VersionMaster, From 0a6cce43fbb7a72582d4e7d1953daf035f8798ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Tue, 21 Jul 2020 17:11:29 +0200 Subject: [PATCH 5/5] ceph: add clusterResourceDeleted flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit During the deletion sequence of the CR, we now set a success flag at the end, right before removing the finalizer. This avoids the case where we fail to remove the finalizer. 2020-07-21 13:29:46.334816 E | ceph-object-controller: failed to reconcile failed to remove finalizer: failed to remove finalizer "cephobjectstore.ceph.rook.io" on "teststore": Operation cannot be fulfilled on cephobjectstores.ceph.rook.io "teststore": the object has been modified; please apply your changes to the latest version and try again Signed-off-by: Sébastien Han --- pkg/operator/ceph/object/controller.go | 70 ++++++++++++++------------ 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index 06eb4c56e..3e630fe65 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -75,13 +75,14 @@ var controllerTypeMeta = metav1.TypeMeta{ // ReconcileCephObjectStore reconciles a cephObjectStore object type ReconcileCephObjectStore struct { - client client.Client - bktclient bktclient.Interface - scheme *runtime.Scheme - context *clusterd.Context - cephClusterSpec *cephv1.ClusterSpec - clusterInfo *cephclient.ClusterInfo - objectStoreChannels map[string]*objectStoreHealth + client client.Client + bktclient bktclient.Interface + scheme *runtime.Scheme + context *clusterd.Context + cephClusterSpec *cephv1.ClusterSpec + clusterInfo *cephclient.ClusterInfo + objectStoreChannels map[string]*objectStoreHealth + clusterResourceDeleted bool } type objectStoreHealth struct { @@ -221,35 +222,40 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci if !cephObjectStore.GetDeletionTimestamp().IsZero() { logger.Debugf("deleting store %q", cephObjectStore.Name) - response, okToDelete := r.verifyObjectBucketCleanup(cephObjectStore) - if !okToDelete { - // If the object store cannot be deleted, requeue the request for deletion to see if the conditions - // will eventually be satisfied such as the object buckets being removed - return response, nil - } + if !r.clusterResourceDeleted { + response, okToDelete := r.verifyObjectBucketCleanup(cephObjectStore) + if !okToDelete { + // If the object store cannot be deleted, requeue the request for deletion to see if the conditions + // will eventually be satisfied such as the object buckets being removed + return response, nil + } - response, okToDelete = r.verifyObjectUserCleanup(cephObjectStore) - if !okToDelete { - // If the object store cannot be deleted, requeue the request for deletion to see if the conditions - // will eventually be satisfied such as the object users being removed - return response, nil - } + response, okToDelete = r.verifyObjectUserCleanup(cephObjectStore) + if !okToDelete { + // If the object store cannot be deleted, requeue the request for deletion to see if the conditions + // will eventually be satisfied such as the object users being removed + return response, nil + } - // Close the channel to stop the healthcheck of the endpoint - close(r.objectStoreChannels[cephObjectStore.Name].stopChan) + // Close the channel to stop the healthcheck of the endpoint + close(r.objectStoreChannels[cephObjectStore.Name].stopChan) - // Remove object store from the map - delete(r.objectStoreChannels, cephObjectStore.Name) + // Remove object store from the map + delete(r.objectStoreChannels, cephObjectStore.Name) - cfg := clusterConfig{ - context: r.context, - store: cephObjectStore, - clusterSpec: r.cephClusterSpec, - clusterInfo: r.clusterInfo, - } - err = cfg.deleteStore() - if err != nil { - return reconcile.Result{}, errors.Wrapf(err, "failed to delete store %q", cephObjectStore.Name) + cfg := clusterConfig{ + context: r.context, + store: cephObjectStore, + clusterSpec: r.cephClusterSpec, + clusterInfo: r.clusterInfo, + } + err = cfg.deleteStore() + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to delete store %q", cephObjectStore.Name) + } + + // Mark the ressource deletion as complete + r.clusterResourceDeleted = true } // Remove finalizer