feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

Merge pull request #10191 from leseb/fix-10181

Browse Source 
osd: close the encrypted disk after cleanup is done
This commit is contained in:
Sébastien Han authored and GitHub committed 2022-05-03 17:28:36 +02:00
2 parents 5a4c77efa5 6ef491d9fc
commit 83a0dd32c8
7 files changed
+118 -58
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/canary-integration-test.yml
+22
View File
@@ -207,6 +207,17 @@ jobs:
- name: check-ownerreferences
run: tests/scripts/github-action-helper.sh check_ownerreferences
- name: teardown cluster with cleanup policy
run: |
kubectl -n rook-ceph patch cephcluster rook-ceph --type merge -p '{"spec":{"cleanupPolicy":{"confirmation":"yes-really-destroy-data"}}}'
kubectl -n rook-ceph delete cephcluster rook-ceph
kubectl -n rook-ceph logs deploy/rook-ceph-operator
tests/scripts/github-action-helper.sh wait_for_cleanup_pod
lsblk
BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)
sudo head --bytes=60 ${BLOCK}1
sudo head --bytes=60 ${BLOCK}2
- name: collect common logs
if: always()
uses: ./.github/workflows/collect-logs
@@ -363,6 +374,17 @@ jobs:
kubectl -n rook-ceph delete deploy/rook-ceph-osd-0
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
- name: teardown cluster with cleanup policy
run: |
kubectl -n rook-ceph patch cephcluster rook-ceph --type merge -p '{"spec":{"cleanupPolicy":{"confirmation":"yes-really-destroy-data"}}}'
kubectl -n rook-ceph delete cephcluster rook-ceph
kubectl -n rook-ceph logs deploy/rook-ceph-operator
tests/scripts/github-action-helper.sh wait_for_cleanup_pod
BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)
sudo head --bytes=60 ${BLOCK}1
sudo head --bytes=60 ${BLOCK}2
sudo lsblk
- name: collect common logs
if: always()
uses: ./.github/workflows/collect-logs
pkg/daemon/ceph/cleanup/disk.go
+16 -6
View File
@@ -87,7 +87,7 @@ func (s *DiskSanitizer) sanitizeRawDisk(osdRawList []oposd.OSDInfo) {
wg.Add(1)
// Put each sanitize in a go routine to speed things up
go s.executeSanitizeCommand(osd.BlockPath, &wg)
go s.executeSanitizeCommand(osd, &wg)
}
wg.Wait()
@@ -115,7 +115,7 @@ func (s *DiskSanitizer) sanitizeLVMDisk(osdLVMList []oposd.OSDInfo) {
// // purge remaining LVM2 metadata from PV
for _, pv := range pvs {
wg2.Add(1)
go s.executeSanitizeCommand(pv, &wg2)
go s.executeSanitizeCommand(oposd.OSDInfo{BlockPath: pv}, &wg2)
}
wg2.Wait()
}
@@ -175,15 +175,25 @@ func (s *DiskSanitizer) buildShredArgs(disk string) []string {
return shredArgs
}
func (s *DiskSanitizer) executeSanitizeCommand(disk string, wg *sync.WaitGroup) {
func (s *DiskSanitizer) executeSanitizeCommand(osdInfo oposd.OSDInfo, wg *sync.WaitGroup) {
// On return, notify the WaitGroup that we’re done
defer wg.Done()
output, err := s.context.Executor.ExecuteCommandWithCombinedOutput(shredUtility, s.buildShredArgs(disk)...)
output, err := s.context.Executor.ExecuteCommandWithCombinedOutput(shredUtility, s.buildShredArgs(osdInfo.BlockPath)...)
if err != nil {
logger.Errorf("failed to sanitize osd disk %q. %s. %v", disk, output, err)
logger.Errorf("failed to sanitize osd disk %q. %s. %v", osdInfo.BlockPath, output, err)
}
logger.Infof("%s\n", output)
logger.Infof("successfully sanitized osd disk %q", disk)
logger.Infof("successfully sanitized osd disk %q", osdInfo.BlockPath)
// If the device is encrypted let's close it after sanitizing its content
if osdInfo.Encrypted {
err := osd.CloseEncryptedDevice(s.context, osdInfo.BlockPath)
if err != nil {
logger.Errorf("failed to close encrypted osd disk %q. %v", osdInfo.BlockPath, err)
} else {
logger.Infof("successfully closed encrypted osd disk %q", osdInfo.BlockPath)
}
}
}
pkg/daemon/ceph/osd/encryption.go
+1 -1
View File
@@ -39,7 +39,7 @@ var (
luksLabelCephFSID = regexp.MustCompile("ceph_fsid=(.*)")
)
func closeEncryptedDevice(context *clusterd.Context, dmName string) error {
func CloseEncryptedDevice(context *clusterd.Context, dmName string) error {
args := []string{"--verbose", "luksClose", dmName}
cryptsetupOut, err := context.Executor.ExecuteCommandWithCombinedOutput(cryptsetupBinary, args...)
if err != nil {
pkg/daemon/ceph/osd/encryption_test.go
+1 -1
View File
@@ -79,7 +79,7 @@ func TestCloseEncryptedDevice(t *testing.T) {
}
context := &clusterd.Context{Executor: executor}
err := closeEncryptedDevice(context, "/dev/mapper/ceph-43e9efed-0676-4731-b75a-a4c42ece1bb1-xvdbr-block-dmcrypt")
err := CloseEncryptedDevice(context, "/dev/mapper/ceph-43e9efed-0676-4731-b75a-a4c42ece1bb1-xvdbr-block-dmcrypt")
assert.NoError(t, err)
}
pkg/daemon/ceph/osd/volume.go
+67 -50
View File
@@ -938,7 +938,12 @@ func readCVLogContent(cvLogFilePath string) string {
return string(b)
}
// GetCephVolumeRawOSDs list OSD prepared with raw mode
// GetCephVolumeRawOSDs list OSD prepared with raw mode.
// Sometimes this function called against a device, sometimes it's not. For instance, in the cleanup
// scenario, we don't pass any block because we are looking for all the OSDs present on the machine.
// On the other hand, the PVC scenario always uses the PVC block as a block to check whether the
// disk is an OSD or not.
// The same goes for "metadataBlock" and "walBlock" they are only used in the prepare job.
func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, cephfsid, block, metadataBlock, walBlock string, lvBackedPV, skipDeviceClass bool) ([]oposd.OSDInfo, error) {
// lv can be a block device if raw mode is used
cvMode := "raw"
@@ -960,55 +965,57 @@ func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.Cluster
//
// isCephEncryptedBlock() returns false if the disk is not a LUKS device with:
// Device /dev/sdc is not a valid LUKS device.
if isCephEncryptedBlock(context, cephfsid, block) {
childDevice, err := sys.ListDevicesChild(context.Executor, block)
if err != nil {
return nil, err
}
var encryptedBlock string
// Find the encrypted block as part of the output
// Most of the time we get 2 devices, the parent and the child but we don't want to guess
// which one is the child by looking at the index, instead the iterate over the list
// Our encrypted device **always** have "-dmcrypt" in the name.
for _, device := range childDevice {
if strings.Contains(device, "-dmcrypt") {
encryptedBlock = device
break
if block != "" {
if isCephEncryptedBlock(context, cephfsid, block) {
childDevice, err := sys.ListDevicesChild(context.Executor, block)
if err != nil {
return nil, err
}
}
if encryptedBlock == "" {
// The encrypted block is not opened, this is an extreme corner case
// The OSD deployment has been removed manually AND the node rebooted
// So we need to re-open the block to re-hydrate the OSDInfo.
//
// Handling this case would mean, writing the encryption key on a temporary file, then call
// luksOpen to open the encrypted block and then call ceph-volume to list against the opened
// encrypted block.
// We don't implement this, yet and return an error.
return nil, errors.Errorf("failed to find the encrypted block device for %q, not opened?", block)
}
// If we have one child device, it should be the encrypted block but still verifying it
isDeviceEncrypted, err := sys.IsDeviceEncrypted(context.Executor, encryptedBlock)
if err != nil {
return nil, err
}
var encryptedBlock string
// Find the encrypted block as part of the output
// Most of the time we get 2 devices, the parent and the child but we don't want to guess
// which one is the child by looking at the index, instead the iterate over the list
// Our encrypted device **always** have "-dmcrypt" in the name.
for _, device := range childDevice {
if strings.Contains(device, "-dmcrypt") {
encryptedBlock = device
break
}
}
if encryptedBlock == "" {
// The encrypted block is not opened, this is an extreme corner case
// The OSD deployment has been removed manually AND the node rebooted
// So we need to re-open the block to re-hydrate the OSDInfo.
//
// Handling this case would mean, writing the encryption key on a temporary file, then call
// luksOpen to open the encrypted block and then call ceph-volume to list against the opened
// encrypted block.
// We don't implement this, yet and return an error.
return nil, errors.Errorf("failed to find the encrypted block device for %q, not opened?", block)
}
// All good, now we set the right disk for ceph-volume to list against
// The ceph-volume will look like:
// [root@bde85e6b23ec /]# ceph-volume raw list /dev/mapper/ocs-deviceset-thin-1-data-0hmfgp-block-dmcrypt
// {
// "4": {
// "ceph_fsid": "fea59c09-2d35-4096-bc46-edb0fd39ab86",
// "device": "/dev/mapper/ocs-deviceset-thin-1-data-0hmfgp-block-dmcrypt",
// "osd_id": 4,
// "osd_uuid": "fcff2062-e653-4d42-84e5-e8de639bed4b",
// "type": "bluestore"
// }
// }
if isDeviceEncrypted {
block = encryptedBlock
// If we have one child device, it should be the encrypted block but still verifying it
isDeviceEncrypted, err := sys.IsDeviceEncrypted(context.Executor, encryptedBlock)
if err != nil {
return nil, err
}
// All good, now we set the right disk for ceph-volume to list against
// The ceph-volume will look like:
// [root@bde85e6b23ec /]# ceph-volume raw list /dev/mapper/ocs-deviceset-thin-1-data-0hmfgp-block-dmcrypt
// {
// "4": {
// "ceph_fsid": "fea59c09-2d35-4096-bc46-edb0fd39ab86",
// "device": "/dev/mapper/ocs-deviceset-thin-1-data-0hmfgp-block-dmcrypt",
// "osd_id": 4,
// "osd_uuid": "fcff2062-e653-4d42-84e5-e8de639bed4b",
// "type": "bluestore"
// }
// }
if isDeviceEncrypted {
block = encryptedBlock
}
}
}
@@ -1077,6 +1084,7 @@ func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.Cluster
LVBackedPV: lvBackedPV,
CVMode: cvMode,
Store: "bluestore",
Encrypted: strings.Contains(blockPath, "-dmcrypt"),
}
if !skipDeviceClass {
@@ -1089,6 +1097,15 @@ func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.Cluster
}
// If this is an encrypted OSD
// Always rely on the env variable and NOT what we find as a block name, this function is
// called by:
// * the prepare job, which pass the env variable for encryption or not
// * the cleanup job which lists **all** devices and cleans them up
// They do different things, in the case of the prepare job we want to close the encrypted
// device because the device is going to be detached from the pod and re-attached to the OSD
// pod
// For the cleanup pod we don't want to close the encrypted block since it will sanitize it
// first and then close it
if os.Getenv(oposd.CephVolumeEncryptedKeyEnvVarName) != "" {
// If label and subsystem are not set on the encrypted block let's set it
// They will be set if the OSD deployment has been removed manually and the prepare job
@@ -1104,7 +1121,7 @@ func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.Cluster
}
// Close encrypted device
err = closeEncryptedDevice(context, block)
err = CloseEncryptedDevice(context, block)
if err != nil {
return nil, errors.Wrapf(err, "failed to close encrypted device %q for osd %d", block, osdID)
}
@@ -1112,7 +1129,7 @@ func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.Cluster
// If there is a metadata block
if metadataBlock != "" {
// Close encrypted device
err = closeEncryptedDevice(context, metadataBlock)
err = CloseEncryptedDevice(context, metadataBlock)
if err != nil {
return nil, errors.Wrapf(err, "failed to close encrypted db device %q for osd %d", metadataBlock, osdID)
}
@@ -1121,7 +1138,7 @@ func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.Cluster
// If there is a wal block
if walBlock != "" {
// Close encrypted device
err = closeEncryptedDevice(context, walBlock)
err = CloseEncryptedDevice(context, walBlock)
if err != nil {
return nil, errors.Wrapf(err, "failed to close encrypted wal device %q for osd %d", walBlock, osdID)
}
pkg/operator/ceph/cluster/osd/osd.go
+1
View File
@@ -108,6 +108,7 @@ type OSDInfo struct {
Store string `json:"store"`
// Ensure the OSD daemon has affinity with the same topology from the OSD prepare pod
TopologyAffinity string `json:"topologyAffinity"`
Encrypted bool `json:"encrypted"`
}
// OrchestrationStatus represents the status of an OSD orchestration
tests/scripts/github-action-helper.sh
+10
View File
@@ -250,6 +250,16 @@ function wait_for_prepare_pod() {
kubectl -n rook-ceph describe deployment/rook-ceph-osd-0 || true
}
function wait_for_cleanup_pod() {
timeout 180 bash <<EOF
until kubectl --namespace rook-ceph logs job/cluster-cleanup-job-$(uname -n); do
echo "waiting for cleanup up pod to be present"
sleep 1
done
EOF
kubectl --namespace rook-ceph logs --follow job/cluster-cleanup-job-"$(uname -n)"
}
function wait_for_ceph_to_be_ready() {
DAEMONS=$1
OSD_COUNT=$2