From 8ea693a74011c587970dfc28a3d9efe2ef329159 Mon Sep 17 00:00:00 2001 From: Santosh Pillai Date: Thu, 26 Sep 2019 20:43:36 +0530 Subject: [PATCH] ceph: fix operator reconcile to restart osds daemons OSD on PVC does not upgrade when the user upgrades the ceph version on cluster-on-pvc yaml. The solution incudes: - upgrade osd prepare and daemon pods on upgrade - skip c-v prepare if filesystem is already present on the pvc device. - skip lvm release in case of upgrade. Signed-off-by: Santosh Pillai --- pkg/daemon/ceph/osd/agent.go | 24 +++++++++++++++++++++- pkg/daemon/ceph/osd/daemon.go | 2 +- pkg/daemon/ceph/osd/volume.go | 7 ++++--- pkg/daemon/ceph/osd/volume_test.go | 6 +++--- pkg/operator/ceph/cluster/osd/deviceSet.go | 22 -------------------- pkg/operator/ceph/cluster/osd/osd.go | 15 ++++---------- pkg/util/sys/device.go | 17 ++++++++++++++- 7 files changed, 51 insertions(+), 42 deletions(-) diff --git a/pkg/daemon/ceph/osd/agent.go b/pkg/daemon/ceph/osd/agent.go index 121da5df9..cc0b91fa4 100644 --- a/pkg/daemon/ceph/osd/agent.go +++ b/pkg/daemon/ceph/osd/agent.go @@ -166,10 +166,19 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs } var osds []oposd.OSDInfo + var lvPath string + var skipLVRelease bool if devices == nil || len(devices.Entries) == 0 { logger.Infof("no more devices to configure") if cvSupported { - return getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, "") + if a.pvcBacked { + lvPath, err = getDeviceLVPath(context, fmt.Sprintf("/mnt/%s", a.nodeName)) + if err != nil { + return osds, fmt.Errorf("failed to get logical volume path. %+v", err) + } + skipLVRelease = true + } + return getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, lvPath, skipLVRelease) } return osds, nil } @@ -218,6 +227,19 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs return osds, nil } +func getDeviceLVPath(context *clusterd.Context, deviceName string) (string, error) { + cmd := fmt.Sprintf("get logical volume path for device") + output, err := context.Executor.ExecuteCommandWithOutput(false, cmd, "pvdisplay", "-C", "-o", "lvpath", "--noheadings", deviceName) + if err != nil { + return "", fmt.Errorf("failed to retrieve logical volume path. %+v", err) + } + logger.Debugf("logical volume path for device %q is %q", deviceName, output) + if output == "" { + return "", fmt.Errorf("no logical volume path found for device %q", deviceName) + } + return output, nil +} + func (a *OsdAgent) removeDevices(context *clusterd.Context, removedDevicesScheme *config.PerfScheme) error { if removedDevicesScheme == nil || len(removedDevicesScheme.Entries) == 0 { return nil diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index f07458b8e..4761e915f 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -284,7 +284,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent) error { logger.Infof("device osds:%+v\ndir osds: %+v", deviceOSDs, dirOSDs) - if agent.pvcBacked { + if agent.pvcBacked && !deviceOSDs[0].SkipLVRelease { volumeGroupName, err := getVolumeGroupName(deviceOSDs[0].LVPath) if err != nil { return fmt.Errorf("error fetching volume group name. %+v", err) diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index eae45be76..7b102a92d 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -58,7 +58,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device var err error if len(devices.Entries) == 0 { logger.Infof("no new devices to configure. returning devices already configured with ceph-volume.") - osds, err = getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, lv) + osds, err = getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, lv, false) if err != nil { logger.Infof("failed to get devices already provisioned by ceph-volume. %+v", err) } @@ -83,7 +83,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device } } - osds, err = getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, lv) + osds, err = getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, lv, false) return osds, err } @@ -378,7 +378,7 @@ func getCephVolumeSupported(context *clusterd.Context) (bool, error) { return true, nil } -func getCephVolumeOSDs(context *clusterd.Context, clusterName string, cephfsid string, lv string) ([]oposd.OSDInfo, error) { +func getCephVolumeOSDs(context *clusterd.Context, clusterName string, cephfsid string, lv string, skipLVRelease bool) ([]oposd.OSDInfo, error) { result, err := context.Executor.ExecuteCommandWithCombinedOutput(false, "", cephVolumeCmd, "lvm", "list", lv, "--format", "json") if err != nil { @@ -428,6 +428,7 @@ func getCephVolumeOSDs(context *clusterd.Context, clusterName string, cephfsid s CephVolumeInitiated: true, IsFileStore: isFilestore, LVPath: lv, + SkipLVRelease: skipLVRelease, } osds = append(osds, osd) } diff --git a/pkg/daemon/ceph/osd/volume_test.go b/pkg/daemon/ceph/osd/volume_test.go index 6b1d67a08..e9c8b3412 100644 --- a/pkg/daemon/ceph/osd/volume_test.go +++ b/pkg/daemon/ceph/osd/volume_test.go @@ -252,7 +252,7 @@ func TestParseCephVolumeResult(t *testing.T) { } context := &clusterd.Context{Executor: executor} - osds, err := getCephVolumeOSDs(context, "rook", "4bfe8b72-5e69-4330-b6c0-4d914db8ab89", "") + osds, err := getCephVolumeOSDs(context, "rook", "4bfe8b72-5e69-4330-b6c0-4d914db8ab89", "", false) assert.Nil(t, err) require.NotNil(t, osds) assert.Equal(t, 2, len(osds)) @@ -272,7 +272,7 @@ func TestCephVolumeResultMultiClusterSingleOSD(t *testing.T) { } context := &clusterd.Context{Executor: executor} - osds, err := getCephVolumeOSDs(context, "rook", "451267e6-883f-4936-8dff-080d781c67d5", "") + osds, err := getCephVolumeOSDs(context, "rook", "451267e6-883f-4936-8dff-080d781c67d5", "", false) assert.Nil(t, err) require.NotNil(t, osds) assert.Equal(t, 1, len(osds)) @@ -293,7 +293,7 @@ func TestCephVolumeResultMultiClusterMultiOSD(t *testing.T) { } context := &clusterd.Context{Executor: executor} - osds, err := getCephVolumeOSDs(context, "rook", "451267e6-883f-4936-8dff-080d781c67d5", "") + osds, err := getCephVolumeOSDs(context, "rook", "451267e6-883f-4936-8dff-080d781c67d5", "", false) assert.Nil(t, err) require.NotNil(t, osds) assert.Equal(t, 1, len(osds)) diff --git a/pkg/operator/ceph/cluster/osd/deviceSet.go b/pkg/operator/ceph/cluster/osd/deviceSet.go index b7fab38a5..7b474e18a 100644 --- a/pkg/operator/ceph/cluster/osd/deviceSet.go +++ b/pkg/operator/ceph/cluster/osd/deviceSet.go @@ -23,7 +23,6 @@ import ( opspec "github.com/rook/rook/pkg/operator/ceph/spec" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" ) func (c *Cluster) prepareStorageClassDeviceSets(config *provisionConfig) []rookalpha.VolumeSource { @@ -82,27 +81,6 @@ func (c *Cluster) createStorageClassDeviceSetPVC(storageClassDeviceSet rookalpha return nil, fmt.Errorf("more than one PVCs exists with label %v, pvcs %+v", pvcStorageClassDeviceSetPVCIdLabelSelector, presentPVCs) } -// GetValidVolumeSources returns all volumes that has an OSDDeployment -func GetValidVolumeSources(clientset kubernetes.Interface, namespace string, volumeSources []rookalpha.VolumeSource) (validVolumes []rookalpha.VolumeSource, err error) { - if clientset == nil { - return validVolumes, fmt.Errorf("nil clientset received") - } - for _, vol := range volumeSources { - osdDeployments, err := clientset.AppsV1().Deployments(namespace).List(metav1.ListOptions{LabelSelector: OSDOverPVCLabelKey + "=" + vol.PersistentVolumeClaimSource.ClaimName}) - if err != nil { - return validVolumes, err - } - osdPrepareJob, err := clientset.BatchV1().Jobs(namespace).List(metav1.ListOptions{LabelSelector: OSDOverPVCLabelKey + "=" + vol.PersistentVolumeClaimSource.ClaimName}) - if err != nil { - return validVolumes, err - } - if len(osdDeployments.Items) == 0 && len(osdPrepareJob.Items) == 0 { - validVolumes = append(validVolumes, vol) - } - } - return -} - func makeStorageClassDeviceSetPVC(storageClassDeviceSetName, pvcStorageClassDeviceSetPVCId string, pvcIndex, setIndex int, pvcTemplate v1.PersistentVolumeClaim) (pvcs *v1.PersistentVolumeClaim) { pvcLabels := makeStorageClassDeviceSetPVCLabel(storageClassDeviceSetName, pvcStorageClassDeviceSetPVCId, pvcIndex, setIndex) diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 14bf7e8ec..d0a3ecd65 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -138,7 +138,8 @@ type OSDInfo struct { DevicePartUUID string `json:"device-part-uuid"` CephVolumeInitiated bool `json:"ceph-volume-initiated"` //LVPath is the logical Volume path for an OSD created by Ceph-volume with format '/dev//' - LVPath string `json:"lv-path"` + LVPath string `json:"lv-path"` + SkipLVRelease bool `json:"skip-lv-release"` } type OrchestrationStatus struct { @@ -227,22 +228,14 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) { // Parsing storageClassDeviceSets and parsing it to volume sources c.DesiredStorage.VolumeSources = append(c.DesiredStorage.VolumeSources, c.prepareStorageClassDeviceSets(config)...) - // Filtering out valid volume sources - validVolumeSources, err := GetValidVolumeSources(c.context.Clientset, c.Namespace, c.DesiredStorage.VolumeSources) - if err != nil { - config.addError(fmt.Sprintf("%v", err)) - return - } - c.ValidStorage.VolumeSources = validVolumeSources + c.ValidStorage.VolumeSources = c.DesiredStorage.VolumeSources // no validVolumeSource is ready to run an osd if len(c.DesiredStorage.VolumeSources) == 0 && len(c.DesiredStorage.StorageClassDeviceSets) == 0 { - logger.Warningf("no volume sources specified for creating OSDs on PVCs.") + logger.Info("no volume sources defined to configure OSDs on PVCs.") return } - logger.Infof("%d of the %d volumeSources are valid", len(validVolumeSources), len(c.DesiredStorage.VolumeSources)) - //check k8s version k8sVersion, err := k8sutil.GetK8SVersion(c.context.Clientset) if err != nil { diff --git a/pkg/util/sys/device.go b/pkg/util/sys/device.go index e36010a21..4c7857dca 100644 --- a/pkg/util/sys/device.go +++ b/pkg/util/sys/device.go @@ -342,11 +342,26 @@ func CheckIfDeviceAvailable(executor exec.Executor, name string, pvcBacked bool) return 0, false, "", fmt.Errorf("failed to get device %s filesystem: %+v", name, err) } } else { - devFS = "" //Not checking Filesystem in case of PVC block device. + devFS, err = GetPVCDeviceFileSystems(executor, name) + if err != nil { + return 0, false, "", fmt.Errorf("failed to get pvc device %q filesystem. %+v", name, err) + } } return partCount, ownPartitions, devFS, nil } +//GetPVCDeviceFileSystems returns the file system on a PVC device. +func GetPVCDeviceFileSystems(executor exec.Executor, device string) (string, error) { + cmd := fmt.Sprintf("get pvc filesystem type for %q", device) + output, err := executor.ExecuteCommandWithOutput(false, cmd, "lsblk", device, "--bytes", "--nodeps", "--noheadings", "--output", "FSTYPE") + if err != nil { + return "", fmt.Errorf("command %q failed. %+v", cmd, err) + } + logger.Debugf("filesystem on pvc device %q is %q", device, output) + + return output, nil +} + // RookOwnsPartitions check if all partitions in list are owned by Rook func RookOwnsPartitions(partitions []Partition) bool { // if there are partitions, they must all have the rook osd label