ceph: fix operator reconcile to restart osds daemons

OSD on PVC does not upgrade when the user upgrades the ceph version on cluster-on-pvc yaml.
The solution incudes:
   - upgrade osd prepare and daemon pods on upgrade
   - skip c-v prepare if filesystem is already present on the pvc device.
   - skip lvm release in case of upgrade.

Signed-off-by: Santosh Pillai <sapillai@redhat.com>
This commit is contained in:
Santosh Pillai committed 2019-10-23 16:56:15 +05:30
1 parent b7d8338d0c
commit 8ea693a740
7 files changed
+51 -42

No files matched your search

+23 -1
View File
@@ -166,10 +166,19 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs
}
var osds []oposd.OSDInfo
var lvPath string
var skipLVRelease bool
if devices == nil || len(devices.Entries) == 0 {
logger.Infof("no more devices to configure")
if cvSupported {
return getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, "")
if a.pvcBacked {
lvPath, err = getDeviceLVPath(context, fmt.Sprintf("/mnt/%s", a.nodeName))
if err != nil {
return osds, fmt.Errorf("failed to get logical volume path. %+v", err)
}
skipLVRelease = true
}
return getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, lvPath, skipLVRelease)
}
return osds, nil
}
@@ -218,6 +227,19 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs
return osds, nil
}
func getDeviceLVPath(context *clusterd.Context, deviceName string) (string, error) {
cmd := fmt.Sprintf("get logical volume path for device")
output, err := context.Executor.ExecuteCommandWithOutput(false, cmd, "pvdisplay", "-C", "-o", "lvpath", "--noheadings", deviceName)
if err != nil {
return "", fmt.Errorf("failed to retrieve logical volume path. %+v", err)
}
logger.Debugf("logical volume path for device %q is %q", deviceName, output)
if output == "" {
return "", fmt.Errorf("no logical volume path found for device %q", deviceName)
}
return output, nil
}
func (a *OsdAgent) removeDevices(context *clusterd.Context, removedDevicesScheme *config.PerfScheme) error {
if removedDevicesScheme == nil || len(removedDevicesScheme.Entries) == 0 {
return nil
+1 -1
View File
@@ -284,7 +284,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent) error {
logger.Infof("device osds:%+v\ndir osds: %+v", deviceOSDs, dirOSDs)
if agent.pvcBacked {
if agent.pvcBacked && !deviceOSDs[0].SkipLVRelease {
volumeGroupName, err := getVolumeGroupName(deviceOSDs[0].LVPath)
if err != nil {
return fmt.Errorf("error fetching volume group name. %+v", err)
+4 -3
View File
@@ -58,7 +58,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device
var err error
if len(devices.Entries) == 0 {
logger.Infof("no new devices to configure. returning devices already configured with ceph-volume.")
osds, err = getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, lv)
osds, err = getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, lv, false)
if err != nil {
logger.Infof("failed to get devices already provisioned by ceph-volume. %+v", err)
}
@@ -83,7 +83,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device
}
}
osds, err = getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, lv)
osds, err = getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, lv, false)
return osds, err
}
@@ -378,7 +378,7 @@ func getCephVolumeSupported(context *clusterd.Context) (bool, error) {
return true, nil
}
func getCephVolumeOSDs(context *clusterd.Context, clusterName string, cephfsid string, lv string) ([]oposd.OSDInfo, error) {
func getCephVolumeOSDs(context *clusterd.Context, clusterName string, cephfsid string, lv string, skipLVRelease bool) ([]oposd.OSDInfo, error) {
result, err := context.Executor.ExecuteCommandWithCombinedOutput(false, "", cephVolumeCmd, "lvm", "list", lv, "--format", "json")
if err != nil {
@@ -428,6 +428,7 @@ func getCephVolumeOSDs(context *clusterd.Context, clusterName string, cephfsid s
CephVolumeInitiated: true,
IsFileStore: isFilestore,
LVPath: lv,
SkipLVRelease: skipLVRelease,
}
osds = append(osds, osd)
}
+3 -3
View File
@@ -252,7 +252,7 @@ func TestParseCephVolumeResult(t *testing.T) {
}
context := &clusterd.Context{Executor: executor}
osds, err := getCephVolumeOSDs(context, "rook", "4bfe8b72-5e69-4330-b6c0-4d914db8ab89", "")
osds, err := getCephVolumeOSDs(context, "rook", "4bfe8b72-5e69-4330-b6c0-4d914db8ab89", "", false)
assert.Nil(t, err)
require.NotNil(t, osds)
assert.Equal(t, 2, len(osds))
@@ -272,7 +272,7 @@ func TestCephVolumeResultMultiClusterSingleOSD(t *testing.T) {
}
context := &clusterd.Context{Executor: executor}
osds, err := getCephVolumeOSDs(context, "rook", "451267e6-883f-4936-8dff-080d781c67d5", "")
osds, err := getCephVolumeOSDs(context, "rook", "451267e6-883f-4936-8dff-080d781c67d5", "", false)
assert.Nil(t, err)
require.NotNil(t, osds)
assert.Equal(t, 1, len(osds))
@@ -293,7 +293,7 @@ func TestCephVolumeResultMultiClusterMultiOSD(t *testing.T) {
}
context := &clusterd.Context{Executor: executor}
osds, err := getCephVolumeOSDs(context, "rook", "451267e6-883f-4936-8dff-080d781c67d5", "")
osds, err := getCephVolumeOSDs(context, "rook", "451267e6-883f-4936-8dff-080d781c67d5", "", false)
assert.Nil(t, err)
require.NotNil(t, osds)
assert.Equal(t, 1, len(osds))
@@ -23,7 +23,6 @@ import (
opspec "github.com/rook/rook/pkg/operator/ceph/spec"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
func (c *Cluster) prepareStorageClassDeviceSets(config *provisionConfig) []rookalpha.VolumeSource {
@@ -82,27 +81,6 @@ func (c *Cluster) createStorageClassDeviceSetPVC(storageClassDeviceSet rookalpha
return nil, fmt.Errorf("more than one PVCs exists with label %v, pvcs %+v", pvcStorageClassDeviceSetPVCIdLabelSelector, presentPVCs)
}
// GetValidVolumeSources returns all volumes that has an OSDDeployment
func GetValidVolumeSources(clientset kubernetes.Interface, namespace string, volumeSources []rookalpha.VolumeSource) (validVolumes []rookalpha.VolumeSource, err error) {
if clientset == nil {
return validVolumes, fmt.Errorf("nil clientset received")
}
for _, vol := range volumeSources {
osdDeployments, err := clientset.AppsV1().Deployments(namespace).List(metav1.ListOptions{LabelSelector: OSDOverPVCLabelKey + "=" + vol.PersistentVolumeClaimSource.ClaimName})
if err != nil {
return validVolumes, err
}
osdPrepareJob, err := clientset.BatchV1().Jobs(namespace).List(metav1.ListOptions{LabelSelector: OSDOverPVCLabelKey + "=" + vol.PersistentVolumeClaimSource.ClaimName})
if err != nil {
return validVolumes, err
}
if len(osdDeployments.Items) == 0 && len(osdPrepareJob.Items) == 0 {
validVolumes = append(validVolumes, vol)
}
}
return
}
func makeStorageClassDeviceSetPVC(storageClassDeviceSetName, pvcStorageClassDeviceSetPVCId string, pvcIndex, setIndex int, pvcTemplate v1.PersistentVolumeClaim) (pvcs *v1.PersistentVolumeClaim) {
pvcLabels := makeStorageClassDeviceSetPVCLabel(storageClassDeviceSetName, pvcStorageClassDeviceSetPVCId, pvcIndex, setIndex)
+4 -11
View File
@@ -138,7 +138,8 @@ type OSDInfo struct {
DevicePartUUID string `json:"device-part-uuid"`
CephVolumeInitiated bool `json:"ceph-volume-initiated"`
//LVPath is the logical Volume path for an OSD created by Ceph-volume with format '/dev/<Volume Group>/<Logical Volume>'
LVPath string `json:"lv-path"`
LVPath string `json:"lv-path"`
SkipLVRelease bool `json:"skip-lv-release"`
}
type OrchestrationStatus struct {
@@ -227,22 +228,14 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) {
// Parsing storageClassDeviceSets and parsing it to volume sources
c.DesiredStorage.VolumeSources = append(c.DesiredStorage.VolumeSources, c.prepareStorageClassDeviceSets(config)...)
// Filtering out valid volume sources
validVolumeSources, err := GetValidVolumeSources(c.context.Clientset, c.Namespace, c.DesiredStorage.VolumeSources)
if err != nil {
config.addError(fmt.Sprintf("%v", err))
return
}
c.ValidStorage.VolumeSources = validVolumeSources
c.ValidStorage.VolumeSources = c.DesiredStorage.VolumeSources
// no validVolumeSource is ready to run an osd
if len(c.DesiredStorage.VolumeSources) == 0 && len(c.DesiredStorage.StorageClassDeviceSets) == 0 {
logger.Warningf("no volume sources specified for creating OSDs on PVCs.")
logger.Info("no volume sources defined to configure OSDs on PVCs.")
return
}
logger.Infof("%d of the %d volumeSources are valid", len(validVolumeSources), len(c.DesiredStorage.VolumeSources))
//check k8s version
k8sVersion, err := k8sutil.GetK8SVersion(c.context.Clientset)
if err != nil {
+16 -1
View File
@@ -342,11 +342,26 @@ func CheckIfDeviceAvailable(executor exec.Executor, name string, pvcBacked bool)
return 0, false, "", fmt.Errorf("failed to get device %s filesystem: %+v", name, err)
}
} else {
devFS = "" //Not checking Filesystem in case of PVC block device.
devFS, err = GetPVCDeviceFileSystems(executor, name)
if err != nil {
return 0, false, "", fmt.Errorf("failed to get pvc device %q filesystem. %+v", name, err)
}
}
return partCount, ownPartitions, devFS, nil
}
//GetPVCDeviceFileSystems returns the file system on a PVC device.
func GetPVCDeviceFileSystems(executor exec.Executor, device string) (string, error) {
cmd := fmt.Sprintf("get pvc filesystem type for %q", device)
output, err := executor.ExecuteCommandWithOutput(false, cmd, "lsblk", device, "--bytes", "--nodeps", "--noheadings", "--output", "FSTYPE")
if err != nil {
return "", fmt.Errorf("command %q failed. %+v", cmd, err)
}
logger.Debugf("filesystem on pvc device %q is %q", device, output)
return output, nil
}
// RookOwnsPartitions check if all partitions in list are owned by Rook
func RookOwnsPartitions(partitions []Partition) bool {
// if there are partitions, they must all have the rook osd label