ceph: fix operator reconcile to restart osds daemons
OSD on PVC does not upgrade when the user upgrades the ceph version on cluster-on-pvc yaml. The solution incudes: - upgrade osd prepare and daemon pods on upgrade - skip c-v prepare if filesystem is already present on the pvc device. - skip lvm release in case of upgrade. Signed-off-by: Santosh Pillai <sapillai@redhat.com>
This commit is contained in:
1 parent
b7d8338d0c
commit
8ea693a740
7 files changed
+51
-42
No files matched your search
@@ -166,10 +166,19 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs
|
||||
}
|
||||
|
||||
var osds []oposd.OSDInfo
|
||||
var lvPath string
|
||||
var skipLVRelease bool
|
||||
if devices == nil || len(devices.Entries) == 0 {
|
||||
logger.Infof("no more devices to configure")
|
||||
if cvSupported {
|
||||
return getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, "")
|
||||
if a.pvcBacked {
|
||||
lvPath, err = getDeviceLVPath(context, fmt.Sprintf("/mnt/%s", a.nodeName))
|
||||
if err != nil {
|
||||
return osds, fmt.Errorf("failed to get logical volume path. %+v", err)
|
||||
}
|
||||
skipLVRelease = true
|
||||
}
|
||||
return getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, lvPath, skipLVRelease)
|
||||
}
|
||||
return osds, nil
|
||||
}
|
||||
@@ -218,6 +227,19 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs
|
||||
return osds, nil
|
||||
}
|
||||
|
||||
func getDeviceLVPath(context *clusterd.Context, deviceName string) (string, error) {
|
||||
cmd := fmt.Sprintf("get logical volume path for device")
|
||||
output, err := context.Executor.ExecuteCommandWithOutput(false, cmd, "pvdisplay", "-C", "-o", "lvpath", "--noheadings", deviceName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to retrieve logical volume path. %+v", err)
|
||||
}
|
||||
logger.Debugf("logical volume path for device %q is %q", deviceName, output)
|
||||
if output == "" {
|
||||
return "", fmt.Errorf("no logical volume path found for device %q", deviceName)
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func (a *OsdAgent) removeDevices(context *clusterd.Context, removedDevicesScheme *config.PerfScheme) error {
|
||||
if removedDevicesScheme == nil || len(removedDevicesScheme.Entries) == 0 {
|
||||
return nil
|
||||
|
||||
@@ -284,7 +284,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent) error {
|
||||
|
||||
logger.Infof("device osds:%+v\ndir osds: %+v", deviceOSDs, dirOSDs)
|
||||
|
||||
if agent.pvcBacked {
|
||||
if agent.pvcBacked && !deviceOSDs[0].SkipLVRelease {
|
||||
volumeGroupName, err := getVolumeGroupName(deviceOSDs[0].LVPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error fetching volume group name. %+v", err)
|
||||
|
||||
@@ -58,7 +58,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device
|
||||
var err error
|
||||
if len(devices.Entries) == 0 {
|
||||
logger.Infof("no new devices to configure. returning devices already configured with ceph-volume.")
|
||||
osds, err = getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, lv)
|
||||
osds, err = getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, lv, false)
|
||||
if err != nil {
|
||||
logger.Infof("failed to get devices already provisioned by ceph-volume. %+v", err)
|
||||
}
|
||||
@@ -83,7 +83,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device
|
||||
}
|
||||
}
|
||||
|
||||
osds, err = getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, lv)
|
||||
osds, err = getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID, lv, false)
|
||||
return osds, err
|
||||
}
|
||||
|
||||
@@ -378,7 +378,7 @@ func getCephVolumeSupported(context *clusterd.Context) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func getCephVolumeOSDs(context *clusterd.Context, clusterName string, cephfsid string, lv string) ([]oposd.OSDInfo, error) {
|
||||
func getCephVolumeOSDs(context *clusterd.Context, clusterName string, cephfsid string, lv string, skipLVRelease bool) ([]oposd.OSDInfo, error) {
|
||||
|
||||
result, err := context.Executor.ExecuteCommandWithCombinedOutput(false, "", cephVolumeCmd, "lvm", "list", lv, "--format", "json")
|
||||
if err != nil {
|
||||
@@ -428,6 +428,7 @@ func getCephVolumeOSDs(context *clusterd.Context, clusterName string, cephfsid s
|
||||
CephVolumeInitiated: true,
|
||||
IsFileStore: isFilestore,
|
||||
LVPath: lv,
|
||||
SkipLVRelease: skipLVRelease,
|
||||
}
|
||||
osds = append(osds, osd)
|
||||
}
|
||||
|
||||
@@ -252,7 +252,7 @@ func TestParseCephVolumeResult(t *testing.T) {
|
||||
}
|
||||
|
||||
context := &clusterd.Context{Executor: executor}
|
||||
osds, err := getCephVolumeOSDs(context, "rook", "4bfe8b72-5e69-4330-b6c0-4d914db8ab89", "")
|
||||
osds, err := getCephVolumeOSDs(context, "rook", "4bfe8b72-5e69-4330-b6c0-4d914db8ab89", "", false)
|
||||
assert.Nil(t, err)
|
||||
require.NotNil(t, osds)
|
||||
assert.Equal(t, 2, len(osds))
|
||||
@@ -272,7 +272,7 @@ func TestCephVolumeResultMultiClusterSingleOSD(t *testing.T) {
|
||||
}
|
||||
|
||||
context := &clusterd.Context{Executor: executor}
|
||||
osds, err := getCephVolumeOSDs(context, "rook", "451267e6-883f-4936-8dff-080d781c67d5", "")
|
||||
osds, err := getCephVolumeOSDs(context, "rook", "451267e6-883f-4936-8dff-080d781c67d5", "", false)
|
||||
assert.Nil(t, err)
|
||||
require.NotNil(t, osds)
|
||||
assert.Equal(t, 1, len(osds))
|
||||
@@ -293,7 +293,7 @@ func TestCephVolumeResultMultiClusterMultiOSD(t *testing.T) {
|
||||
}
|
||||
|
||||
context := &clusterd.Context{Executor: executor}
|
||||
osds, err := getCephVolumeOSDs(context, "rook", "451267e6-883f-4936-8dff-080d781c67d5", "")
|
||||
osds, err := getCephVolumeOSDs(context, "rook", "451267e6-883f-4936-8dff-080d781c67d5", "", false)
|
||||
assert.Nil(t, err)
|
||||
require.NotNil(t, osds)
|
||||
assert.Equal(t, 1, len(osds))
|
||||
|
||||
@@ -23,7 +23,6 @@ import (
|
||||
opspec "github.com/rook/rook/pkg/operator/ceph/spec"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
func (c *Cluster) prepareStorageClassDeviceSets(config *provisionConfig) []rookalpha.VolumeSource {
|
||||
@@ -82,27 +81,6 @@ func (c *Cluster) createStorageClassDeviceSetPVC(storageClassDeviceSet rookalpha
|
||||
return nil, fmt.Errorf("more than one PVCs exists with label %v, pvcs %+v", pvcStorageClassDeviceSetPVCIdLabelSelector, presentPVCs)
|
||||
}
|
||||
|
||||
// GetValidVolumeSources returns all volumes that has an OSDDeployment
|
||||
func GetValidVolumeSources(clientset kubernetes.Interface, namespace string, volumeSources []rookalpha.VolumeSource) (validVolumes []rookalpha.VolumeSource, err error) {
|
||||
if clientset == nil {
|
||||
return validVolumes, fmt.Errorf("nil clientset received")
|
||||
}
|
||||
for _, vol := range volumeSources {
|
||||
osdDeployments, err := clientset.AppsV1().Deployments(namespace).List(metav1.ListOptions{LabelSelector: OSDOverPVCLabelKey + "=" + vol.PersistentVolumeClaimSource.ClaimName})
|
||||
if err != nil {
|
||||
return validVolumes, err
|
||||
}
|
||||
osdPrepareJob, err := clientset.BatchV1().Jobs(namespace).List(metav1.ListOptions{LabelSelector: OSDOverPVCLabelKey + "=" + vol.PersistentVolumeClaimSource.ClaimName})
|
||||
if err != nil {
|
||||
return validVolumes, err
|
||||
}
|
||||
if len(osdDeployments.Items) == 0 && len(osdPrepareJob.Items) == 0 {
|
||||
validVolumes = append(validVolumes, vol)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func makeStorageClassDeviceSetPVC(storageClassDeviceSetName, pvcStorageClassDeviceSetPVCId string, pvcIndex, setIndex int, pvcTemplate v1.PersistentVolumeClaim) (pvcs *v1.PersistentVolumeClaim) {
|
||||
pvcLabels := makeStorageClassDeviceSetPVCLabel(storageClassDeviceSetName, pvcStorageClassDeviceSetPVCId, pvcIndex, setIndex)
|
||||
|
||||
|
||||
@@ -138,7 +138,8 @@ type OSDInfo struct {
|
||||
DevicePartUUID string `json:"device-part-uuid"`
|
||||
CephVolumeInitiated bool `json:"ceph-volume-initiated"`
|
||||
//LVPath is the logical Volume path for an OSD created by Ceph-volume with format '/dev/<Volume Group>/<Logical Volume>'
|
||||
LVPath string `json:"lv-path"`
|
||||
LVPath string `json:"lv-path"`
|
||||
SkipLVRelease bool `json:"skip-lv-release"`
|
||||
}
|
||||
|
||||
type OrchestrationStatus struct {
|
||||
@@ -227,22 +228,14 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) {
|
||||
// Parsing storageClassDeviceSets and parsing it to volume sources
|
||||
c.DesiredStorage.VolumeSources = append(c.DesiredStorage.VolumeSources, c.prepareStorageClassDeviceSets(config)...)
|
||||
|
||||
// Filtering out valid volume sources
|
||||
validVolumeSources, err := GetValidVolumeSources(c.context.Clientset, c.Namespace, c.DesiredStorage.VolumeSources)
|
||||
if err != nil {
|
||||
config.addError(fmt.Sprintf("%v", err))
|
||||
return
|
||||
}
|
||||
c.ValidStorage.VolumeSources = validVolumeSources
|
||||
c.ValidStorage.VolumeSources = c.DesiredStorage.VolumeSources
|
||||
|
||||
// no validVolumeSource is ready to run an osd
|
||||
if len(c.DesiredStorage.VolumeSources) == 0 && len(c.DesiredStorage.StorageClassDeviceSets) == 0 {
|
||||
logger.Warningf("no volume sources specified for creating OSDs on PVCs.")
|
||||
logger.Info("no volume sources defined to configure OSDs on PVCs.")
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof("%d of the %d volumeSources are valid", len(validVolumeSources), len(c.DesiredStorage.VolumeSources))
|
||||
|
||||
//check k8s version
|
||||
k8sVersion, err := k8sutil.GetK8SVersion(c.context.Clientset)
|
||||
if err != nil {
|
||||
|
||||
+16
-1
@@ -342,11 +342,26 @@ func CheckIfDeviceAvailable(executor exec.Executor, name string, pvcBacked bool)
|
||||
return 0, false, "", fmt.Errorf("failed to get device %s filesystem: %+v", name, err)
|
||||
}
|
||||
} else {
|
||||
devFS = "" //Not checking Filesystem in case of PVC block device.
|
||||
devFS, err = GetPVCDeviceFileSystems(executor, name)
|
||||
if err != nil {
|
||||
return 0, false, "", fmt.Errorf("failed to get pvc device %q filesystem. %+v", name, err)
|
||||
}
|
||||
}
|
||||
return partCount, ownPartitions, devFS, nil
|
||||
}
|
||||
|
||||
//GetPVCDeviceFileSystems returns the file system on a PVC device.
|
||||
func GetPVCDeviceFileSystems(executor exec.Executor, device string) (string, error) {
|
||||
cmd := fmt.Sprintf("get pvc filesystem type for %q", device)
|
||||
output, err := executor.ExecuteCommandWithOutput(false, cmd, "lsblk", device, "--bytes", "--nodeps", "--noheadings", "--output", "FSTYPE")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("command %q failed. %+v", cmd, err)
|
||||
}
|
||||
logger.Debugf("filesystem on pvc device %q is %q", device, output)
|
||||
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// RookOwnsPartitions check if all partitions in list are owned by Rook
|
||||
func RookOwnsPartitions(partitions []Partition) bool {
|
||||
// if there are partitions, they must all have the rook osd label
|
||||
|
||||
Reference in new issue
Block a user