If an OSD is down and needs to be removed from the cluster, a job can be run that will remove the OSD deployment and purge the OSD from ceph. If the OSD is still up, the purge will be rejected. To purge multiple OSDs at the same time, the OSD IDs can be specified as a comma-separated list. Signed-off-by: Servesha Dudhgaonkar <sdudhgao@redhat.com>
138 lines
5.1 KiB
Go
138 lines
5.1 KiB
Go
/*
|
|
Copyright 2020 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package osd
|
|
|
|
import (
|
|
"fmt"
|
|
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"strconv"
|
|
|
|
"github.com/pkg/errors"
|
|
"github.com/rook/rook/pkg/clusterd"
|
|
"github.com/rook/rook/pkg/daemon/ceph/client"
|
|
"github.com/rook/rook/pkg/operator/k8sutil"
|
|
)
|
|
|
|
// RemoveOSDs purges a list of OSDs from the cluster
|
|
func RemoveOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, osdsToRemove []string) error {
|
|
|
|
// Generate the ceph config for running ceph commands similar to the operator
|
|
if err := writeCephConfig(context, clusterInfo); err != nil {
|
|
return errors.Wrap(err, "failed to write the ceph config")
|
|
}
|
|
|
|
osdDump, err := client.GetOSDDump(context, clusterInfo)
|
|
if err != nil {
|
|
return errors.Wrap(err, "failed to get osd dump")
|
|
}
|
|
|
|
for _, osdIDStr := range osdsToRemove {
|
|
osdID, err := strconv.Atoi(osdIDStr)
|
|
if err != nil {
|
|
logger.Errorf("invalid OSD ID: %s. %v", osdIDStr, err)
|
|
continue
|
|
}
|
|
logger.Debugf("validating status of osd.%d", osdID)
|
|
status, _, err := osdDump.StatusByID(int64(osdID))
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to get osd status for osd %d", osdID)
|
|
|
|
}
|
|
const upStatus int64 = 1
|
|
if status == upStatus {
|
|
logger.Debugf("osd.%d is healthy. It cannot be removed unless it is 'down'", osdID)
|
|
continue
|
|
}
|
|
logger.Debugf("osd.%d is marked 'DOWN'. Removing it", osdID)
|
|
removeOSD(context, clusterInfo, osdID)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func removeOSD(context *clusterd.Context, clusterInfo *client.ClusterInfo, osdID int) {
|
|
// Get the host where the OSD is found
|
|
hostName, err := client.GetCrushHostName(context, clusterInfo, osdID)
|
|
if err != nil {
|
|
logger.Errorf("failed to get the host where osd.%d is running. %v", osdID, err)
|
|
}
|
|
|
|
// Mark the OSD as out.
|
|
args := []string{"osd", "out", fmt.Sprintf("osd.%d", osdID)}
|
|
_, err = client.NewCephCommand(context, clusterInfo, args).Run()
|
|
if err != nil {
|
|
logger.Errorf("failed to exclude osd.%d out of the crush map. %v", osdID, err)
|
|
}
|
|
|
|
// Remove the OSD deployment
|
|
deploymentName := fmt.Sprintf("rook-ceph-osd-%d", osdID)
|
|
deployment, err := context.Clientset.AppsV1().Deployments(clusterInfo.Namespace).Get(deploymentName, metav1.GetOptions{})
|
|
if err != nil {
|
|
logger.Errorf("failed to fetch the deployment %q. %v", deploymentName, err)
|
|
} else {
|
|
logger.Infof("removing the OSD deployment %q", deploymentName)
|
|
if err := k8sutil.DeleteDeployment(context.Clientset, clusterInfo.Namespace, deploymentName); err != nil {
|
|
if err != nil {
|
|
// Continue purging the OSD even if the deployment fails to be deleted
|
|
logger.Errorf("failed to delete deployment for OSD %d. %v", osdID, err)
|
|
}
|
|
}
|
|
const pvcLabelName = "ceph.rook.io/pvc"
|
|
if pvcName, ok := deployment.GetLabels()[pvcLabelName]; ok {
|
|
prepareJobList, err := context.Clientset.BatchV1().Jobs(clusterInfo.Namespace).List(metav1.ListOptions{LabelSelector: pvcLabelName + pvcName})
|
|
if err != nil && !kerrors.IsNotFound(err) {
|
|
logger.Errorf("failed to list prepareJobs with labels %q. %v ", pvcLabelName+pvcName, err)
|
|
}
|
|
// Remove osd prepare job
|
|
for _, prepareJob := range prepareJobList.Items {
|
|
logger.Infof("removing the osd prepare job %q", prepareJob.GetName())
|
|
if err := k8sutil.DeleteBatchJob(context.Clientset, clusterInfo.Namespace, prepareJob.GetName(), false); err != nil {
|
|
if err != nil {
|
|
// Continue deleting the OSD prepare job even if the deployment fails to be deleted
|
|
logger.Errorf("failed to delete prepare job for osd %q. %v", prepareJob.GetName(), err)
|
|
}
|
|
}
|
|
|
|
logger.Infof("removing the OSD PVC %q", pvcName)
|
|
if err := context.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Delete(pvcName, &metav1.DeleteOptions{}); err != nil {
|
|
if err != nil {
|
|
// Continue deleting the OSD PVC even if PVC deletion fails
|
|
logger.Errorf("failed to delete pvc for OSD %q. %v", pvcName, err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// purge the osd
|
|
purgeosdargs := []string{"osd", "purge", fmt.Sprintf("osd.%d", osdID), "--force", "--yes-i-really-mean-it"}
|
|
_, err = client.NewCephCommand(context, clusterInfo, purgeosdargs).Run()
|
|
if err != nil {
|
|
logger.Errorf("failed to purge osd.%d. %v", osdID, err)
|
|
}
|
|
|
|
// Attempting to remove the parent host. Errors can be ignored if there are other OSDs on the same host
|
|
hostargs := []string{"osd", "crush", "rm", hostName}
|
|
_, err = client.NewCephCommand(context, clusterInfo, hostargs).Run()
|
|
if err != nil {
|
|
logger.Errorf("failed to remove CRUSH host %q. %v", hostName, err)
|
|
}
|
|
|
|
logger.Infof("completed removal of OSD %d", osdID)
|
|
}
|