my-rook-config/pkg/operator/ceph/cluster/osd/replace.go

/*
Copyright 2023 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package osd for the Ceph OSDs.
package osd

import (
	"encoding/json"

	"github.com/pkg/errors"
	"github.com/rook/rook/pkg/clusterd"
	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
	"github.com/rook/rook/pkg/operator/k8sutil"
	v1 "k8s.io/api/core/v1"
	kerrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	OSDReplaceConfigName       = "osd-replace-config"
	OSDReplaceConfigKey        = "config"
	OSDStoreUpdateConfirmation = "yes-really-update-store"
)

// OSDReplaceInfo represents an OSD that needs to replaced
type OSDReplaceInfo struct {
	ID   int    `json:"id"`
	Path string `json:"path"`
	Node string `json:"node"`
}

type OSDReplaceInfoList []OSDReplaceInfo

func (o *OSDReplaceInfoList) getOSDIds() []int {
	osdIDs := []int{}
	for _, osd := range *o {
		osdIDs = append(osdIDs, osd.ID)
	}
	return osdIDs
}

func (o *OSDReplaceInfo) saveAsConfig(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo) error {
	configStr, err := o.string()
	if err != nil {
		return errors.Wrapf(err, "failed to convert osd replace config to string")
	}

	newConfigMap := &v1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name:      OSDReplaceConfigName,
			Namespace: clusterInfo.Namespace,
		},
		Data: map[string]string{
			OSDReplaceConfigKey: configStr,
		},
	}

	err = clusterInfo.OwnerInfo.SetControllerReference(newConfigMap)
	if err != nil {
		return errors.Wrapf(err, "failed to set owner reference on %q configMap", newConfigMap.Name)
	}

	_, err = k8sutil.CreateOrUpdateConfigMap(clusterInfo.Context, context.Clientset, newConfigMap)
	if err != nil {
		return errors.Wrapf(err, "failed to create or update %q configMap", newConfigMap.Name)
	}

	return nil
}

func (o *OSDReplaceInfo) string() (string, error) {
	configInBytes, err := json.Marshal(o)
	if err != nil {
		return "", errors.Wrap(err, "failed to marshal osd replace config")
	}

	return string(configInBytes), nil
}

func (c *Cluster) replaceOSDForNewStore() (OSDReplaceInfoList, error) {
	if c.spec.Storage.Store.UpdateStore != OSDStoreUpdateConfirmation {
		logger.Info("no replacement of osds is requested")
		return nil, nil
	}

	osdsToBeReplaced, err := c.getOSDWithNonMatchingStore()
	if err != nil {
		return nil, errors.Wrapf(err, "failed to get information about the OSDs where backing store does not match the spec in namespace %q", c.clusterInfo.Namespace)
	}

	if len(osdsToBeReplaced) == 0 {
		logger.Debug("all OSDs are using the desired backing store. No replacement is required.")
		return osdsToBeReplaced, nil
	}

	// replace an OSD only if Pgs are healthy
	pgHealthMsg, pgClean, err := cephclient.IsClusterClean(c.context, c.clusterInfo, c.spec.DisruptionManagement.PGHealthyRegex)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to check if the pgs are clean before replacing OSDs")
	}

	if !pgClean {
		logger.Warningf("skipping OSD replacement because pgs are not healthy. PG status: %q", pgHealthMsg)
		return osdsToBeReplaced, nil
	}

	// Check for an existing OSDs in the configmap
	osdToBeReplaced, err := GetOSDReplaceConfigMap(c.context, c.clusterInfo)
	if err != nil {
		return nil, errors.Wrap(err, "failed to get any existing OSD in replace configmap")
	}
	if osdToBeReplaced != nil {
		c.replaceOSD = osdToBeReplaced
	} else {
		c.replaceOSD = &osdsToBeReplaced[0]
	}

	logger.Infof("replacing OSD.%d to the new backing store %q", c.replaceOSD.ID, c.spec.Storage.Store.Type)
	err = c.deleteOSDDeployment(c.replaceOSD.ID)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to delete OSD deployment that needs migration to new backend store in namespace %q", c.clusterInfo.Namespace)
	}

	return osdsToBeReplaced, nil
}

// getOSDWithNonMatchingStore returns OSDs with osd-store label different from expected store in cephCluster spec
func (c *Cluster) getOSDWithNonMatchingStore() (OSDReplaceInfoList, error) {
	osdReplaceList := []OSDReplaceInfo{}
	// get existing OSD deployments
	osdDeployments, err := c.getOSDDeployments()
	if err != nil {
		return osdReplaceList, errors.Wrapf(err, "failed to get existing OSD deployments in namespace %q", c.clusterInfo.Namespace)
	}
	for i := range osdDeployments.Items {
		if osdStore, ok := osdDeployments.Items[i].Labels[osdStore]; ok {
			if osdStore != string(c.spec.Storage.Store.Type) {
				osdInfo, err := c.getOSDInfo(&osdDeployments.Items[i])
				if err != nil {
					return nil, errors.Wrapf(err, "failed to details about the OSD %q", osdDeployments.Items[i].Name)
				}
				var path string
				if osdInfo.PVCName != "" {
					path = osdInfo.PVCName
				} else {
					path = osdInfo.BlockPath
				}
				osdReplaceList = append(osdReplaceList, OSDReplaceInfo{ID: osdInfo.ID, Path: path, Node: osdInfo.NodeName})
			}
		}
	}

	return osdReplaceList, nil
}

// GetOSDReplaceConfigMap returns the OSD replace config map
func GetOSDReplaceConfigMap(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo) (*OSDReplaceInfo, error) {
	cm, err := context.Clientset.CoreV1().ConfigMaps(clusterInfo.Namespace).Get(clusterInfo.Context, OSDReplaceConfigName, metav1.GetOptions{})
	if err != nil {
		if kerrors.IsNotFound(err) {
			return nil, nil
		}
	}

	configStr, ok := cm.Data[OSDReplaceConfigKey]
	if !ok || configStr == "" {
		logger.Debugf("empty config map %q", OSDReplaceConfigName)
		return nil, nil
	}

	config := &OSDReplaceInfo{}
	err = json.Unmarshal([]byte(configStr), config)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to JSON unmarshal osd replace status from the (%q)", configStr)
	}

	return config, nil
}