diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md
index aba50fd31..ab7cb5837 100755
--- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md
+++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md
@@ -80,6 +80,8 @@ For more details on the mons and when to choose a number other than `3`, see the
`useAllNodes` must be set to `false` to use specific nodes and their config.
See [node settings](#node-settings) below.
* `config`: Config settings applied to all OSDs on the node unless overridden by `devices`. See the [config settings](#osd-configuration-settings) below.
+ * `allowDeviceClassUpdate`: Whether to allow changing the device class of an OSD after it is created. The default is false
+ to prevent unintentional data movement or CRUSH changes if the device class is changed accidentally.
* [storage selection settings](#storage-selection-settings)
* [Storage Class Device Sets](#storage-class-device-sets)
* `onlyApplyOSDPlacement`: Whether the placement specific for OSDs is merged with the `all` placement. If `false`, the OSD placement will be merged with the `all` placement. If true, the `OSD placement will be applied` and the `all` placement will be ignored. The placement for OSDs is computed from several different places depending on the type of OSD:
@@ -340,7 +342,7 @@ The following storage selection settings are specific to Ceph and do not apply t
* `metadataDevice`: Name of a device, [partition](#limitations-of-metadata-device) or lvm to use for the metadata of OSDs on each node. Performance can be improved by using a low latency device (such as SSD or NVMe) as the metadata device, while other spinning platter (HDD) devices on a node are used to store data. Provisioning will fail if the user specifies a `metadataDevice` but that device is not used as a metadata device by Ceph. Notably, `ceph-volume` will not use a device of the same device class (HDD, SSD, NVMe) as OSD devices for metadata, resulting in this failure.
* `databaseSizeMB`: The size in MB of a bluestore database. Include quotes around the size.
* `walSizeMB`: The size in MB of a bluestore write ahead log (WAL). Include quotes around the size.
-* `deviceClass`: The [CRUSH device class](https://ceph.io/community/new-luminous-crush-device-classes/) to use for this selection of storage devices. (By default, if a device's class has not already been set, OSDs will automatically set a device's class to either `hdd`, `ssd`, or `nvme` based on the hardware properties exposed by the Linux kernel.) These storage classes can then be used to select the devices backing a storage pool by specifying them as the value of [the pool spec's `deviceClass` field](../Block-Storage/ceph-block-pool-crd.md#spec).
+* `deviceClass`: The [CRUSH device class](https://ceph.io/community/new-luminous-crush-device-classes/) to use for this selection of storage devices. (By default, if a device's class has not already been set, OSDs will automatically set a device's class to either `hdd`, `ssd`, or `nvme` based on the hardware properties exposed by the Linux kernel.) These storage classes can then be used to select the devices backing a storage pool by specifying them as the value of [the pool spec's `deviceClass` field](../Block-Storage/ceph-block-pool-crd.md#spec). If updating the device class of an OSD after the OSD is already created, `allowDeviceClassUpdate: true` must be set. Otherwise updates to this `deviceClass` will be ignored.
* `initialWeight`: The initial OSD weight in TiB units. By default, this value is derived from OSD's capacity.
* `primaryAffinity`: The [primary-affinity](https://docs.ceph.com/en/latest/rados/operations/crush-map/#primary-affinity) value of an OSD, within range `[0, 1]` (default: `1`).
* `osdsPerDevice`**: The number of OSDs to create on each device. High performance devices such as NVMe can handle running multiple OSDs. If desired, this can be overridden for each node and each device.
diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md
index d79438615..03e733c98 100644
--- a/Documentation/CRDs/specification.md
+++ b/Documentation/CRDs/specification.md
@@ -12218,6 +12218,18 @@ float64
BackfillFullRatio is the ratio at which the cluster is too full for backfill. Backfill will be disabled if above this threshold. Default is 0.90.
+
+
+allowDeviceClassUpdate
+
+bool
+
+ |
+
+(Optional)
+ Whether to allow updating the device class after the OSD is initially provisioned
+ |
+
StoreType
diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md
index 556160b33..40d5e84f4 100644
--- a/PendingReleaseNotes.md
+++ b/PendingReleaseNotes.md
@@ -9,3 +9,4 @@ please update the `BucketClass` and `BucketAccessClass` for resolving refer [her
## Features
- Added support for Ceph Squid (v19)
+- Allow updating the device class of OSDs, if `allowDeviceClassUpdate: true` is set
diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml
index a9ea8ce8d..340af674f 100644
--- a/deploy/charts/rook-ceph/templates/resources.yaml
+++ b/deploy/charts/rook-ceph/templates/resources.yaml
@@ -3204,6 +3204,9 @@ spec:
description: A spec for available storage in the cluster and how it should be used
nullable: true
properties:
+ allowDeviceClassUpdate:
+ description: Whether to allow updating the device class after the OSD is initially provisioned
+ type: boolean
backfillFullRatio:
description: BackfillFullRatio is the ratio at which the cluster is too full for backfill. Backfill will be disabled if above this threshold. Default is 0.90.
maximum: 1
diff --git a/deploy/examples/cluster-on-pvc.yaml b/deploy/examples/cluster-on-pvc.yaml
index 9efaf587a..31007322e 100644
--- a/deploy/examples/cluster-on-pvc.yaml
+++ b/deploy/examples/cluster-on-pvc.yaml
@@ -53,6 +53,7 @@ spec:
periodicity: daily # one of: hourly, daily, weekly, monthly
maxLogSize: 500M # SUFFIX may be 'M' or 'G'. Must be at least 1M.
storage:
+ allowDeviceClassUpdate: false # whether to allow changing the device class of an OSD after it is created
storageClassDeviceSets:
- name: set1
# The number of OSDs to create from this device set
@@ -123,7 +124,7 @@ spec:
volumeClaimTemplates:
- metadata:
name: data
- # if you are looking at giving your OSD a different CRUSH device class than the one detected by Ceph
+ # set a different CRUSH device class on the OSD than the one detected by Ceph
# annotations:
# crushDeviceClass: hybrid
spec:
diff --git a/deploy/examples/cluster-test.yaml b/deploy/examples/cluster-test.yaml
index db4c6d75f..7d33a8e93 100644
--- a/deploy/examples/cluster-test.yaml
+++ b/deploy/examples/cluster-test.yaml
@@ -33,7 +33,10 @@ spec:
storage:
useAllNodes: true
useAllDevices: true
+ allowDeviceClassUpdate: true
#deviceFilter:
+ #config:
+ # deviceClass: testclass
monitoring:
enabled: false
healthCheck:
diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml
index 902577817..c27cb8ce2 100644
--- a/deploy/examples/cluster.yaml
+++ b/deploy/examples/cluster.yaml
@@ -255,6 +255,8 @@ spec:
# databaseSizeMB: "1024" # uncomment if the disks are smaller than 100 GB
# osdsPerDevice: "1" # this value can be overridden at the node or device level
# encryptedDevice: "true" # the default value for this option is "false"
+ # deviceClass: "myclass" # specify a device class for OSDs in the cluster
+ allowDeviceClassUpdate: false # whether to allow changing the device class of an OSD after it is created
# Individual nodes and their config can be specified as well, but 'useAllNodes' above must be set to false. Then, only the named
# nodes below will be used as storage resources. Each node's 'name' field should match their 'kubernetes.io/hostname' label.
# nodes:
diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml
index 62e5496c1..e75eb93ad 100644
--- a/deploy/examples/crds.yaml
+++ b/deploy/examples/crds.yaml
@@ -3202,6 +3202,9 @@ spec:
description: A spec for available storage in the cluster and how it should be used
nullable: true
properties:
+ allowDeviceClassUpdate:
+ description: Whether to allow updating the device class after the OSD is initially provisioned
+ type: boolean
backfillFullRatio:
description: BackfillFullRatio is the ratio at which the cluster is too full for backfill. Backfill will be disabled if above this threshold. Default is 0.90.
maximum: 1
diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go
index 8a3762573..624e2020c 100755
--- a/pkg/apis/ceph.rook.io/v1/types.go
+++ b/pkg/apis/ceph.rook.io/v1/types.go
@@ -2861,6 +2861,9 @@ type StorageScopeSpec struct {
// +optional
// +nullable
BackfillFullRatio *float64 `json:"backfillFullRatio,omitempty"`
+ // Whether to allow updating the device class after the OSD is initially provisioned
+ // +optional
+ AllowDeviceClassUpdate bool `json:"allowDeviceClassUpdate,omitempty"`
}
// OSDStore is the backend storage type used for creating the OSDs
diff --git a/pkg/daemon/ceph/client/osd.go b/pkg/daemon/ceph/client/osd.go
index 65c266d2d..63b9341c2 100644
--- a/pkg/daemon/ceph/client/osd.go
+++ b/pkg/daemon/ceph/client/osd.go
@@ -38,6 +38,7 @@ type OSDUsage struct {
type OSDNodeUsage struct {
ID int `json:"id"`
Name string `json:"name"`
+ DeviceClass string `json:"device_class"`
CrushWeight json.Number `json:"crush_weight"`
Depth json.Number `json:"depth"`
Reweight json.Number `json:"reweight"`
@@ -219,6 +220,24 @@ func GetOSDUsage(context *clusterd.Context, clusterInfo *ClusterInfo) (*OSDUsage
return &osdUsage, nil
}
+func SetDeviceClass(context *clusterd.Context, clusterInfo *ClusterInfo, osdID int, deviceClass string) error {
+ // First remove the existing device class
+ args := []string{"osd", "crush", "rm-device-class", fmt.Sprintf("osd.%d", osdID)}
+ buf, err := NewCephCommand(context, clusterInfo, args).Run()
+ if err != nil {
+ return errors.Wrap(err, "failed to remove device class. "+string(buf))
+ }
+
+ // Second, apply the desired device class
+ args = []string{"osd", "crush", "set-device-class", deviceClass, fmt.Sprintf("osd.%d", osdID)}
+ buf, err = NewCephCommand(context, clusterInfo, args).Run()
+ if err != nil {
+ return errors.Wrap(err, "failed to set the device class. "+string(buf))
+ }
+
+ return nil
+}
+
func GetOSDPerfStats(context *clusterd.Context, clusterInfo *ClusterInfo) (*OSDPerfStats, error) {
args := []string{"osd", "perf"}
buf, err := NewCephCommand(context, clusterInfo, args).Run()
diff --git a/pkg/operator/ceph/cluster/osd/create.go b/pkg/operator/ceph/cluster/osd/create.go
index ea3d5d2d2..31f7eb530 100644
--- a/pkg/operator/ceph/cluster/osd/create.go
+++ b/pkg/operator/ceph/cluster/osd/create.go
@@ -88,7 +88,7 @@ func (c *createConfig) createNewOSDsFromStatus(
return
}
- for _, osd := range status.OSDs {
+ for i, osd := range status.OSDs {
if c.deployments.Exists(osd.ID) {
// This OSD will be handled by the updater
logger.Debugf("not creating deployment for OSD %d which already exists", osd.ID)
@@ -96,13 +96,13 @@ func (c *createConfig) createNewOSDsFromStatus(
}
if status.PvcBackedOSD {
logger.Infof("creating OSD %d on PVC %q", osd.ID, nodeOrPVCName)
- err := createDaemonOnPVCFunc(c.cluster, osd, nodeOrPVCName, c.provisionConfig)
+ err := createDaemonOnPVCFunc(c.cluster, &status.OSDs[i], nodeOrPVCName, c.provisionConfig)
if err != nil {
errs.addError("%v", errors.Wrapf(err, "failed to create OSD %d on PVC %q", osd.ID, nodeOrPVCName))
}
} else {
logger.Infof("creating OSD %d on node %q", osd.ID, nodeOrPVCName)
- err := createDaemonOnNodeFunc(c.cluster, osd, nodeOrPVCName, c.provisionConfig)
+ err := createDaemonOnNodeFunc(c.cluster, &status.OSDs[i], nodeOrPVCName, c.provisionConfig)
if err != nil {
errs.addError("%v", errors.Wrapf(err, "failed to create OSD %d on node %q", osd.ID, nodeOrPVCName))
}
@@ -372,7 +372,7 @@ func (c *Cluster) runPrepareJob(osdProps *osdProperties, config *provisionConfig
return nil
}
-func createDaemonOnPVC(c *Cluster, osd OSDInfo, pvcName string, config *provisionConfig) error {
+func createDaemonOnPVC(c *Cluster, osd *OSDInfo, pvcName string, config *provisionConfig) error {
d, err := deploymentOnPVC(c, osd, pvcName, config)
if err != nil {
return err
@@ -402,7 +402,7 @@ func createDaemonOnPVC(c *Cluster, osd OSDInfo, pvcName string, config *provisio
return nil
}
-func createDaemonOnNode(c *Cluster, osd OSDInfo, nodeName string, config *provisionConfig) error {
+func createDaemonOnNode(c *Cluster, osd *OSDInfo, nodeName string, config *provisionConfig) error {
d, err := deploymentOnNode(c, osd, nodeName, config)
if err != nil {
return err
diff --git a/pkg/operator/ceph/cluster/osd/create_test.go b/pkg/operator/ceph/cluster/osd/create_test.go
index 54722eb53..6a5aea9de 100644
--- a/pkg/operator/ceph/cluster/osd/create_test.go
+++ b/pkg/operator/ceph/cluster/osd/create_test.go
@@ -62,7 +62,7 @@ func Test_createNewOSDsFromStatus(t *testing.T) {
}()
createCallsOnNode := []int{}
induceFailureCreatingOSD := -1 // allow causing the create call to fail for a given OSD ID
- createDaemonOnNodeFunc = func(c *Cluster, osd OSDInfo, nodeName string, config *provisionConfig) error {
+ createDaemonOnNodeFunc = func(c *Cluster, osd *OSDInfo, nodeName string, config *provisionConfig) error {
createCallsOnNode = append(createCallsOnNode, osd.ID)
if induceFailureCreatingOSD == osd.ID {
return errors.Errorf("createOSDDaemonOnNode: induced failure on OSD %d", osd.ID)
@@ -76,7 +76,7 @@ func Test_createNewOSDsFromStatus(t *testing.T) {
}()
createCallsOnPVC := []int{}
// reuse induceFailureCreatingOSD from above
- createDaemonOnPVCFunc = func(c *Cluster, osd OSDInfo, pvcName string, config *provisionConfig) error {
+ createDaemonOnPVCFunc = func(c *Cluster, osd *OSDInfo, pvcName string, config *provisionConfig) error {
createCallsOnPVC = append(createCallsOnPVC, osd.ID)
if induceFailureCreatingOSD == osd.ID {
return errors.Errorf("createOSDDaemonOnNode: induced failure on OSD %d", osd.ID)
diff --git a/pkg/operator/ceph/cluster/osd/integration_test.go b/pkg/operator/ceph/cluster/osd/integration_test.go
index 54928ab13..9ff325f8a 100644
--- a/pkg/operator/ceph/cluster/osd/integration_test.go
+++ b/pkg/operator/ceph/cluster/osd/integration_test.go
@@ -585,6 +585,9 @@ func osdIntegrationTestExecutor(t *testing.T, clientset *fake.Clientset, namespa
return `["ssd"]`, nil
}
}
+ if args[1] == "df" {
+ return osdDFResults, nil
+ }
}
if args[0] == "versions" {
// the update deploy code only cares about the mons from the ceph version command results
diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go
index cab46aa5c..24dbc52a1 100644
--- a/pkg/operator/ceph/cluster/osd/osd.go
+++ b/pkg/operator/ceph/cluster/osd/osd.go
@@ -283,6 +283,11 @@ func (c *Cluster) Start() error {
return errors.Wrapf(err, "failed to reconcile key rotation cron jobs")
}
+ err = c.postReconcileUpdateOSDProperties(updateConfig.osdDesiredState)
+ if err != nil {
+ return errors.Wrap(err, "failed post reconcile of osd properties")
+ }
+
err = c.updateCephStorageStatus()
if err != nil {
return errors.Wrapf(err, "failed to update ceph storage status")
@@ -314,6 +319,40 @@ func (c *Cluster) Start() error {
return nil
}
+func (c *Cluster) postReconcileUpdateOSDProperties(desiredOSDs map[int]*OSDInfo) error {
+ osdUsage, err := cephclient.GetOSDUsage(c.context, c.clusterInfo)
+ if err != nil {
+ return errors.Wrap(err, "failed to get osd usage")
+ }
+ logger.Debugf("post processing osd properties with %d actual osds from ceph osd df and %d existing osds found during reconcile", len(osdUsage.OSDNodes), len(desiredOSDs))
+ for _, actualOSD := range osdUsage.OSDNodes {
+ if desiredOSD, ok := desiredOSDs[actualOSD.ID]; ok {
+ if err := c.updateDeviceClassIfChanged(actualOSD.ID, desiredOSD.DeviceClass, actualOSD.DeviceClass); err != nil {
+ // Log the error and allow other updates to continue
+ logger.Error(err)
+ }
+ }
+ }
+ return nil
+}
+
+func (c *Cluster) updateDeviceClassIfChanged(osdID int, desiredDeviceClass, actualDeviceClass string) error {
+ if !c.spec.Storage.AllowDeviceClassUpdate {
+ // device class updates are not allowed by default
+ return nil
+ }
+ if desiredDeviceClass != "" && desiredDeviceClass != actualDeviceClass {
+ logger.Infof("updating osd.%d device class from %q to %q", osdID, actualDeviceClass, desiredDeviceClass)
+ err := cephclient.SetDeviceClass(c.context, c.clusterInfo, osdID, desiredDeviceClass)
+ if err != nil {
+ return errors.Wrapf(err, "failed to set device class on osd %d", osdID)
+ }
+ return nil
+ }
+ logger.Debugf("no device class change needed for osd.%d. desired=%q, actual=%q", osdID, desiredDeviceClass, actualDeviceClass)
+ return nil
+}
+
func (c *Cluster) getExistingOSDDeploymentsOnPVCs() (sets.Set[string], error) {
listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s,%s", k8sutil.AppAttr, AppName, OSDOverPVCLabelKey)}
@@ -332,7 +371,7 @@ func (c *Cluster) getExistingOSDDeploymentsOnPVCs() (sets.Set[string], error) {
return result, nil
}
-func deploymentOnNode(c *Cluster, osd OSDInfo, nodeName string, config *provisionConfig) (*appsv1.Deployment, error) {
+func deploymentOnNode(c *Cluster, osd *OSDInfo, nodeName string, config *provisionConfig) (*appsv1.Deployment, error) {
osdLongName := fmt.Sprintf("OSD %d on node %q", osd.ID, nodeName)
osdProps, err := c.getOSDPropsForNode(nodeName, osd.DeviceClass)
@@ -353,7 +392,7 @@ func deploymentOnNode(c *Cluster, osd OSDInfo, nodeName string, config *provisio
return d, nil
}
-func deploymentOnPVC(c *Cluster, osd OSDInfo, pvcName string, config *provisionConfig) (*appsv1.Deployment, error) {
+func deploymentOnPVC(c *Cluster, osd *OSDInfo, pvcName string, config *provisionConfig) (*appsv1.Deployment, error) {
osdLongName := fmt.Sprintf("OSD %d on PVC %q", osd.ID, pvcName)
osdProps, err := c.getOSDPropsForPVC(pvcName, osd.DeviceClass)
@@ -376,7 +415,7 @@ func deploymentOnPVC(c *Cluster, osd OSDInfo, pvcName string, config *provisionC
// setOSDProperties is used to configure an OSD with parameters which can not be set via explicit
// command-line arguments.
-func setOSDProperties(c *Cluster, osdProps osdProperties, osd OSDInfo) error {
+func setOSDProperties(c *Cluster, osdProps osdProperties, osd *OSDInfo) error {
// OSD's 'primary-affinity' has to be configured via command which goes through mons
if osdProps.storeConfig.PrimaryAffinity != "" {
return cephclient.SetPrimaryAffinity(c.context, c.clusterInfo, osd.ID, osdProps.storeConfig.PrimaryAffinity)
@@ -460,6 +499,7 @@ func (c *Cluster) getOSDPropsForPVC(pvcName, osdDeviceClass string) (osdProperti
}
osdProps.storeConfig.InitialWeight = deviceSet.CrushInitialWeight
osdProps.storeConfig.PrimaryAffinity = deviceSet.CrushPrimaryAffinity
+ osdProps.storeConfig.DeviceClass = deviceSet.CrushDeviceClass
// If OSD isn't portable, we're getting the host name either from the osd deployment that was already initialized
// or from the osd prepare job from initial creation.
diff --git a/pkg/operator/ceph/cluster/osd/osd_test.go b/pkg/operator/ceph/cluster/osd/osd_test.go
index 062fe3614..8d38bcae7 100644
--- a/pkg/operator/ceph/cluster/osd/osd_test.go
+++ b/pkg/operator/ceph/cluster/osd/osd_test.go
@@ -51,6 +51,12 @@ import (
const (
healthyCephStatus = `{"fsid":"877a47e0-7f6c-435e-891a-76983ab8c509","health":{"checks":{},"status":"HEALTH_OK"},"election_epoch":12,"quorum":[0,1,2],"quorum_names":["a","b","c"],"monmap":{"epoch":3,"fsid":"877a47e0-7f6c-435e-891a-76983ab8c509","modified":"2020-11-02 09:58:23.015313","created":"2020-11-02 09:57:37.719235","min_mon_release":14,"min_mon_release_name":"nautilus","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.30.74.42:3300","nonce":0},{"type":"v1","addr":"172.30.74.42:6789","nonce":0}]},"addr":"172.30.74.42:6789/0","public_addr":"172.30.74.42:6789/0"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.30.101.61:3300","nonce":0},{"type":"v1","addr":"172.30.101.61:6789","nonce":0}]},"addr":"172.30.101.61:6789/0","public_addr":"172.30.101.61:6789/0"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.30.250.55:3300","nonce":0},{"type":"v1","addr":"172.30.250.55:6789","nonce":0}]},"addr":"172.30.250.55:6789/0","public_addr":"172.30.250.55:6789/0"}]},"osdmap":{"osdmap":{"epoch":19,"num_osds":3,"num_up_osds":3,"num_in_osds":3,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":96}],"num_pgs":96,"num_pools":3,"num_objects":79,"data_bytes":81553681,"bytes_used":3255447552,"bytes_avail":1646011994112,"bytes_total":1649267441664,"read_bytes_sec":853,"write_bytes_sec":5118,"read_op_per_sec":1,"write_op_per_sec":0},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"ocs-storagecluster-cephfilesystem-b","status":"up:active","gid":14161},{"filesystem_id":1,"rank":0,"name":"ocs-storagecluster-cephfilesystem-a","status":"up:standby-replay","gid":24146}],"up:standby":0},"mgrmap":{"epoch":10,"active_gid":14122,"active_name":"a","active_addrs":{"addrvec":[{"type":"v2","addr":"10.131.0.28:6800","nonce":1},{"type":"v1","addr":"10.131.0.28:6801","nonce":1}]}}}`
unHealthyCephStatus = `{"fsid":"613975f3-3025-4802-9de1-a2280b950e75","health":{"checks":{"OSD_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 osds down"}},"OSD_HOST_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 host (1 osds) down"}},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","summary":{"message":"Reduced data availability: 101 pgs stale"}},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"application not enabled on 1 pool(s)"}}},"status":"HEALTH_WARN","overall_status":"HEALTH_WARN"},"election_epoch":12,"quorum":[0,1,2],"quorum_names":["rook-ceph-mon0","rook-ceph-mon2","rook-ceph-mon1"],"monmap":{"epoch":3,"fsid":"613975f3-3025-4802-9de1-a2280b950e75","modified":"2017-08-11 20:13:02.075679","created":"2017-08-11 20:12:35.314510","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"10.3.0.45:6789/0","public_addr":"10.3.0.45:6789/0"},{"rank":1,"name":"rook-ceph-mon2","addr":"10.3.0.249:6789/0","public_addr":"10.3.0.249:6789/0"},{"rank":2,"name":"rook-ceph-mon1","addr":"10.3.0.252:6789/0","public_addr":"10.3.0.252:6789/0"}]},"osdmap":{"osdmap":{"epoch":17,"num_osds":2,"num_up_osds":1,"num_in_osds":2,"full":false,"nearfull":true,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"stale+active+clean","count":101},{"state_name":"active+clean","count":99}],"num_pgs":200,"num_pools":2,"num_objects":243,"data_bytes":976793635,"bytes_used":13611479040,"bytes_avail":19825307648,"bytes_total":33436786688},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":14111,"active_name":"rook-ceph-mgr0","active_addr":"10.2.73.6:6800/9","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","prometheus","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}`
+ osdDFResults = `
+ {"nodes":[
+ {"id":0,"device_class":"hdd","name":"osd.0","type":"osd","type_id":0,"crush_weight":0.039093017578125,"depth":2,"pool_weights":{},"reweight":1,"kb":41943040,"kb_used":27640,"kb_used_data":432,"kb_used_omap":1,"kb_used_meta":27198,"kb_avail":41915400,"utilization":0.065898895263671875,"var":0.99448308946989694,"pgs":9,"status":"up"},
+ {"id":1,"device_class":"hdd","name":"osd.1","type":"osd","type_id":0,"crush_weight":0.039093017578125,"depth":2,"pool_weights":{},"reweight":1,"kb":41943040,"kb_used":27960,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":27198,"kb_avail":41915080,"utilization":0.066661834716796875,"var":1.005996641880547,"pgs":15,"status":"up"},
+ {"id":2,"device_class":"hdd","name":"osd.2","type":"osd","type_id":0,"crush_weight":0.039093017578125,"depth":2,"pool_weights":{},"reweight":1,"kb":41943040,"kb_used":27780,"kb_used_data":564,"kb_used_omap":1,"kb_used_meta":27198,"kb_avail":41915260,"utilization":0.066232681274414062,"var":0.99952026864955634,"pgs":8,"status":"up"}],
+ "stray":[],"summary":{"total_kb":125829120,"total_kb_used":83380,"total_kb_used_data":1748,"total_kb_used_omap":3,"total_kb_used_meta":81596,"total_kb_avail":125745740,"average_utilization":0.066264470418294266,"min_var":0.99448308946989694,"max_var":1.005996641880547,"dev":0.00031227879054369131}}`
)
func TestOSDProperties(t *testing.T) {
@@ -80,6 +86,9 @@ func TestStart(t *testing.T) {
// Mock executor for OSD crush class list command, returning ssd as available device class
return `["ssd"]`, nil
}
+ if args[0] == "osd" && args[1] == "df" {
+ return osdDFResults, nil
+ }
return "", nil
},
}
@@ -361,6 +370,68 @@ func TestAddRemoveNode(t *testing.T) {
assert.True(t, k8serrors.IsNotFound(err))
}
+func TestUpdateDeviceClass(t *testing.T) {
+ namespace := "ns"
+ clientset := fake.NewSimpleClientset()
+ removedDeviceClassOSD := ""
+ setDeviceClassOSD := ""
+ setDeviceClass := ""
+ executor := &exectest.MockExecutor{
+ MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
+ logger.Infof("ExecuteCommandWithOutput: %s %v", command, args)
+ if args[0] == "osd" {
+ if args[1] == "df" {
+ return osdDFResults, nil
+ }
+ if args[1] == "crush" {
+ if args[2] == "rm-device-class" {
+ removedDeviceClassOSD = args[3]
+ } else if args[2] == "set-device-class" {
+ setDeviceClass = args[3]
+ setDeviceClassOSD = args[4]
+ }
+ }
+ }
+ return "", nil
+ },
+ }
+
+ cephCluster := &cephv1.CephCluster{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "testing",
+ Namespace: namespace,
+ },
+ Spec: cephv1.ClusterSpec{Storage: cephv1.StorageScopeSpec{AllowDeviceClassUpdate: true}},
+ }
+ // Objects to track in the fake client.
+ object := []runtime.Object{
+ cephCluster,
+ }
+ s := scheme.Scheme
+ // Create a fake client to mock API calls.
+ client := clientfake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
+ clusterInfo := &cephclient.ClusterInfo{
+ Namespace: namespace,
+ CephVersion: cephver.Quincy,
+ Context: context.TODO(),
+ }
+ clusterInfo.SetName("rook-ceph-test")
+ context := &clusterd.Context{Clientset: clientset, Client: client, ConfigDir: "/var/lib/rook", Executor: executor}
+ c := New(context, clusterInfo, cephCluster.Spec, "myversion")
+
+ // Start the first time
+ desiredOSDs := map[int]*OSDInfo{
+ 0: {ID: 0, DeviceClass: "hdd"},
+ 1: {ID: 1, DeviceClass: "hdd"},
+ 2: {ID: 2, DeviceClass: "newclass"},
+ }
+ err := c.postReconcileUpdateOSDProperties(desiredOSDs)
+ assert.Nil(t, err)
+ assert.Equal(t, "newclass", setDeviceClass)
+ assert.Equal(t, "osd.2", setDeviceClassOSD)
+ assert.Equal(t, "osd.2", removedDeviceClassOSD)
+}
+
func TestAddNodeFailure(t *testing.T) {
// create a storage spec with the given nodes/devices/dirs
nodeName := "node1672"
@@ -481,9 +552,9 @@ func TestGetOSDInfo(t *testing.T) {
node := "n1"
location := "root=default host=myhost zone=myzone"
- osd1 := OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "dev/logical-volume-path", CVMode: "raw", Location: location, TopologyAffinity: "topology.rook.io/rack=rack0"}
- osd2 := OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "vg1/lv1", CVMode: "lvm", LVBackedPV: true}
- osd3 := OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "", CVMode: "raw"}
+ osd1 := &OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "dev/logical-volume-path", CVMode: "raw", Location: location, TopologyAffinity: "topology.rook.io/rack=rack0"}
+ osd2 := &OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "vg1/lv1", CVMode: "lvm", LVBackedPV: true}
+ osd3 := &OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "", CVMode: "raw"}
osdProp := osdProperties{
crushHostname: node,
pvc: corev1.PersistentVolumeClaimVolumeSource{ClaimName: "pvc"},
@@ -533,8 +604,8 @@ func TestGetOSDInfo(t *testing.T) {
t.Run("get info from node-based OSDs", func(t *testing.T) {
useAllDevices := true
- osd4 := OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "", CVMode: "lvm", Location: location}
- osd5 := OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "vg1/lv1", CVMode: "lvm"}
+ osd4 := &OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "", CVMode: "lvm", Location: location}
+ osd5 := &OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "vg1/lv1", CVMode: "lvm"}
osdProp = osdProperties{
crushHostname: node,
devices: []cephv1.Device{},
@@ -668,9 +739,9 @@ func TestGetOSDInfoWithCustomRoot(t *testing.T) {
node := "n1"
location := "root=custom-root host=myhost zone=myzone"
- osd1 := OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "dev/logical-volume-path", CVMode: "raw", Location: location}
- osd2 := OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "vg1/lv1", CVMode: "lvm", LVBackedPV: true, Location: location}
- osd3 := OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "", CVMode: "lvm", Location: location}
+ osd1 := &OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "dev/logical-volume-path", CVMode: "raw", Location: location}
+ osd2 := &OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "vg1/lv1", CVMode: "lvm", LVBackedPV: true, Location: location}
+ osd3 := &OSDInfo{ID: 3, UUID: "osd-uuid", BlockPath: "", CVMode: "lvm", Location: location}
osdProp := osdProperties{
crushHostname: node,
pvc: corev1.PersistentVolumeClaimVolumeSource{ClaimName: "pvc"},
diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go
index bf8066e04..b44aa4ee3 100644
--- a/pkg/operator/ceph/cluster/osd/spec.go
+++ b/pkg/operator/ceph/cluster/osd/spec.go
@@ -319,7 +319,7 @@ func deploymentName(osdID int) string {
return fmt.Sprintf(osdAppNameFmt, osdID)
}
-func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionConfig *provisionConfig) (*apps.Deployment, error) {
+func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provisionConfig *provisionConfig) (*apps.Deployment, error) {
deploymentName := deploymentName(osd.ID)
replicaCount := int32(1)
volumeMounts := controller.CephVolumeMounts(provisionConfig.DataPathMap, false)
@@ -339,6 +339,11 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
return nil, errors.Errorf("failed to generate deployment for OSD %d. required CVMode is not specified for this OSD", osd.ID)
}
+ if c.spec.Storage.AllowDeviceClassUpdate && osdProps.storeConfig.DeviceClass != "" && osdProps.storeConfig.DeviceClass != osd.DeviceClass {
+ logger.Infof("The device class for osd %d is changing from %q to %q", osd.ID, osd.DeviceClass, osdProps.storeConfig.DeviceClass)
+ osd.DeviceClass = osdProps.storeConfig.DeviceClass
+ }
+
dataDir := k8sutil.DataDir
// Create volume config for /dev so the pod can access devices on the host
// Only valid when running OSD on device or OSD on LV-backed PVC
@@ -507,10 +512,10 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
// needed for luksOpen synchronization when devices are encrypted and the osd is prepared with LVM
hostIPC := osdProps.storeConfig.EncryptedDevice || osdProps.encrypted
- osdLabels := c.getOSDLabels(osd, failureDomainValue, osdProps.portable)
+ osdLabels := c.getOSDLabels(*osd, failureDomainValue, osdProps.portable)
if osd.ExportService {
- osdService, err := c.createOSDService(osd, osdLabels)
+ osdService, err := c.createOSDService(*osd, osdLabels)
if err != nil {
return nil, errors.Wrapf(err, "failed to configure osd service for osd.%d", osd.ID)
}
@@ -598,11 +603,11 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
// so that it can pick the already mounted/activated osd metadata path
// This container will activate the OSD and place the activated files into an empty dir
// The empty dir will be shared by the "activate-osd" pod and the "osd" main pod
- activateOSDVolume, activateOSDContainer := c.getActivateOSDInitContainer(c.spec.DataDirHostPath, c.clusterInfo.Namespace, osdID, osd, osdProps)
+ activateOSDVolume, activateOSDContainer := c.getActivateOSDInitContainer(c.spec.DataDirHostPath, c.clusterInfo.Namespace, osdID, *osd, osdProps)
volumes = append(volumes, activateOSDVolume...)
volumeMounts = append(volumeMounts, activateOSDContainer.VolumeMounts[0])
initContainers = append(initContainers, *activateOSDContainer)
- initContainers = append(initContainers, c.getExpandInitContainer(osdProps, c.spec.DataDirHostPath, c.clusterInfo.Namespace, osdID, osd))
+ initContainers = append(initContainers, c.getExpandInitContainer(osdProps, c.spec.DataDirHostPath, c.clusterInfo.Namespace, osdID, *osd))
}
// Doing a chown in a post start lifecycle hook does not reliably complete before the OSD
@@ -734,7 +739,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
}
if osdProps.portable {
// portable OSDs must have affinity to the topology where the osd prepare job was executed
- if err := applyTopologyAffinity(&deployment.Spec.Template.Spec, osd); err != nil {
+ if err := applyTopologyAffinity(&deployment.Spec.Template.Spec, *osd); err != nil {
return nil, err
}
} else {
diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go
index 6532d7c85..b4dbfc2d6 100644
--- a/pkg/operator/ceph/cluster/osd/spec_test.go
+++ b/pkg/operator/ceph/cluster/osd/spec_test.go
@@ -114,7 +114,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
if len(devices) == 0 && len(dataDir) == 0 {
return
}
- osd := OSDInfo{
+ osd := &OSDInfo{
ID: 0,
CVMode: "raw",
}
@@ -190,7 +190,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
pvc: corev1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc"},
}
// Not needed when running on PVC
- osd = OSDInfo{
+ osd = &OSDInfo{
ID: 0,
CVMode: "lvm",
}
@@ -212,7 +212,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
assert.Equal(t, 9, len(cont.VolumeMounts), cont.VolumeMounts)
// Test OSD on PVC with RAW
- osd = OSDInfo{
+ osd = &OSDInfo{
ID: 0,
CVMode: "raw",
}
@@ -251,7 +251,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
assert.Equal(t, 10, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes)
// // Test OSD on PVC with RAW and metadata device
- osd = OSDInfo{
+ osd = &OSDInfo{
ID: 0,
CVMode: "raw",
}
@@ -275,7 +275,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
assert.Equal(t, 10, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes)
// // Test encrypted OSD on PVC with RAW and metadata device
- osd = OSDInfo{
+ osd = &OSDInfo{
ID: 0,
CVMode: "raw",
}
@@ -307,7 +307,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
assert.Equal(t, 12, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes)
// // Test OSD on PVC with RAW / metadata and wal device
- osd = OSDInfo{
+ osd = &OSDInfo{
ID: 0,
CVMode: "raw",
}
@@ -333,7 +333,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
assert.Equal(t, 12, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes)
// // Test encrypted OSD on PVC with RAW / metadata and wal device
- osd = OSDInfo{
+ osd = &OSDInfo{
ID: 0,
CVMode: "raw",
}
@@ -608,7 +608,7 @@ func TestHostNetwork(t *testing.T) {
c := New(context, clusterInfo, spec, "rook/rook:myversion")
n := c.spec.Storage.ResolveNode(storageSpec.Nodes[0].Name)
- osd := OSDInfo{
+ osd := &OSDInfo{
ID: 0,
CVMode: "raw",
}
@@ -712,7 +712,7 @@ func TestClusterGetPVCEncryptionInitContainerActivate(t *testing.T) {
// WARNING! modifies c.deviceSets
func getDummyDeploymentOnPVC(clientset *fake.Clientset, c *Cluster, pvcName string, osdID int) *appsv1.Deployment {
- osd := OSDInfo{
+ osd := &OSDInfo{
ID: osdID,
UUID: "some-uuid",
BlockPath: "/some/path",
@@ -736,7 +736,7 @@ func getDummyDeploymentOnPVC(clientset *fake.Clientset, c *Cluster, pvcName stri
// WARNING! modifies c.ValidStorage
func getDummyDeploymentOnNode(clientset *fake.Clientset, c *Cluster, nodeName string, osdID int) *appsv1.Deployment {
- osd := OSDInfo{
+ osd := &OSDInfo{
ID: osdID,
UUID: "some-uuid",
BlockPath: "/dev/vda",
@@ -855,7 +855,7 @@ func TestOSDPlacement(t *testing.T) {
}
c := New(context, clusterInfo, spec, "rook/rook:myversion")
- osd := OSDInfo{
+ osd := &OSDInfo{
ID: 0,
CVMode: "raw",
}
diff --git a/pkg/operator/ceph/cluster/osd/update.go b/pkg/operator/ceph/cluster/osd/update.go
index ea4076fbe..eb6ac31a1 100644
--- a/pkg/operator/ceph/cluster/osd/update.go
+++ b/pkg/operator/ceph/cluster/osd/update.go
@@ -49,6 +49,7 @@ type updateConfig struct {
numUpdatesNeeded int // the number of OSDs that needed updating
deployments *existenceList // these OSDs have existing deployments
osdsToSkipReconcile sets.Set[string] // these OSDs should not be updated during reconcile
+ osdDesiredState map[int]*OSDInfo // the desired state of the OSDs determined during the reconcile
}
func (c *Cluster) newUpdateConfig(
@@ -64,6 +65,7 @@ func (c *Cluster) newUpdateConfig(
queue.Len(),
deployments,
osdsToSkipReconcile,
+ map[int]*OSDInfo{},
}
}
@@ -142,6 +144,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
errs.addError("failed to update OSD %d. failed to extract OSD info from existing deployment %q. %v", osdID, depName, err)
continue
}
+ c.osdDesiredState[osdID] = &osdInfo
if c.osdsToSkipReconcile.Has(strconv.Itoa(osdID)) {
logger.Warningf("Skipping update for OSD %d since labeled with %s", osdID, cephv1.SkipReconcileLabelKey)
@@ -174,7 +177,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
if osdIsOnPVC(dep) {
logger.Infof("updating OSD %d on PVC %q", osdID, nodeOrPVCName)
- updatedDep, err = deploymentOnPVCFunc(c.cluster, osdInfo, nodeOrPVCName, c.provisionConfig)
+ updatedDep, err = deploymentOnPVCFunc(c.cluster, &osdInfo, nodeOrPVCName, c.provisionConfig)
message := fmt.Sprintf("Processing OSD %d on PVC %q", osdID, nodeOrPVCName)
updateConditionFunc(c.cluster.clusterInfo.Context, c.cluster.context, c.cluster.clusterInfo.NamespacedName(), k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, message)
} else {
@@ -189,7 +192,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
}
logger.Infof("updating OSD %d on node %q", osdID, nodeOrPVCName)
- updatedDep, err = deploymentOnNodeFunc(c.cluster, osdInfo, nodeOrPVCName, c.provisionConfig)
+ updatedDep, err = deploymentOnNodeFunc(c.cluster, &osdInfo, nodeOrPVCName, c.provisionConfig)
message := fmt.Sprintf("Processing OSD %d on node %q", osdID, nodeOrPVCName)
updateConditionFunc(c.cluster.clusterInfo.Context, c.cluster.context, c.cluster.clusterInfo.NamespacedName(), k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, message)
}
diff --git a/pkg/operator/ceph/cluster/osd/update_test.go b/pkg/operator/ceph/cluster/osd/update_test.go
index 4c7bfc786..134ba2dba 100644
--- a/pkg/operator/ceph/cluster/osd/update_test.go
+++ b/pkg/operator/ceph/cluster/osd/update_test.go
@@ -175,11 +175,11 @@ func Test_updateExistingOSDs(t *testing.T) {
}
// simple wrappers to allow us to count how many OSDs on nodes/PVCs are identified
- deploymentOnNodeFunc = func(c *Cluster, osd OSDInfo, nodeName string, config *provisionConfig) (*appsv1.Deployment, error) {
+ deploymentOnNodeFunc = func(c *Cluster, osd *OSDInfo, nodeName string, config *provisionConfig) (*appsv1.Deployment, error) {
osdsOnNodes = append(osdsOnNodes, osd.ID)
return deploymentOnNode(c, osd, nodeName, config)
}
- deploymentOnPVCFunc = func(c *Cluster, osd OSDInfo, pvcName string, config *provisionConfig) (*appsv1.Deployment, error) {
+ deploymentOnPVCFunc = func(c *Cluster, osd *OSDInfo, pvcName string, config *provisionConfig) (*appsv1.Deployment, error) {
osdsOnPVCs = append(osdsOnPVCs, osd.ID)
return deploymentOnPVC(c, osd, pvcName, config)
}