diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index d35846f28..c8c0e7947 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -464,6 +464,8 @@ The following storage selection settings are specific to Ceph and do not apply t * `databaseSizeMB`: The size in MB of a bluestore database. Include quotes around the size. * `walSizeMB`: The size in MB of a bluestore write ahead log (WAL). Include quotes around the size. * `deviceClass`: The [CRUSH device class](https://ceph.io/community/new-luminous-crush-device-classes/) to use for this selection of storage devices. (By default, if a device's class has not already been set, OSDs will automatically set a device's class to either `hdd`, `ssd`, or `nvme` based on the hardware properties exposed by the Linux kernel.) These storage classes can then be used to select the devices backing a storage pool by specifying them as the value of [the pool spec's `deviceClass` field](ceph-pool-crd.md#spec). +* `initialWeight`: The initial OSD weight in TiB units. By default, this value is derived from OSD's capacity. +* `primaryAffinity`: The [primary-affinity](https://docs.ceph.com/en/latest/rados/operations/crush-map/#primary-affinity) value of an OSD, within range `[0, 1]` (default: `1`). * `osdsPerDevice`**: The number of OSDs to create on each device. High performance devices such as NVMe can handle running multiple OSDs. If desired, this can be overridden for each node and each device. * `encryptedDevice`**: Encrypt OSD volumes using dmcrypt ("true" or "false"). By default this option is disabled. See [encryption](http://docs.ceph.com/docs/nautilus/ceph-volume/lvm/encryption/) for more information on encryption in Ceph. * `crushRoot`: The value of the `root` CRUSH map label. The default is `default`. Generally, you should not need to change this. However, if any of your topology labels may have the value `default`, you need to change `crushRoot` to avoid conflicts, since CRUSH map values need to be unique. diff --git a/pkg/daemon/ceph/client/osd.go b/pkg/daemon/ceph/client/osd.go index 9d77f67ec..1cdfd60c0 100644 --- a/pkg/daemon/ceph/client/osd.go +++ b/pkg/daemon/ceph/client/osd.go @@ -355,3 +355,15 @@ func OSDOkToStop(context *clusterd.Context, clusterInfo *ClusterInfo, osdID, max return stats.OSDs, nil } + +// SetPrimaryAffinity assigns primary-affinity (within range [0.0, 1.0]) to a specific OSD. +func SetPrimaryAffinity(context *clusterd.Context, clusterInfo *ClusterInfo, osdID int, affinity string) error { + logger.Infof("setting osd.%d with primary-affinity %q", osdID, affinity) + args := []string{"osd", "primary-affinity", fmt.Sprintf("osd.%d", osdID), affinity} + _, err := NewCephCommand(context, clusterInfo, args).Run() + if err != nil { + return errors.Wrapf(err, "failed to set osd.%d with primary-affinity %q", osdID, affinity) + } + logger.Infof("successfully applied osd.%d primary-affinity %q", osdID, affinity) + return nil +} diff --git a/pkg/operator/ceph/cluster/osd/config/config.go b/pkg/operator/ceph/cluster/osd/config/config.go index d0daa4dc1..bae425afb 100644 --- a/pkg/operator/ceph/cluster/osd/config/config.go +++ b/pkg/operator/ceph/cluster/osd/config/config.go @@ -30,6 +30,7 @@ const ( MetadataDeviceKey = "metadataDevice" DeviceClassKey = "deviceClass" InitialWeightKey = "initialWeight" + PrimaryAffinityKey = "primaryAffinity" ) // StoreConfig represents the configuration of an OSD on a device. @@ -41,6 +42,7 @@ type StoreConfig struct { MetadataDevice string `json:"metadataDevice,omitempty"` DeviceClass string `json:"deviceClass,omitempty"` InitialWeight string `json:"initialWeight,omitempty"` + PrimaryAffinity string `json:"primaryAffinity,omitempty"` } // NewStoreConfig returns a StoreConfig with proper defaults set. @@ -72,6 +74,8 @@ func ToStoreConfig(config map[string]string) StoreConfig { storeConfig.DeviceClass = v case InitialWeightKey: storeConfig.InitialWeight = v + case PrimaryAffinityKey: + storeConfig.PrimaryAffinity = v } } diff --git a/pkg/operator/ceph/cluster/osd/deviceSet.go b/pkg/operator/ceph/cluster/osd/deviceSet.go index eee60e8ae..5e120fe4e 100644 --- a/pkg/operator/ceph/cluster/osd/deviceSet.go +++ b/pkg/operator/ceph/cluster/osd/deviceSet.go @@ -43,6 +43,8 @@ type deviceSet struct { CrushDeviceClass string // CrushInitialWeight represents initial OSD weight in TiB units CrushInitialWeight string + // CrushPrimaryAffinity represents initial OSD primary-affinity within range [0, 1] + CrushPrimaryAffinity string // Size represents the size requested for the PVC Size string // Resources requests/limits for the devices @@ -128,6 +130,7 @@ func (c *Cluster) createDeviceSetPVCsForIndex(newDeviceSet rookv1.StorageClassDe var dataSize string var crushDeviceClass string var crushInitialWeight string + var crushPrimaryAffinity string typesFound := util.NewSet() for _, pvcTemplate := range newDeviceSet.VolumeClaimTemplates { if pvcTemplate.Name == "" { @@ -159,6 +162,7 @@ func (c *Cluster) createDeviceSetPVCsForIndex(newDeviceSet rookv1.StorageClassDe crushDeviceClass = pvcTemplate.Annotations["crushDeviceClass"] } crushInitialWeight = pvcTemplate.Annotations["crushInitialWeight"] + crushPrimaryAffinity = pvcTemplate.Annotations["crushPrimaryAffinity"] pvcSources[pvcType] = v1.PersistentVolumeClaimVolumeSource{ ClaimName: pvc.GetName(), @@ -167,20 +171,21 @@ func (c *Cluster) createDeviceSetPVCsForIndex(newDeviceSet rookv1.StorageClassDe } return deviceSet{ - Name: newDeviceSet.Name, - Resources: newDeviceSet.Resources, - Placement: newDeviceSet.Placement, - PreparePlacement: newDeviceSet.PreparePlacement, - Config: newDeviceSet.Config, - Size: dataSize, - PVCSources: pvcSources, - Portable: newDeviceSet.Portable, - TuneSlowDeviceClass: newDeviceSet.TuneSlowDeviceClass, - TuneFastDeviceClass: newDeviceSet.TuneFastDeviceClass, - SchedulerName: newDeviceSet.SchedulerName, - CrushDeviceClass: crushDeviceClass, - CrushInitialWeight: crushInitialWeight, - Encrypted: newDeviceSet.Encrypted, + Name: newDeviceSet.Name, + Resources: newDeviceSet.Resources, + Placement: newDeviceSet.Placement, + PreparePlacement: newDeviceSet.PreparePlacement, + Config: newDeviceSet.Config, + Size: dataSize, + PVCSources: pvcSources, + Portable: newDeviceSet.Portable, + TuneSlowDeviceClass: newDeviceSet.TuneSlowDeviceClass, + TuneFastDeviceClass: newDeviceSet.TuneFastDeviceClass, + SchedulerName: newDeviceSet.SchedulerName, + CrushDeviceClass: crushDeviceClass, + CrushInitialWeight: crushInitialWeight, + CrushPrimaryAffinity: crushPrimaryAffinity, + Encrypted: newDeviceSet.Encrypted, } } diff --git a/pkg/operator/ceph/cluster/osd/deviceset_test.go b/pkg/operator/ceph/cluster/osd/deviceset_test.go index 6def701a0..da5a15c8e 100644 --- a/pkg/operator/ceph/cluster/osd/deviceset_test.go +++ b/pkg/operator/ceph/cluster/osd/deviceset_test.go @@ -245,7 +245,11 @@ func TestPrepareDeviceSetsWithCrushParams(t *testing.T) { VolumeClaimTemplates: []corev1.PersistentVolumeClaim{testVolumeClaim("testwithcrushparams1")}, SchedulerName: "custom-scheduler", } - deviceSet.VolumeClaimTemplates[0].Annotations = map[string]string{"crushDeviceClass": "ssd", "crushInitialWeight": "0.75"} + deviceSet.VolumeClaimTemplates[0].Annotations = map[string]string{ + "crushDeviceClass": "ssd", + "crushInitialWeight": "0.75", + "crushPrimaryAffinity": "0.666", + } spec := cephv1.ClusterSpec{ Storage: rookv1.StorageScopeSpec{StorageClassDeviceSets: []rookv1.StorageClassDeviceSet{deviceSet}}, @@ -261,6 +265,7 @@ func TestPrepareDeviceSetsWithCrushParams(t *testing.T) { assert.Equal(t, 1, len(cluster.deviceSets)) assert.Equal(t, cluster.deviceSets[0].CrushDeviceClass, "ssd") assert.Equal(t, cluster.deviceSets[0].CrushInitialWeight, "0.75") + assert.Equal(t, cluster.deviceSets[0].CrushPrimaryAffinity, "0.666") pvcs, err := clientset.CoreV1().PersistentVolumeClaims(cluster.clusterInfo.Namespace).List(ctx, metav1.ListOptions{}) assert.NoError(t, err) diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 23011ace0..b2da72963 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -267,6 +267,11 @@ func deploymentOnNode(c *Cluster, osd OSDInfo, nodeName string, config *provisio return nil, errors.Wrapf(err, "failed to generate deployment for %s", osdLongName) } + err = setOSDProperties(c, osdProps, osd) + if err != nil { + return nil, errors.Wrapf(err, "failed to prepare deployment for %s", osdLongName) + } + return d, nil } @@ -283,9 +288,24 @@ func deploymentOnPVC(c *Cluster, osd OSDInfo, pvcName string, config *provisionC return nil, errors.Wrapf(err, "failed to generate deployment for %s", osdLongName) } + err = setOSDProperties(c, osdProps, osd) + if err != nil { + return nil, errors.Wrapf(err, "failed to prepare deployment for %s", osdLongName) + } + return d, nil } +// setOSDProperties is used to configure an OSD with parameters which can not be set via explicit +// command-line arguments. +func setOSDProperties(c *Cluster, osdProps osdProperties, osd OSDInfo) error { + // OSD's 'primary-affinity' has to be configured via command which goes through mons + if osdProps.storeConfig.PrimaryAffinity != "" { + return cephclient.SetPrimaryAffinity(c.context, c.clusterInfo, osd.ID, osdProps.storeConfig.PrimaryAffinity) + } + return nil +} + func (c *Cluster) resolveNode(nodeName string) *rookv1.Node { // fully resolve the storage config and resources for this node rookNode := c.ValidStorage.ResolveNode(nodeName) @@ -361,6 +381,7 @@ func (c *Cluster) getOSDPropsForPVC(pvcName string) (osdProperties, error) { deviceSetName: deviceSet.Name, } osdProps.storeConfig.InitialWeight = deviceSet.CrushInitialWeight + osdProps.storeConfig.PrimaryAffinity = deviceSet.CrushPrimaryAffinity // If OSD isn't portable, we're getting the host name either from the osd deployment that was already initialized // or from the osd prepare job from initial creation.