diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index 0005d2d8a..1cb8df45b 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -69,7 +69,6 @@ spec: - name: set1 count: 3 portable: false - tuneDeviceClass: false encrypted: false volumeClaimTemplates: - metadata: @@ -314,7 +313,8 @@ The following are the settings for Storage Class Device Sets which can be config * `preparePlacement`: The placement criteria for the preparation of the OSD devices. Creating OSDs is a two-step process and the prepare job may require different placement than the OSD daemons. If the `preparePlacement` is not specified, the `placement` will instead be applied for consistent placement for the OSD prepare jobs and OSD deployments. The `preparePlacement` is only useful for `portable` OSDs in the device sets. OSDs that are not portable will be tied to the host where the OSD prepare job initially runs. * For example, provisioning may require topology spread constraints across zones, but the OSD daemons may require constraints across hosts within the zones. * `portable`: If `true`, the OSDs will be allowed to move between nodes during failover. This requires a storage class that supports portability (e.g. `aws-ebs`, but not the local storage provisioner). If `false`, the OSDs will be assigned to a node permanently. Rook will configure Ceph's CRUSH map to support the portability. -* `tuneDeviceClass`: If `true`, because the OSD can be on a slow device class, Rook will adapt to that by tuning the OSD process. This will make Ceph perform better under that slow device. +* `tuneDeviceClass`: For example, Ceph cannot detect AWS volumes as HDDs from the storage class "gp2", so you can improve Ceph performance by setting this to true. +* `tuneFastDeviceClass`: For example, Ceph cannot detect Azure disks as SSDs from the storage class "managed-premium", so you can improve Ceph performance by setting this to true.. * `volumeClaimTemplates`: A list of PVC templates to use for provisioning the underlying storage devices. * `resources.requests.storage`: The desired capacity for the underlying storage devices. * `storageClassName`: The StorageClass to provision PVCs from. Default would be to use the cluster-default StorageClass. This StorageClass should provide a raw block device, multipath device, or logical volume. Other types are not supported. If you want to use logical volume, please see [known issue of OSD on LV-backed PVC](ceph-common-issues.md#lvm-metadata-can-be-corrupted-with-osd-on-lv-backed-pvc) @@ -810,7 +810,6 @@ spec: - name: set1 count: 3 portable: false - tuneDeviceClass: false resources: limits: cpu: "500m" @@ -864,7 +863,6 @@ So just taking the `storage` section this will give something like: - name: set1 count: 3 portable: false - tuneDeviceClass: false volumeClaimTemplates: - metadata: name: data @@ -909,7 +907,6 @@ The bluestore partition has the following reference combinations supported by th - name: set1 count: 3 portable: false - tuneDeviceClass: false volumeClaimTemplates: - metadata: name: data @@ -932,7 +929,6 @@ The bluestore partition has the following reference combinations supported by th - name: set1 count: 3 portable: false - tuneDeviceClass: false volumeClaimTemplates: - metadata: name: data @@ -969,7 +965,6 @@ There is no separate "metadata" device in this case, the data of main OSD block - name: set1 count: 3 portable: false - tuneDeviceClass: false volumeClaimTemplates: - metadata: name: data @@ -1004,7 +999,6 @@ There is no separate "metadata" device in this case, the data of main OSD block - name: set1 count: 3 portable: false - tuneDeviceClass: false volumeClaimTemplates: - metadata: name: data diff --git a/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml b/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml index f44b9762c..f383ed17e 100644 --- a/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml +++ b/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml @@ -58,6 +58,10 @@ spec: # Rook can configure the OSD running on PVC to accommodate that by tuning some of the Ceph internal # Currently, "gp2" has been identified as such tuneDeviceClass: true + # Certain storage class in the Cloud are fast + # Rook can configure the OSD running on PVC to accommodate that by tuning some of the Ceph internal + # Currently, "managed-premium" has been identified as such + tuneFastDeviceClass: false # whether to encrypt the deviceSet or not encrypted: false # Since the OSDs could end up on any node, an effort needs to be made to spread the OSDs diff --git a/pkg/apis/rook.io/v1/types.go b/pkg/apis/rook.io/v1/types.go index 0193f16f3..00c7430f0 100644 --- a/pkg/apis/rook.io/v1/types.go +++ b/pkg/apis/rook.io/v1/types.go @@ -125,6 +125,7 @@ type StorageClassDeviceSet struct { VolumeClaimTemplates []v1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"` // List of PVC templates for the underlying storage devices Portable bool `json:"portable,omitempty"` // OSD portability across the hosts TuneSlowDeviceClass bool `json:"tuneDeviceClass,omitempty"` // TuneSlowDeviceClass Tune the OSD when running on a slow Device Class + TuneFastDeviceClass bool `json:"tuneFastDeviceClass,omitempty"` // TuneFastDeviceClass Tune the OSD when running on a fast Device Class SchedulerName string `json:"schedulerName,omitempty"` // Scheduler name for OSD pod placement Encrypted bool `json:"encrypted,omitempty"` // Whether to encrypt the deviceSet } @@ -137,10 +138,11 @@ type VolumeSource struct { Placement Placement `json:"placement,omitempty"` PreparePlacement *Placement `json:"preparePlacement,omitempty"` Config map[string]string `json:"config,omitempty"` - Portable bool `json:"portable,omitempty"` // Portable OSD portability across the hosts - TuneSlowDeviceClass bool `json:"tuneDeviceClass,omitempty"` // TuneSlowDeviceClass Tune the OSD when running on a slow Device Class - SchedulerName string `json:"schedulerName,omitempty"` // Scheduler name for OSD pod placement - CrushDeviceClass string `json:"crushDeviceClass,omitempty"` // CrushDeviceClass represents the crush device class for an OSD - Size string `json:"size,omitempty"` // Size represents the size requested for the PVC - Encrypted bool `json:"encrypted,omitempty"` // Whether to encrypt the deviceSet + Portable bool `json:"portable,omitempty"` // Portable OSD portability across the hosts + TuneSlowDeviceClass bool `json:"tuneDeviceClass,omitempty"` // TuneSlowDeviceClass Tune the OSD when running on a slow Device Class + TuneFastDeviceClass bool `json:"tuneFastDeviceClass,omitempty"` // TuneFastDeviceClass Tune the OSD when running on a Fast Device Class + SchedulerName string `json:"schedulerName,omitempty"` // Scheduler name for OSD pod placement + CrushDeviceClass string `json:"crushDeviceClass,omitempty"` // CrushDeviceClass represents the crush device class for an OSD + Size string `json:"size,omitempty"` // Size represents the size requested for the PVC + Encrypted bool `json:"encrypted,omitempty"` // Whether to encrypt the deviceSet } diff --git a/pkg/operator/ceph/cluster/osd/config.go b/pkg/operator/ceph/cluster/osd/config.go index de8ad888b..cbc952f61 100644 --- a/pkg/operator/ceph/cluster/osd/config.go +++ b/pkg/operator/ceph/cluster/osd/config.go @@ -37,11 +37,6 @@ const ( key = %s ` - // OSDs on PVC using a certain storage class need to do some tuning - osdRecoverySleep = "0.1" - osdSnapTrimSleep = "2" - osdDeleteSleep = "2" - // OsdEncryptionSecretNameKeyName is the key name of the Secret that contains the OSD encryption key // #nosec G101 since this is not leaking any hardcoded credentials, it's just the secret key name OsdEncryptionSecretNameKeyName = "dmcrypt-key" @@ -68,29 +63,6 @@ func (c *Cluster) generateKeyring(osdID int) (string, error) { return keyring, s.CreateOrUpdate(deploymentName, keyring) } -func (c *Cluster) osdRunFlagTuningOnPVC(osdID int) error { - who := fmt.Sprintf("osd.%d", osdID) - do := make(map[string]string) - - // Time in seconds to sleep before next recovery or backfill op - do["osd_recovery_sleep"] = osdRecoverySleep - // Time in seconds to sleep before next snap trim - do["osd_snap_trim_sleep"] = osdSnapTrimSleep - // Time in seconds to sleep before next removal transaction - do["osd_delete_sleep"] = osdDeleteSleep - - monStore := opconfig.GetMonStore(c.context, c.clusterInfo) - - for flag, val := range do { - err := monStore.Set(who, flag, val) - if err != nil { - return errors.Wrapf(err, "failed to set %q to %q on %q", flag, val, who) - } - } - - return nil -} - // PrivilegedContext returns a privileged Pod security context func PrivilegedContext() *v1.SecurityContext { privileged := true diff --git a/pkg/operator/ceph/cluster/osd/deviceSet.go b/pkg/operator/ceph/cluster/osd/deviceSet.go index 2abf4a1f7..1585a51e5 100644 --- a/pkg/operator/ceph/cluster/osd/deviceSet.go +++ b/pkg/operator/ceph/cluster/osd/deviceSet.go @@ -94,6 +94,7 @@ func (c *Cluster) prepareStorageClassDeviceSets(config *provisionConfig) []rookv PVCSources: pvcSources, Portable: storageClassDeviceSet.Portable, TuneSlowDeviceClass: storageClassDeviceSet.TuneSlowDeviceClass, + TuneFastDeviceClass: storageClassDeviceSet.TuneFastDeviceClass, SchedulerName: storageClassDeviceSet.SchedulerName, CrushDeviceClass: crushDeviceClass, Encrypted: storageClassDeviceSet.Encrypted, diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index dd0b0eb85..20cb6f2a5 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -135,6 +135,7 @@ type osdProperties struct { metadataDevice string portable bool tuneSlowDeviceClass bool + tuneFastDeviceClass bool schedulerName string crushDeviceClass string encrypted bool @@ -705,6 +706,7 @@ func (c *Cluster) getOSDPropsForPVC(pvcName string) (osdProperties, error) { preparePlacement: volumeSource.PreparePlacement, portable: volumeSource.Portable, tuneSlowDeviceClass: volumeSource.TuneSlowDeviceClass, + tuneFastDeviceClass: volumeSource.TuneFastDeviceClass, pvcSize: volumeSource.Size, schedulerName: volumeSource.SchedulerName, encrypted: volumeSource.Encrypted, diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index e78a05e58..d27923d6c 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -146,6 +146,30 @@ fi ` ) +// OSDs on PVC using a certain fast storage class need to do some tuning +var defaultTuneFastSettings = map[string]string{ + "osd_op_num_threads_per_shard_ssd": "2", // Default value of osd_op_num_threads_per_shard for SSDs + "osd_op_num_shards_ssd": "8", // Default value of osd_op_num_shards for SSDs + "osd_recovery_sleep_ssd": "0", // Time in seconds to sleep before next recovery or backfill op for SSDs + "osd_snap_trim_sleep_ssd": "0", // Time in seconds to sleep before next snap trim for SSDs + "osd_delete_sleep_ssd": "0", // Time in seconds to sleep before next removal transaction for SSDs + "bluestore_min_alloc_size_ssd": "4096", // Default min_alloc_size value for SSDs + "bluestore_prefer_deferred_size_ssd": "0", // Default value of bluestore_prefer_deferred_size for SSDs + "bluestore_compression_min_blob_size_ssd": "8912", // Default value of bluestore_compression_min_blob_size for SSDs + "bluestore_compression_max_blob_size_ssd": "65536", // Default value of bluestore_compression_max_blob_size for SSDs + "bluestore_max_blob_size_ssd": "65536", // Default value of bluestore_max_blob_size for SSDs + "bluestore_cache_size_ssd": "3221225472", // Default value of bluestore_cache_size for SSDs + "bluestore_throttle_cost_per_io_ssd": "4000", // Default value of bluestore_throttle_cost_per_io for SSDs + "bluestore_deferred_batch_ops_ssd": "16", // Default value of bluestore_deferred_batch_ops for SSDs +} + +// OSDs on PVC using a certain slow storage class need to do some tuning +var defaultTuneSlowSettings = map[string]string{ + "osd_recovery_sleep": "0.1", // Time in seconds to sleep before next recovery or backfill op + "osd_snap_trim_sleep": "2", // Time in seconds to sleep before next snap trim + "osd_delete_sleep": "2", // Time in seconds to sleep before next removal transaction +} + func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionConfig *provisionConfig) (*apps.Deployment, error) { // If running on Octopus, we don't need to use the host PID namespace var hostPID = !c.clusterInfo.CephVersion.IsAtLeastOctopus() @@ -220,20 +244,6 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC {Name: "ROOK_IS_DEVICE", Value: "true"}, }...) - // If the OSD runs on PVC - if osdProps.onPVC() { - // add the PVC size to the pod spec so that if the size changes the OSD will be restarted and pick up the change - envVars = append(envVars, v1.EnvVar{Name: "ROOK_OSD_PVC_SIZE", Value: osdProps.pvcSize}) - - // Append tuning flag if necessary - if osdProps.tuneSlowDeviceClass { - err := c.osdRunFlagTuningOnPVC(osd.ID) - if err != nil { - return nil, errors.Wrapf(err, "failed to apply tuning on osd %q", strconv.Itoa(osd.ID)) - } - } - } - var command []string var args []string // If the OSD was prepared with ceph-volume and running on PVC and using the LVM mode @@ -279,6 +289,23 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC } } + // If the OSD runs on PVC + if osdProps.onPVC() { + // add the PVC size to the pod spec so that if the size changes the OSD will be restarted and pick up the change + envVars = append(envVars, v1.EnvVar{Name: "ROOK_OSD_PVC_SIZE", Value: osdProps.pvcSize}) + + // Append slow tuning flag if necessary + if osdProps.tuneSlowDeviceClass { + for flag, val := range defaultTuneSlowSettings { + args = append(args, opconfig.NewFlag(flag, val)) + } + } else if osdProps.tuneFastDeviceClass { // Append fast tuning flag if necessary + for flag, val := range defaultTuneFastSettings { + args = append(args, opconfig.NewFlag(flag, val)) + } + } + } + // The osd itself needs to talk to udev to report information about the device (vendor/serial etc) udevVolume, udevVolumeMount := getUdevVolume() volumes = append(volumes, udevVolume) diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index 236924dc6..ed9dd5e28 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -331,6 +331,22 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, 1, len(blkInitCont.VolumeDevices)) blkMetaInitCont = deployment.Spec.Template.Spec.InitContainers[8] assert.Equal(t, 1, len(blkMetaInitCont.VolumeDevices)) + + // Test tune Fast settings when OSD on PVC + osdProp.tuneFastDeviceClass = true + deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) + assert.NoError(t, err) + for flag, val := range defaultTuneFastSettings { + assert.Contains(t, deployment.Spec.Template.Spec.Containers[0].Args, opconfig.NewFlag(flag, val)) + } + + // Test tune Slow settings when OSD on PVC + osdProp.tuneSlowDeviceClass = true + deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) + assert.NoError(t, err) + for flag, val := range defaultTuneSlowSettings { + assert.Contains(t, deployment.Spec.Template.Spec.Containers[0].Args, opconfig.NewFlag(flag, val)) + } } func verifyEnvVar(t *testing.T, envVars []v1.EnvVar, expectedName, expectedValue string, expectedFound bool) {