Merge pull request #6303 from subhamkrai/tunefast-deviceclass

ceph: tune fast device class for OSD on PVC in the Azure

Documentation/ceph-cluster-crd.md

@@ -69,7 +69,6 @@ spec:
     - name: set1
       count: 3
       portable: false
-      tuneDeviceClass: false
       encrypted: false
       volumeClaimTemplates:
       - metadata:
@@ -314,7 +313,8 @@ The following are the settings for Storage Class Device Sets which can be config
 * `preparePlacement`: The placement criteria for the preparation of the OSD devices. Creating OSDs is a two-step process and the prepare job may require different placement than the OSD daemons. If the `preparePlacement` is not specified, the `placement` will instead be applied for consistent placement for the OSD prepare jobs and OSD deployments. The `preparePlacement` is only useful for `portable` OSDs in the device sets. OSDs that are not portable will be tied to the host where the OSD prepare job initially runs.
    * For example, provisioning may require topology spread constraints across zones, but the OSD daemons may require constraints across hosts within the zones.
 * `portable`: If `true`, the OSDs will be allowed to move between nodes during failover. This requires a storage class that supports portability (e.g. `aws-ebs`, but not the local storage provisioner). If `false`, the OSDs will be assigned to a node permanently. Rook will configure Ceph's CRUSH map to support the portability.
-* `tuneDeviceClass`: If `true`, because the OSD can be on a slow device class, Rook will adapt to that by tuning the OSD process. This will make Ceph perform better under that slow device.
+* `tuneDeviceClass`: For example, Ceph cannot detect AWS volumes as HDDs from the storage class "gp2", so you can improve Ceph performance by setting this to true.
+* `tuneFastDeviceClass`: For example, Ceph cannot detect Azure disks as SSDs from the storage class "managed-premium", so you can improve Ceph performance by setting this to true..
 * `volumeClaimTemplates`: A list of PVC templates to use for provisioning the underlying storage devices.
   * `resources.requests.storage`: The desired capacity for the underlying storage devices.
   * `storageClassName`: The StorageClass to provision PVCs from. Default would be to use the cluster-default StorageClass. This StorageClass should provide a raw block device, multipath device, or logical volume. Other types are not supported. If you want to use logical volume, please see [known issue of OSD on LV-backed PVC](ceph-common-issues.md#lvm-metadata-can-be-corrupted-with-osd-on-lv-backed-pvc)
@@ -810,7 +810,6 @@ spec:
     - name: set1
       count: 3
       portable: false
-      tuneDeviceClass: false
       resources:
         limits:
           cpu: "500m"
@@ -864,7 +863,6 @@ So just taking the `storage` section this will give something like:
     - name: set1
       count: 3
       portable: false
-      tuneDeviceClass: false
       volumeClaimTemplates:
       - metadata:
           name: data
@@ -909,7 +907,6 @@ The bluestore partition has the following reference combinations supported by th
       - name: set1
         count: 3
         portable: false
-        tuneDeviceClass: false
         volumeClaimTemplates:
         - metadata:
             name: data
@@ -932,7 +929,6 @@ The bluestore partition has the following reference combinations supported by th
       - name: set1
         count: 3
         portable: false
-        tuneDeviceClass: false
         volumeClaimTemplates:
         - metadata:
             name: data
@@ -969,7 +965,6 @@ There is no separate "metadata" device in this case, the data of main OSD block
       - name: set1
         count: 3
         portable: false
-        tuneDeviceClass: false
         volumeClaimTemplates:
         - metadata:
             name: data
@@ -1004,7 +999,6 @@ There is no separate "metadata" device in this case, the data of main OSD block
       - name: set1
         count: 3
         portable: false
-        tuneDeviceClass: false
         volumeClaimTemplates:
         - metadata:
             name: data

cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml

@@ -58,6 +58,10 @@ spec:
       # Rook can configure the OSD running on PVC to accommodate that by tuning some of the Ceph internal
       # Currently, "gp2" has been identified as such
       tuneDeviceClass: true
+      # Certain storage class in the Cloud are fast
+      # Rook can configure the OSD running on PVC to accommodate that by tuning some of the Ceph internal
+      # Currently, "managed-premium" has been identified as such
+      tuneFastDeviceClass: false
       # whether to encrypt the deviceSet or not
       encrypted: false
       # Since the OSDs could end up on any node, an effort needs to be made to spread the OSDs

pkg/apis/rook.io/v1/types.go

@@ -125,6 +125,7 @@ type StorageClassDeviceSet struct {
 	VolumeClaimTemplates []v1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"` // List of PVC templates for the underlying storage devices
 	Portable             bool                       `json:"portable,omitempty"`             // OSD portability across the hosts
 	TuneSlowDeviceClass  bool                       `json:"tuneDeviceClass,omitempty"`      // TuneSlowDeviceClass Tune the OSD when running on a slow Device Class
+	TuneFastDeviceClass  bool                       `json:"tuneFastDeviceClass,omitempty"`  // TuneFastDeviceClass Tune the OSD when running on a fast Device Class
 	SchedulerName        string                     `json:"schedulerName,omitempty"`        // Scheduler name for OSD pod placement
 	Encrypted            bool                       `json:"encrypted,omitempty"`            // Whether to encrypt the deviceSet
 }
@@ -137,10 +138,11 @@ type VolumeSource struct {
 	Placement           Placement                                       `json:"placement,omitempty"`
 	PreparePlacement    *Placement                                      `json:"preparePlacement,omitempty"`
 	Config              map[string]string                               `json:"config,omitempty"`
-	Portable            bool                                            `json:"portable,omitempty"`         // Portable OSD portability across the hosts
-	TuneSlowDeviceClass bool                                            `json:"tuneDeviceClass,omitempty"`  // TuneSlowDeviceClass Tune the OSD when running on a slow Device Class
-	SchedulerName       string                                          `json:"schedulerName,omitempty"`    // Scheduler name for OSD pod placement
-	CrushDeviceClass    string                                          `json:"crushDeviceClass,omitempty"` // CrushDeviceClass represents the crush device class for an OSD
-	Size                string                                          `json:"size,omitempty"`             // Size represents the size requested for the PVC
-	Encrypted           bool                                            `json:"encrypted,omitempty"`        // Whether to encrypt the deviceSet
+	Portable            bool                                            `json:"portable,omitempty"`            // Portable OSD portability across the hosts
+	TuneSlowDeviceClass bool                                            `json:"tuneDeviceClass,omitempty"`     // TuneSlowDeviceClass Tune the OSD when running on a slow Device Class
+	TuneFastDeviceClass bool                                            `json:"tuneFastDeviceClass,omitempty"` // TuneFastDeviceClass Tune the OSD when running on a Fast Device Class
+	SchedulerName       string                                          `json:"schedulerName,omitempty"`       // Scheduler name for OSD pod placement
+	CrushDeviceClass    string                                          `json:"crushDeviceClass,omitempty"`    // CrushDeviceClass represents the crush device class for an OSD
+	Size                string                                          `json:"size,omitempty"`                // Size represents the size requested for the PVC
+	Encrypted           bool                                            `json:"encrypted,omitempty"`           // Whether to encrypt the deviceSet
 }

pkg/operator/ceph/cluster/osd/config.go

@@ -37,11 +37,6 @@ const (
 key = %s
 `
 
-	// OSDs on PVC using a certain storage class need to do some tuning
-	osdRecoverySleep = "0.1"
-	osdSnapTrimSleep = "2"
-	osdDeleteSleep   = "2"
-
 	// OsdEncryptionSecretNameKeyName is the key name of the Secret that contains the OSD encryption key
 	// #nosec G101 since this is not leaking any hardcoded credentials, it's just the secret key name
 	OsdEncryptionSecretNameKeyName = "dmcrypt-key"
@@ -68,29 +63,6 @@ func (c *Cluster) generateKeyring(osdID int) (string, error) {
 	return keyring, s.CreateOrUpdate(deploymentName, keyring)
 }
 
-func (c *Cluster) osdRunFlagTuningOnPVC(osdID int) error {
-	who := fmt.Sprintf("osd.%d", osdID)
-	do := make(map[string]string)
-
-	// Time in seconds to sleep before next recovery or backfill op
-	do["osd_recovery_sleep"] = osdRecoverySleep
-	// Time in seconds to sleep before next snap trim
-	do["osd_snap_trim_sleep"] = osdSnapTrimSleep
-	// Time in seconds to sleep before next removal transaction
-	do["osd_delete_sleep"] = osdDeleteSleep
-
-	monStore := opconfig.GetMonStore(c.context, c.clusterInfo)
-
-	for flag, val := range do {
-		err := monStore.Set(who, flag, val)
-		if err != nil {
-			return errors.Wrapf(err, "failed to set %q to %q on %q", flag, val, who)
-		}
-	}
-
-	return nil
-}
-
 // PrivilegedContext returns a privileged Pod security context
 func PrivilegedContext() *v1.SecurityContext {
 	privileged := true

pkg/operator/ceph/cluster/osd/deviceSet.go

@@ -94,6 +94,7 @@ func (c *Cluster) prepareStorageClassDeviceSets(config *provisionConfig) []rookv
 				PVCSources:          pvcSources,
 				Portable:            storageClassDeviceSet.Portable,
 				TuneSlowDeviceClass: storageClassDeviceSet.TuneSlowDeviceClass,
+				TuneFastDeviceClass: storageClassDeviceSet.TuneFastDeviceClass,
 				SchedulerName:       storageClassDeviceSet.SchedulerName,
 				CrushDeviceClass:    crushDeviceClass,
 				Encrypted:           storageClassDeviceSet.Encrypted,

pkg/operator/ceph/cluster/osd/osd.go

@@ -135,6 +135,7 @@ type osdProperties struct {
 	metadataDevice      string
 	portable            bool
 	tuneSlowDeviceClass bool
+	tuneFastDeviceClass bool
 	schedulerName       string
 	crushDeviceClass    string
 	encrypted           bool
@@ -705,6 +706,7 @@ func (c *Cluster) getOSDPropsForPVC(pvcName string) (osdProperties, error) {
 				preparePlacement:    volumeSource.PreparePlacement,
 				portable:            volumeSource.Portable,
 				tuneSlowDeviceClass: volumeSource.TuneSlowDeviceClass,
+				tuneFastDeviceClass: volumeSource.TuneFastDeviceClass,
 				pvcSize:             volumeSource.Size,
 				schedulerName:       volumeSource.SchedulerName,
 				encrypted:           volumeSource.Encrypted,

pkg/operator/ceph/cluster/osd/spec.go

@@ -146,6 +146,30 @@ fi
 `
 )
 
+// OSDs on PVC using a certain fast storage class need to do some tuning
+var defaultTuneFastSettings = map[string]string{
+	"osd_op_num_threads_per_shard_ssd":        "2",          // Default value of osd_op_num_threads_per_shard for SSDs
+	"osd_op_num_shards_ssd":                   "8",          // Default value of osd_op_num_shards for SSDs
+	"osd_recovery_sleep_ssd":                  "0",          // Time in seconds to sleep before next recovery or backfill op for SSDs
+	"osd_snap_trim_sleep_ssd":                 "0",          // Time in seconds to sleep before next snap trim for SSDs
+	"osd_delete_sleep_ssd":                    "0",          // Time in seconds to sleep before next removal transaction for SSDs
+	"bluestore_min_alloc_size_ssd":            "4096",       // Default min_alloc_size value for SSDs
+	"bluestore_prefer_deferred_size_ssd":      "0",          // Default value of bluestore_prefer_deferred_size for SSDs
+	"bluestore_compression_min_blob_size_ssd": "8912",       // Default value of bluestore_compression_min_blob_size for SSDs
+	"bluestore_compression_max_blob_size_ssd": "65536",      // Default value of bluestore_compression_max_blob_size for SSDs
+	"bluestore_max_blob_size_ssd":             "65536",      // Default value of bluestore_max_blob_size for SSDs
+	"bluestore_cache_size_ssd":                "3221225472", // Default value of bluestore_cache_size for SSDs
+	"bluestore_throttle_cost_per_io_ssd":      "4000",       // Default value of bluestore_throttle_cost_per_io for SSDs
+	"bluestore_deferred_batch_ops_ssd":        "16",         // Default value of bluestore_deferred_batch_ops for SSDs
+}
+
+// OSDs on PVC using a certain slow storage class need to do some tuning
+var defaultTuneSlowSettings = map[string]string{
+	"osd_recovery_sleep":  "0.1", // Time in seconds to sleep before next recovery or backfill op
+	"osd_snap_trim_sleep": "2",   // Time in seconds to sleep before next snap trim
+	"osd_delete_sleep":    "2",   // Time in seconds to sleep before next removal transaction
+}
+
 func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionConfig *provisionConfig) (*apps.Deployment, error) {
 	// If running on Octopus, we don't need to use the host PID namespace
 	var hostPID = !c.clusterInfo.CephVersion.IsAtLeastOctopus()
@@ -220,20 +244,6 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
 		{Name: "ROOK_IS_DEVICE", Value: "true"},
 	}...)
 
-	// If the OSD runs on PVC
-	if osdProps.onPVC() {
-		// add the PVC size to the pod spec so that if the size changes the OSD will be restarted and pick up the change
-		envVars = append(envVars, v1.EnvVar{Name: "ROOK_OSD_PVC_SIZE", Value: osdProps.pvcSize})
-
-		// Append tuning flag if necessary
-		if osdProps.tuneSlowDeviceClass {
-			err := c.osdRunFlagTuningOnPVC(osd.ID)
-			if err != nil {
-				return nil, errors.Wrapf(err, "failed to apply tuning on osd %q", strconv.Itoa(osd.ID))
-			}
-		}
-	}
-
 	var command []string
 	var args []string
 	// If the OSD was prepared with ceph-volume and running on PVC and using the LVM mode
@@ -279,6 +289,23 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
 		}
 	}
 
+	// If the OSD runs on PVC
+	if osdProps.onPVC() {
+		// add the PVC size to the pod spec so that if the size changes the OSD will be restarted and pick up the change
+		envVars = append(envVars, v1.EnvVar{Name: "ROOK_OSD_PVC_SIZE", Value: osdProps.pvcSize})
+
+		// Append slow tuning flag if necessary
+		if osdProps.tuneSlowDeviceClass {
+			for flag, val := range defaultTuneSlowSettings {
+				args = append(args, opconfig.NewFlag(flag, val))
+			}
+		} else if osdProps.tuneFastDeviceClass { // Append fast tuning flag if necessary
+			for flag, val := range defaultTuneFastSettings {
+				args = append(args, opconfig.NewFlag(flag, val))
+			}
+		}
+	}
+
 	// The osd itself needs to talk to udev to report information about the device (vendor/serial etc)
 	udevVolume, udevVolumeMount := getUdevVolume()
 	volumes = append(volumes, udevVolume)

pkg/operator/ceph/cluster/osd/spec_test.go

@@ -331,6 +331,22 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
 	assert.Equal(t, 1, len(blkInitCont.VolumeDevices))
 	blkMetaInitCont = deployment.Spec.Template.Spec.InitContainers[8]
 	assert.Equal(t, 1, len(blkMetaInitCont.VolumeDevices))
+
+	// Test tune Fast settings when OSD on PVC
+	osdProp.tuneFastDeviceClass = true
+	deployment, err = c.makeDeployment(osdProp, osd, dataPathMap)
+	assert.NoError(t, err)
+	for flag, val := range defaultTuneFastSettings {
+		assert.Contains(t, deployment.Spec.Template.Spec.Containers[0].Args, opconfig.NewFlag(flag, val))
+	}
+
+	// Test tune Slow settings when OSD on PVC
+	osdProp.tuneSlowDeviceClass = true
+	deployment, err = c.makeDeployment(osdProp, osd, dataPathMap)
+	assert.NoError(t, err)
+	for flag, val := range defaultTuneSlowSettings {
+		assert.Contains(t, deployment.Spec.Template.Spec.Containers[0].Args, opconfig.NewFlag(flag, val))
+	}
 }
 
 func verifyEnvVar(t *testing.T, envVars []v1.EnvVar, expectedName, expectedValue string, expectedFound bool) {