ceph: allow OSDs on both pvcs and devices

The OSD configuration on local devices was being skipped if a
storageClassDeviceSet was specified. Both types of OSDs should
be allowed in the same cluster, which allows a cluster in the cloud
to also take advantage of local devices with different perf
properties.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>

pkg/operator/ceph/cluster/osd/spec.go

@@ -238,10 +238,11 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
 	volumeMounts := controller.CephVolumeMounts(provisionConfig.DataPathMap, false)
 	configVolumeMounts := controller.RookVolumeMounts(provisionConfig.DataPathMap, false)
 	// When running on PVC, the OSDs don't need a bindmount on dataDirHostPath, only the monitors do
+	dataDirHostPath := c.spec.DataDirHostPath
 	if osdProps.onPVC() {
-		c.spec.DataDirHostPath = ""
+		dataDirHostPath = ""
 	}
-	volumes := controller.PodVolumes(provisionConfig.DataPathMap, c.spec.DataDirHostPath, false)
+	volumes := controller.PodVolumes(provisionConfig.DataPathMap, dataDirHostPath, false)
 	failureDomainValue := osdProps.crushHostname
 	doConfigInit := true       // initialize ceph.conf in init container?
 	doBinaryCopyInit := true   // copy tini and rook binaries in an init container?