From 0eee2b4dee069420ed5d893afbad4a99b99b5b26 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Thu, 5 Mar 2020 18:15:58 -0700 Subject: [PATCH 1/2] ceph: always mount /dev in the osd prepare pod The osd prepare job is now always working with devices so we always need to mount /dev and run privileged. There is no scenario for not mounting /dev since filestore support was removed. Signed-off-by: Travis Nielsen --- pkg/operator/ceph/cluster/osd/spec.go | 45 +++++++-------------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 5c5526160..e91bda701 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -19,7 +19,6 @@ package osd import ( "fmt" - "os" "path" "strconv" "strings" @@ -570,14 +569,11 @@ func (c *Cluster) provisionPodTemplateSpec(osdProps osdProperties, restart v1.Re // overrides will apply to ceph-volume, but this is unnecessary anyway volumes := append(controller.PodVolumes(provisionConfig.DataPathMap, c.dataDirHostPath, true), copyBinariesVolume) - // by default, don't define any volume config unless it is required - if len(osdProps.devices) > 0 || osdProps.selection.DeviceFilter != "" || osdProps.selection.DevicePathFilter != "" || osdProps.selection.GetUseAllDevices() || osdProps.metadataDevice != "" || osdOnPVC { - // create volume config for the data dir and /dev so the pod can access devices on the host - devVolume := v1.Volume{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}} - volumes = append(volumes, devVolume) - udevVolume := v1.Volume{Name: "udev", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/run/udev"}}} - volumes = append(volumes, udevVolume) - } + // create a volume on /dev so the pod can access devices on the host + devVolume := v1.Volume{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}} + volumes = append(volumes, devVolume) + udevVolume := v1.Volume{Name: "udev", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/run/udev"}}} + volumes = append(volumes, udevVolume) if osdOnPVC { // Create volume config for PVCs @@ -837,12 +833,6 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun osdOnPVCMetadata := osdProps.metadataPVC.ClaimName != "" envVars := c.getConfigEnvVars(osdProps, k8sutil.DataDir) - devMountNeeded := false - if osdOnPVC { - devMountNeeded = true - } - privileged := false - // only 1 of device list, device filter, device path filter and use all devices can be specified. We prioritize in that order. if len(osdProps.devices) > 0 { deviceNames := make([]string, len(osdProps.devices)) @@ -879,33 +869,24 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun deviceNames[i] = deviceID + devSuffix } envVars = append(envVars, dataDevicesEnvVar(strings.Join(deviceNames, ","))) - devMountNeeded = true } else if osdProps.selection.DeviceFilter != "" { envVars = append(envVars, deviceFilterEnvVar(osdProps.selection.DeviceFilter)) - devMountNeeded = true } else if osdProps.selection.DevicePathFilter != "" { envVars = append(envVars, devicePathFilterEnvVar(osdProps.selection.DevicePathFilter)) - devMountNeeded = true } else if osdProps.selection.GetUseAllDevices() { envVars = append(envVars, deviceFilterEnvVar("all")) - devMountNeeded = true } envVars = append(envVars, v1.EnvVar{Name: "ROOK_CEPH_VERSION", Value: c.clusterInfo.CephVersion.CephVersionFormatted()}) if osdProps.metadataDevice != "" { envVars = append(envVars, metadataDeviceEnvVar(osdProps.metadataDevice)) - devMountNeeded = true } - // ceph-volume is currently set up to use /etc/ceph/ceph.conf; this means no user config - // overrides will apply to ceph-volume, but this is unnecessary anyway - volumeMounts := append(controller.CephVolumeMounts(provisionConfig.DataPathMap, true), copyBinariesMount) - if devMountNeeded { - devMount := v1.VolumeMount{Name: "devices", MountPath: "/dev"} - volumeMounts = append(volumeMounts, devMount) - udevMount := v1.VolumeMount{Name: "udev", MountPath: "/run/udev"} - volumeMounts = append(volumeMounts, udevMount) - } + volumeMounts := append(controller.CephVolumeMounts(provisionConfig.DataPathMap, true), []v1.VolumeMount{ + {Name: "devices", MountPath: "/dev"}, + {Name: "udev", MountPath: "/run/udev"}, + copyBinariesMount, + }...) // If the OSD runs on PVC if osdOnPVC { @@ -921,10 +902,8 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun envVars = append(envVars, crushDeviceClassEnvVar(osdProps.crushDeviceClass)) } - // elevate to be privileged if it is going to mount devices or if running in a restricted environment such as openshift - if devMountNeeded || os.Getenv("ROOK_HOSTPATH_REQUIRES_PRIVILEGED") == "true" || osdOnPVC { - privileged = true - } + // run privileged always since we always mount /dev + privileged := true runAsUser := int64(0) runAsNonRoot := false readOnlyRootFilesystem := false From 69c9ed4206f47644687733396d87022e93d312a3 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Thu, 5 Mar 2020 18:26:32 -0700 Subject: [PATCH 2/2] ceph: storage warn iare specified but ignored In the cluster CR the nodes element will be ignored under storage when useAllNodes is true. The nodes will only be applied when useAllNodes is false. It is a common usability problem that users can't figure out why their nodes are not configured. At least this log entry can give them a clue. Signed-off-by: Travis Nielsen --- pkg/operator/ceph/cluster/osd/osd.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 74f99030c..176171b97 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -350,6 +350,10 @@ func (c *Cluster) startProvisioningOverNodes(config *provisionConfig) { } if c.DesiredStorage.UseAllNodes { + if len(c.DesiredStorage.Nodes) > 0 { + logger.Warningf("useAllNodes is TRUE, but nodes are specified. NODES in the cluster CR will be IGNORED unless useAllNodes is FALSE.") + } + // Get the list of all nodes in the cluster. The placement settings will be applied below. hostnameMap, err := k8sutil.GetNodeHostNames(c.context.Clientset) if err != nil {