osd: enable filestore on device scenario to run with ceph image

update osd specs to be consistent with other daemons

Signed-off-by: travisn <tnielsen@redhat.com>
This commit is contained in:
travisn committed 2018-09-26 10:36:16 -06:00
1 parent aae5c97de3
commit 15f6ee5b14
6 files changed
+102 -59

No files matched your search

+11 -2
View File
@@ -40,7 +40,7 @@ var osdCmd = &cobra.Command{
Hidden: true,
}
var osdConfigCmd = &cobra.Command{
Use: "config",
Use: "init",
Short: "Updates ceph.conf for the osd",
Hidden: true,
}
@@ -60,6 +60,7 @@ var (
mountSourcePath string
mountPath string
osdID int
copyBinariesPath string
)
func addOSDFlags(command *cobra.Command) {
@@ -74,8 +75,9 @@ func addOSDFlags(command *cobra.Command) {
provisionCmd.Flags().BoolVar(&cfg.forceFormat, "force-format", false,
"true to force the format of any specified devices, even if they already have a filesystem. BE CAREFUL!")
// flags for generating the osd config file
// flags for generating the osd config
osdConfigCmd.Flags().IntVar(&osdID, "osd-id", -1, "osd id for which to generate config")
osdConfigCmd.Flags().StringVar(&copyBinariesPath, "copy-binaries-path", "", "If specified, copy the rook binaries to this path for use by the daemon container")
// flags for running filestore on a device
filestoreDeviceCmd.Flags().StringVar(&mountSourcePath, "source-path", "", "the source path of the device to mount")
@@ -172,6 +174,13 @@ func writeOSDConfig(cmd *cobra.Command, args []string) error {
if err := osddaemon.WriteConfigFile(context, &clusterInfo, kv, osdID, cfg.storeConfig, cfg.nodeName, crushLocation); err != nil {
logger.Errorf("failed to write osd config file. %+v", err)
}
if copyBinariesPath != "" {
if err := osddaemon.CopyBinariesForDaemon(copyBinariesPath); err != nil {
logger.Errorf("failed to copy rook binaries for filestore device. %+v", err)
} else {
logger.Infof("successfully copied rook binaries")
}
}
return nil
}
@@ -18,6 +18,8 @@ package osd
import (
"fmt"
"io"
"os"
"path"
"path/filepath"
@@ -75,3 +77,36 @@ func createOSDBootstrapKeyring(context *clusterd.Context, clusterName string) er
return cephconfig.CreateKeyring(context, clusterName, username, keyringPath, access, keyringEval)
}
// CopyBinariesForDaemon copies the "tini" and "rook" binaries to a shared volume at the target path.
// This is necessary for the filestore on a device scenario when rook needs to mount a directory
// in the same container as the ceph process so it can be unmounted upon exit.
func CopyBinariesForDaemon(target string) error {
if err := copyBinary("/usr/local/bin", target, "rook"); err != nil {
return err
}
return copyBinary("/", target, "tini")
}
func copyBinary(sourceDir, targetDir, filename string) error {
sourcePath := path.Join(sourceDir, filename)
targetPath := path.Join(targetDir, filename)
logger.Infof("copying %s to %s", sourcePath, targetPath)
sourceFile, err := os.Open(sourcePath)
if err != nil {
return err
}
defer sourceFile.Close()
destinationFile, err := os.Create(targetPath)
if err != nil {
return err
}
defer destinationFile.Close()
if _, err := io.Copy(destinationFile, sourceFile); err != nil {
return err
}
return os.Chmod(targetPath, 0755)
}
@@ -28,12 +28,11 @@ import (
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon"
"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
opspec "github.com/rook/rook/pkg/operator/ceph/spec"
"github.com/rook/rook/pkg/operator/k8sutil"
batch "k8s.io/api/batch/v1"
"k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/kubelet/apis"
)
@@ -77,19 +76,9 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
storeConfig config.StoreConfig, metadataDevice, location string, osd OSDInfo) (*extensions.Deployment, error) {
replicaCount := int32(1)
var volumeMounts []v1.VolumeMount
configVolumeMounts := []v1.VolumeMount{
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
k8sutil.ConfigOverrideMount(),
}
volumes := []v1.Volume{k8sutil.ConfigOverrideVolume()}
if c.dataDirHostPath != "" {
// the user has specified a host path to use for the data dir, use that instead
dataDirSource := v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: c.dataDirHostPath}}
volumes = append(volumes, v1.Volume{Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource})
} else {
logger.Infof("no data dir provided")
}
volumeMounts := opspec.CephVolumeMounts()
configVolumeMounts := opspec.RookVolumeMounts()
volumes := opspec.PodVolumes(c.dataDirHostPath)
var dataDir string
if osd.IsDirectory {
@@ -104,17 +93,16 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
dataDirSource := v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: parentDir}}
volumes = append(volumes, v1.Volume{Name: volumeName, VolumeSource: dataDirSource})
configVolumeMounts = append(configVolumeMounts, v1.VolumeMount{Name: volumeName, MountPath: parentDir})
volumeMounts = append(volumeMounts, v1.VolumeMount{Name: volumeName, MountPath: parentDir})
}
volumeMounts = configVolumeMounts
} else {
// Mount the required device
dataDir = k8sutil.DataDir
// create volume config for the data dir and /dev so the pod can access devices on the host
// Create volume config for /dev so the pod can access devices on the host
devVolume := v1.Volume{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}}
volumes = append(volumes, devVolume)
devMount := v1.VolumeMount{Name: "devices", MountPath: "/dev"}
volumeMounts = append(configVolumeMounts, devMount)
volumeMounts = append(volumeMounts, devMount)
}
if len(volumes) == 0 {
@@ -151,20 +139,37 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
var command []string
var args []string
if !osd.IsDirectory && osd.IsFileStore {
// filestore on a device requires indirection through the rook entrypoint so we can mount the image
// All scenarios except one can call the ceph-osd daemon directly. The one different scenario is when
// filestore is running on a device. Rook needs to mount the device, run the ceph-osd daemon, and then
// when the daemon exits, rook needs to unmount the device. Since rook needs to be in the container
// for this scenario, we will copy the binaries necessary to a mount, which will then be mounted
// to the daemon container.
sourcePath := path.Join("/dev/disk/by-partuuid", osd.DevicePartUUID)
mountPath := "/rook"
command = []string{path.Join(mountPath, "tini")}
args = append([]string{
"--", path.Join(mountPath, "rook"),
"ceph", "osd", "filestore-device",
"--source-path", sourcePath,
"--mount-path", osd.DataPath,
"--",
}, commonArgs...)
"--"},
commonArgs...)
// To get rook inside the container, the config init container needs to copy "tini" and "rook" binaries into a volume.
// Set the config flag so rook will copy the binaries.
configEnvVars = append(configEnvVars, v1.EnvVar{Name: "ROOK_COPY_BINARIES_PATH", Value: mountPath})
// Create the volume and mount that will be shared between the init container and the daemon container
volumeName := "rookbinaries"
binariesVolume := v1.Volume{Name: volumeName, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}
volumes = append(volumes, binariesVolume)
binariesMount := v1.VolumeMount{Name: volumeName, MountPath: mountPath}
volumeMounts = append(volumeMounts, binariesMount)
configVolumeMounts = append(configVolumeMounts, binariesMount)
} else {
// other osds can launch the osd daemon directly
command = append([]string{"/tini", "--", "ceph-osd",
fmt.Sprintf("--public-addr=$(%s)", k8sutil.PublicIPEnvVar),
fmt.Sprintf("--cluster-addr=$(%s)", k8sutil.PrivateIPEnvVar),
}, commonArgs...)
command = []string{"ceph-osd"}
args = commonArgs
}
privileged := true
@@ -213,8 +218,8 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
DNSPolicy: DNSPolicy,
InitContainers: []v1.Container{
{
Args: []string{"ceph", "osd", "config"},
Name: "osd-init-config",
Args: []string{"ceph", "osd", "init"},
Name: opspec.ConfigInitContainerName,
Image: k8sutil.MakeRookImage(c.Version),
VolumeMounts: configVolumeMounts,
Env: configEnvVars,
@@ -225,7 +230,7 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
{
Command: command,
Args: args,
Name: appName,
Name: "osd",
Image: k8sutil.MakeRookImage(c.Version),
VolumeMounts: volumeMounts,
Env: envVars,
@@ -246,13 +251,8 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
func (c *Cluster) provisionPodTemplateSpec(devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements,
storeConfig config.StoreConfig, metadataDevice, location string, restart v1.RestartPolicy) (*v1.PodTemplateSpec, error) {
volumes := []v1.Volume{k8sutil.ConfigOverrideVolume()}
if c.dataDirHostPath != "" {
// the user has specified a host path to use for the data dir, use that instead
dataDirSource := v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: c.dataDirHostPath}}
volumes = append(volumes, v1.Volume{Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource})
}
volumes := opspec.PodVolumes(c.dataDirHostPath)
// by default, don't define any volume config unless it is required
if len(devices) > 0 || selection.DeviceFilter != "" || selection.GetUseAllDevices() || metadataDevice != "" {
@@ -366,10 +366,7 @@ func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection ro
devMountNeeded = true
}
volumeMounts := []v1.VolumeMount{
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
k8sutil.ConfigOverrideMount(),
}
volumeMounts := opspec.CephVolumeMounts()
if devMountNeeded {
devMount := v1.VolumeMount{Name: "devices", MountPath: "/dev"}
volumeMounts = append(volumeMounts, devMount)
@@ -85,17 +85,17 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
assert.Equal(t, "node1", deployment.Spec.Template.Spec.NodeSelector[apis.LabelHostname])
assert.Equal(t, v1.RestartPolicyAlways, deployment.Spec.Template.Spec.RestartPolicy)
if devMountNeeded && len(dataDir) > 0 {
assert.Equal(t, 3, len(deployment.Spec.Template.Spec.Volumes))
assert.Equal(t, 4, len(deployment.Spec.Template.Spec.Volumes))
}
if devMountNeeded && len(dataDir) == 0 {
assert.Equal(t, 2, len(deployment.Spec.Template.Spec.Volumes))
assert.Equal(t, 4, len(deployment.Spec.Template.Spec.Volumes))
}
if !devMountNeeded && len(dataDir) > 0 {
assert.Equal(t, 2, len(deployment.Spec.Template.Spec.Volumes))
}
//assert.Equal(t, "rook-data", deployment.Spec.Template.Spec.Volumes[0].Name)
assert.Equal(t, "rook-config-override", deployment.Spec.Template.Spec.Volumes[0].Name)
assert.Equal(t, "rook-data", deployment.Spec.Template.Spec.Volumes[0].Name)
assert.Equal(t, "ceph-default-config-dir", deployment.Spec.Template.Spec.Volumes[1].Name)
assert.Equal(t, appName, deployment.Spec.Template.ObjectMeta.Name)
assert.Equal(t, appName, deployment.Spec.Template.ObjectMeta.Labels["app"])
@@ -105,8 +105,8 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
assert.Equal(t, 1, len(deployment.Spec.Template.Spec.InitContainers))
initCont := deployment.Spec.Template.Spec.InitContainers[0]
assert.Equal(t, "rook/rook:myversion", initCont.Image)
assert.Equal(t, "osd-init-config", initCont.Name)
assert.Equal(t, 2, len(initCont.VolumeMounts))
assert.Equal(t, "config-init", initCont.Name)
assert.Equal(t, 3, len(initCont.VolumeMounts))
assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers))
cont := deployment.Spec.Template.Spec.Containers[0]
@@ -116,7 +116,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
} else {
assert.Equal(t, 3, len(cont.VolumeMounts))
}
assert.Equal(t, "/tini", cont.Command[0])
assert.Equal(t, "ceph-osd", cont.Command[0])
}
func verifyEnvVar(t *testing.T, envVars []v1.EnvVar, expectedName, expectedValue string, expectedFound bool) {
@@ -163,20 +163,21 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) {
assert.Nil(t, err)
// pod spec should have a volume for the given dir in the main container and the init container
podSpec := deployment.Spec.Template.Spec
assert.Equal(t, 3, len(podSpec.Volumes))
assert.Equal(t, 4, len(podSpec.Volumes))
require.Equal(t, 1, len(podSpec.Containers))
cont := podSpec.Containers[0]
assert.Equal(t, 3, len(cont.VolumeMounts))
assert.Equal(t, "/var/lib/rook", cont.VolumeMounts[0].MountPath)
assert.Equal(t, "/etc/rook/config", cont.VolumeMounts[1].MountPath)
assert.Equal(t, "/etc/ceph", cont.VolumeMounts[1].MountPath)
assert.Equal(t, "/my/root/path", cont.VolumeMounts[2].MountPath)
require.Equal(t, 1, len(podSpec.InitContainers))
initCont := podSpec.InitContainers[0]
assert.Equal(t, 3, len(initCont.VolumeMounts))
assert.Equal(t, 4, len(initCont.VolumeMounts))
assert.Equal(t, "/var/lib/rook", initCont.VolumeMounts[0].MountPath)
assert.Equal(t, "/etc/rook/config", initCont.VolumeMounts[1].MountPath)
assert.Equal(t, "/my/root/path", initCont.VolumeMounts[2].MountPath)
assert.Equal(t, "/etc/ceph", initCont.VolumeMounts[1].MountPath)
assert.Equal(t, "/etc/rook/config", initCont.VolumeMounts[2].MountPath)
assert.Equal(t, "/my/root/path", initCont.VolumeMounts[3].MountPath)
// the default osd created on a node will be under /var/lib/rook, which won't need an extra mount
osd = OSDInfo{
@@ -189,20 +190,21 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) {
assert.Nil(t, err)
// pod spec should have a volume for the given dir in the main container and the init container
podSpec = deployment.Spec.Template.Spec
assert.Equal(t, 2, len(podSpec.Volumes))
assert.Equal(t, 3, len(podSpec.Volumes))
require.Equal(t, 1, len(podSpec.Containers))
cont = podSpec.Containers[0]
require.Equal(t, 2, len(cont.VolumeMounts))
assert.Equal(t, "/var/lib/rook", cont.VolumeMounts[0].MountPath)
assert.Equal(t, "/etc/rook/config", cont.VolumeMounts[1].MountPath)
assert.Equal(t, "/etc/ceph", cont.VolumeMounts[1].MountPath)
assert.Equal(t, (4 + len(k8sutil.ClusterDaemonEnvVars())), len(cont.Env))
require.Equal(t, 1, len(podSpec.InitContainers))
initCont = podSpec.InitContainers[0]
require.Equal(t, 2, len(initCont.VolumeMounts))
require.Equal(t, 3, len(initCont.VolumeMounts))
assert.Equal(t, "/var/lib/rook", initCont.VolumeMounts[0].MountPath)
assert.Equal(t, "/etc/rook/config", initCont.VolumeMounts[1].MountPath)
assert.Equal(t, "/etc/ceph", initCont.VolumeMounts[1].MountPath)
assert.Equal(t, "/etc/rook/config", initCont.VolumeMounts[2].MountPath)
}
func TestStorageSpecConfig(t *testing.T) {
+1 -1
View File
@@ -362,7 +362,7 @@ spec:
fieldPath: metadata.namespace`
}
//GetRookCluster returns rook-cluster manifest
//GetClusterRoles returns rook-cluster manifest
func (m *CephManifestsMaster) GetClusterRoles(namespace, systemNamespace string) string {
return `apiVersion: v1
kind: ServiceAccount
+1 -1
View File
@@ -434,7 +434,7 @@ func (k8sh *K8sHelper) WaitForPodCount(label, namespace string, count int) error
return fmt.Errorf("failed to find pod with label %s. %+v", label, err)
}
if len(pods.Items) == count {
if len(pods.Items) >= count {
logger.Infof("found %d pods with label %s", count, label)
return nil
}