osd: enable filestore on device scenario to run with ceph image
update osd specs to be consistent with other daemons Signed-off-by: travisn <tnielsen@redhat.com>
This commit is contained in:
6 files changed
+102
-59
No files matched your search
+11
-2
@@ -40,7 +40,7 @@ var osdCmd = &cobra.Command{
|
||||
Hidden: true,
|
||||
}
|
||||
var osdConfigCmd = &cobra.Command{
|
||||
Use: "config",
|
||||
Use: "init",
|
||||
Short: "Updates ceph.conf for the osd",
|
||||
Hidden: true,
|
||||
}
|
||||
@@ -60,6 +60,7 @@ var (
|
||||
mountSourcePath string
|
||||
mountPath string
|
||||
osdID int
|
||||
copyBinariesPath string
|
||||
)
|
||||
|
||||
func addOSDFlags(command *cobra.Command) {
|
||||
@@ -74,8 +75,9 @@ func addOSDFlags(command *cobra.Command) {
|
||||
provisionCmd.Flags().BoolVar(&cfg.forceFormat, "force-format", false,
|
||||
"true to force the format of any specified devices, even if they already have a filesystem. BE CAREFUL!")
|
||||
|
||||
// flags for generating the osd config file
|
||||
// flags for generating the osd config
|
||||
osdConfigCmd.Flags().IntVar(&osdID, "osd-id", -1, "osd id for which to generate config")
|
||||
osdConfigCmd.Flags().StringVar(©BinariesPath, "copy-binaries-path", "", "If specified, copy the rook binaries to this path for use by the daemon container")
|
||||
|
||||
// flags for running filestore on a device
|
||||
filestoreDeviceCmd.Flags().StringVar(&mountSourcePath, "source-path", "", "the source path of the device to mount")
|
||||
@@ -172,6 +174,13 @@ func writeOSDConfig(cmd *cobra.Command, args []string) error {
|
||||
if err := osddaemon.WriteConfigFile(context, &clusterInfo, kv, osdID, cfg.storeConfig, cfg.nodeName, crushLocation); err != nil {
|
||||
logger.Errorf("failed to write osd config file. %+v", err)
|
||||
}
|
||||
if copyBinariesPath != "" {
|
||||
if err := osddaemon.CopyBinariesForDaemon(copyBinariesPath); err != nil {
|
||||
logger.Errorf("failed to copy rook binaries for filestore device. %+v", err)
|
||||
} else {
|
||||
logger.Infof("successfully copied rook binaries")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,8 @@ package osd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -75,3 +77,36 @@ func createOSDBootstrapKeyring(context *clusterd.Context, clusterName string) er
|
||||
|
||||
return cephconfig.CreateKeyring(context, clusterName, username, keyringPath, access, keyringEval)
|
||||
}
|
||||
|
||||
// CopyBinariesForDaemon copies the "tini" and "rook" binaries to a shared volume at the target path.
|
||||
// This is necessary for the filestore on a device scenario when rook needs to mount a directory
|
||||
// in the same container as the ceph process so it can be unmounted upon exit.
|
||||
func CopyBinariesForDaemon(target string) error {
|
||||
if err := copyBinary("/usr/local/bin", target, "rook"); err != nil {
|
||||
return err
|
||||
}
|
||||
return copyBinary("/", target, "tini")
|
||||
}
|
||||
|
||||
func copyBinary(sourceDir, targetDir, filename string) error {
|
||||
sourcePath := path.Join(sourceDir, filename)
|
||||
targetPath := path.Join(targetDir, filename)
|
||||
logger.Infof("copying %s to %s", sourcePath, targetPath)
|
||||
|
||||
sourceFile, err := os.Open(sourcePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer sourceFile.Close()
|
||||
|
||||
destinationFile, err := os.Create(targetPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer destinationFile.Close()
|
||||
if _, err := io.Copy(destinationFile, sourceFile); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.Chmod(targetPath, 0755)
|
||||
}
|
||||
@@ -28,12 +28,11 @@ import (
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
|
||||
opspec "github.com/rook/rook/pkg/operator/ceph/spec"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
|
||||
batch "k8s.io/api/batch/v1"
|
||||
"k8s.io/api/core/v1"
|
||||
extensions "k8s.io/api/extensions/v1beta1"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/kubernetes/pkg/kubelet/apis"
|
||||
)
|
||||
@@ -77,19 +76,9 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
|
||||
storeConfig config.StoreConfig, metadataDevice, location string, osd OSDInfo) (*extensions.Deployment, error) {
|
||||
|
||||
replicaCount := int32(1)
|
||||
var volumeMounts []v1.VolumeMount
|
||||
configVolumeMounts := []v1.VolumeMount{
|
||||
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
|
||||
k8sutil.ConfigOverrideMount(),
|
||||
}
|
||||
volumes := []v1.Volume{k8sutil.ConfigOverrideVolume()}
|
||||
if c.dataDirHostPath != "" {
|
||||
// the user has specified a host path to use for the data dir, use that instead
|
||||
dataDirSource := v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: c.dataDirHostPath}}
|
||||
volumes = append(volumes, v1.Volume{Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource})
|
||||
} else {
|
||||
logger.Infof("no data dir provided")
|
||||
}
|
||||
volumeMounts := opspec.CephVolumeMounts()
|
||||
configVolumeMounts := opspec.RookVolumeMounts()
|
||||
volumes := opspec.PodVolumes(c.dataDirHostPath)
|
||||
|
||||
var dataDir string
|
||||
if osd.IsDirectory {
|
||||
@@ -104,17 +93,16 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
|
||||
dataDirSource := v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: parentDir}}
|
||||
volumes = append(volumes, v1.Volume{Name: volumeName, VolumeSource: dataDirSource})
|
||||
configVolumeMounts = append(configVolumeMounts, v1.VolumeMount{Name: volumeName, MountPath: parentDir})
|
||||
volumeMounts = append(volumeMounts, v1.VolumeMount{Name: volumeName, MountPath: parentDir})
|
||||
}
|
||||
volumeMounts = configVolumeMounts
|
||||
} else {
|
||||
// Mount the required device
|
||||
dataDir = k8sutil.DataDir
|
||||
|
||||
// create volume config for the data dir and /dev so the pod can access devices on the host
|
||||
// Create volume config for /dev so the pod can access devices on the host
|
||||
devVolume := v1.Volume{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}}
|
||||
volumes = append(volumes, devVolume)
|
||||
devMount := v1.VolumeMount{Name: "devices", MountPath: "/dev"}
|
||||
volumeMounts = append(configVolumeMounts, devMount)
|
||||
volumeMounts = append(volumeMounts, devMount)
|
||||
}
|
||||
|
||||
if len(volumes) == 0 {
|
||||
@@ -151,20 +139,37 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
|
||||
var command []string
|
||||
var args []string
|
||||
if !osd.IsDirectory && osd.IsFileStore {
|
||||
// filestore on a device requires indirection through the rook entrypoint so we can mount the image
|
||||
// All scenarios except one can call the ceph-osd daemon directly. The one different scenario is when
|
||||
// filestore is running on a device. Rook needs to mount the device, run the ceph-osd daemon, and then
|
||||
// when the daemon exits, rook needs to unmount the device. Since rook needs to be in the container
|
||||
// for this scenario, we will copy the binaries necessary to a mount, which will then be mounted
|
||||
// to the daemon container.
|
||||
sourcePath := path.Join("/dev/disk/by-partuuid", osd.DevicePartUUID)
|
||||
mountPath := "/rook"
|
||||
command = []string{path.Join(mountPath, "tini")}
|
||||
args = append([]string{
|
||||
"--", path.Join(mountPath, "rook"),
|
||||
"ceph", "osd", "filestore-device",
|
||||
"--source-path", sourcePath,
|
||||
"--mount-path", osd.DataPath,
|
||||
"--",
|
||||
}, commonArgs...)
|
||||
"--"},
|
||||
commonArgs...)
|
||||
|
||||
// To get rook inside the container, the config init container needs to copy "tini" and "rook" binaries into a volume.
|
||||
// Set the config flag so rook will copy the binaries.
|
||||
configEnvVars = append(configEnvVars, v1.EnvVar{Name: "ROOK_COPY_BINARIES_PATH", Value: mountPath})
|
||||
|
||||
// Create the volume and mount that will be shared between the init container and the daemon container
|
||||
volumeName := "rookbinaries"
|
||||
binariesVolume := v1.Volume{Name: volumeName, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}
|
||||
volumes = append(volumes, binariesVolume)
|
||||
binariesMount := v1.VolumeMount{Name: volumeName, MountPath: mountPath}
|
||||
volumeMounts = append(volumeMounts, binariesMount)
|
||||
configVolumeMounts = append(configVolumeMounts, binariesMount)
|
||||
} else {
|
||||
// other osds can launch the osd daemon directly
|
||||
command = append([]string{"/tini", "--", "ceph-osd",
|
||||
fmt.Sprintf("--public-addr=$(%s)", k8sutil.PublicIPEnvVar),
|
||||
fmt.Sprintf("--cluster-addr=$(%s)", k8sutil.PrivateIPEnvVar),
|
||||
}, commonArgs...)
|
||||
command = []string{"ceph-osd"}
|
||||
args = commonArgs
|
||||
}
|
||||
|
||||
privileged := true
|
||||
@@ -213,8 +218,8 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
|
||||
DNSPolicy: DNSPolicy,
|
||||
InitContainers: []v1.Container{
|
||||
{
|
||||
Args: []string{"ceph", "osd", "config"},
|
||||
Name: "osd-init-config",
|
||||
Args: []string{"ceph", "osd", "init"},
|
||||
Name: opspec.ConfigInitContainerName,
|
||||
Image: k8sutil.MakeRookImage(c.Version),
|
||||
VolumeMounts: configVolumeMounts,
|
||||
Env: configEnvVars,
|
||||
@@ -225,7 +230,7 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
|
||||
{
|
||||
Command: command,
|
||||
Args: args,
|
||||
Name: appName,
|
||||
Name: "osd",
|
||||
Image: k8sutil.MakeRookImage(c.Version),
|
||||
VolumeMounts: volumeMounts,
|
||||
Env: envVars,
|
||||
@@ -246,13 +251,8 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
|
||||
|
||||
func (c *Cluster) provisionPodTemplateSpec(devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements,
|
||||
storeConfig config.StoreConfig, metadataDevice, location string, restart v1.RestartPolicy) (*v1.PodTemplateSpec, error) {
|
||||
volumes := []v1.Volume{k8sutil.ConfigOverrideVolume()}
|
||||
|
||||
if c.dataDirHostPath != "" {
|
||||
// the user has specified a host path to use for the data dir, use that instead
|
||||
dataDirSource := v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: c.dataDirHostPath}}
|
||||
volumes = append(volumes, v1.Volume{Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource})
|
||||
}
|
||||
volumes := opspec.PodVolumes(c.dataDirHostPath)
|
||||
|
||||
// by default, don't define any volume config unless it is required
|
||||
if len(devices) > 0 || selection.DeviceFilter != "" || selection.GetUseAllDevices() || metadataDevice != "" {
|
||||
@@ -366,10 +366,7 @@ func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection ro
|
||||
devMountNeeded = true
|
||||
}
|
||||
|
||||
volumeMounts := []v1.VolumeMount{
|
||||
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
|
||||
k8sutil.ConfigOverrideMount(),
|
||||
}
|
||||
volumeMounts := opspec.CephVolumeMounts()
|
||||
if devMountNeeded {
|
||||
devMount := v1.VolumeMount{Name: "devices", MountPath: "/dev"}
|
||||
volumeMounts = append(volumeMounts, devMount)
|
||||
+18
-16
@@ -85,17 +85,17 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
|
||||
assert.Equal(t, "node1", deployment.Spec.Template.Spec.NodeSelector[apis.LabelHostname])
|
||||
assert.Equal(t, v1.RestartPolicyAlways, deployment.Spec.Template.Spec.RestartPolicy)
|
||||
if devMountNeeded && len(dataDir) > 0 {
|
||||
assert.Equal(t, 3, len(deployment.Spec.Template.Spec.Volumes))
|
||||
assert.Equal(t, 4, len(deployment.Spec.Template.Spec.Volumes))
|
||||
}
|
||||
if devMountNeeded && len(dataDir) == 0 {
|
||||
assert.Equal(t, 2, len(deployment.Spec.Template.Spec.Volumes))
|
||||
assert.Equal(t, 4, len(deployment.Spec.Template.Spec.Volumes))
|
||||
}
|
||||
if !devMountNeeded && len(dataDir) > 0 {
|
||||
assert.Equal(t, 2, len(deployment.Spec.Template.Spec.Volumes))
|
||||
}
|
||||
|
||||
//assert.Equal(t, "rook-data", deployment.Spec.Template.Spec.Volumes[0].Name)
|
||||
assert.Equal(t, "rook-config-override", deployment.Spec.Template.Spec.Volumes[0].Name)
|
||||
assert.Equal(t, "rook-data", deployment.Spec.Template.Spec.Volumes[0].Name)
|
||||
assert.Equal(t, "ceph-default-config-dir", deployment.Spec.Template.Spec.Volumes[1].Name)
|
||||
|
||||
assert.Equal(t, appName, deployment.Spec.Template.ObjectMeta.Name)
|
||||
assert.Equal(t, appName, deployment.Spec.Template.ObjectMeta.Labels["app"])
|
||||
@@ -105,8 +105,8 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
|
||||
assert.Equal(t, 1, len(deployment.Spec.Template.Spec.InitContainers))
|
||||
initCont := deployment.Spec.Template.Spec.InitContainers[0]
|
||||
assert.Equal(t, "rook/rook:myversion", initCont.Image)
|
||||
assert.Equal(t, "osd-init-config", initCont.Name)
|
||||
assert.Equal(t, 2, len(initCont.VolumeMounts))
|
||||
assert.Equal(t, "config-init", initCont.Name)
|
||||
assert.Equal(t, 3, len(initCont.VolumeMounts))
|
||||
|
||||
assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers))
|
||||
cont := deployment.Spec.Template.Spec.Containers[0]
|
||||
@@ -116,7 +116,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
|
||||
} else {
|
||||
assert.Equal(t, 3, len(cont.VolumeMounts))
|
||||
}
|
||||
assert.Equal(t, "/tini", cont.Command[0])
|
||||
assert.Equal(t, "ceph-osd", cont.Command[0])
|
||||
}
|
||||
|
||||
func verifyEnvVar(t *testing.T, envVars []v1.EnvVar, expectedName, expectedValue string, expectedFound bool) {
|
||||
@@ -163,20 +163,21 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
// pod spec should have a volume for the given dir in the main container and the init container
|
||||
podSpec := deployment.Spec.Template.Spec
|
||||
assert.Equal(t, 3, len(podSpec.Volumes))
|
||||
assert.Equal(t, 4, len(podSpec.Volumes))
|
||||
require.Equal(t, 1, len(podSpec.Containers))
|
||||
cont := podSpec.Containers[0]
|
||||
assert.Equal(t, 3, len(cont.VolumeMounts))
|
||||
assert.Equal(t, "/var/lib/rook", cont.VolumeMounts[0].MountPath)
|
||||
assert.Equal(t, "/etc/rook/config", cont.VolumeMounts[1].MountPath)
|
||||
assert.Equal(t, "/etc/ceph", cont.VolumeMounts[1].MountPath)
|
||||
assert.Equal(t, "/my/root/path", cont.VolumeMounts[2].MountPath)
|
||||
|
||||
require.Equal(t, 1, len(podSpec.InitContainers))
|
||||
initCont := podSpec.InitContainers[0]
|
||||
assert.Equal(t, 3, len(initCont.VolumeMounts))
|
||||
assert.Equal(t, 4, len(initCont.VolumeMounts))
|
||||
assert.Equal(t, "/var/lib/rook", initCont.VolumeMounts[0].MountPath)
|
||||
assert.Equal(t, "/etc/rook/config", initCont.VolumeMounts[1].MountPath)
|
||||
assert.Equal(t, "/my/root/path", initCont.VolumeMounts[2].MountPath)
|
||||
assert.Equal(t, "/etc/ceph", initCont.VolumeMounts[1].MountPath)
|
||||
assert.Equal(t, "/etc/rook/config", initCont.VolumeMounts[2].MountPath)
|
||||
assert.Equal(t, "/my/root/path", initCont.VolumeMounts[3].MountPath)
|
||||
|
||||
// the default osd created on a node will be under /var/lib/rook, which won't need an extra mount
|
||||
osd = OSDInfo{
|
||||
@@ -189,20 +190,21 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
// pod spec should have a volume for the given dir in the main container and the init container
|
||||
podSpec = deployment.Spec.Template.Spec
|
||||
assert.Equal(t, 2, len(podSpec.Volumes))
|
||||
assert.Equal(t, 3, len(podSpec.Volumes))
|
||||
require.Equal(t, 1, len(podSpec.Containers))
|
||||
cont = podSpec.Containers[0]
|
||||
require.Equal(t, 2, len(cont.VolumeMounts))
|
||||
assert.Equal(t, "/var/lib/rook", cont.VolumeMounts[0].MountPath)
|
||||
assert.Equal(t, "/etc/rook/config", cont.VolumeMounts[1].MountPath)
|
||||
assert.Equal(t, "/etc/ceph", cont.VolumeMounts[1].MountPath)
|
||||
|
||||
assert.Equal(t, (4 + len(k8sutil.ClusterDaemonEnvVars())), len(cont.Env))
|
||||
|
||||
require.Equal(t, 1, len(podSpec.InitContainers))
|
||||
initCont = podSpec.InitContainers[0]
|
||||
require.Equal(t, 2, len(initCont.VolumeMounts))
|
||||
require.Equal(t, 3, len(initCont.VolumeMounts))
|
||||
assert.Equal(t, "/var/lib/rook", initCont.VolumeMounts[0].MountPath)
|
||||
assert.Equal(t, "/etc/rook/config", initCont.VolumeMounts[1].MountPath)
|
||||
assert.Equal(t, "/etc/ceph", initCont.VolumeMounts[1].MountPath)
|
||||
assert.Equal(t, "/etc/rook/config", initCont.VolumeMounts[2].MountPath)
|
||||
}
|
||||
|
||||
func TestStorageSpecConfig(t *testing.T) {
|
||||
@@ -362,7 +362,7 @@ spec:
|
||||
fieldPath: metadata.namespace`
|
||||
}
|
||||
|
||||
//GetRookCluster returns rook-cluster manifest
|
||||
//GetClusterRoles returns rook-cluster manifest
|
||||
func (m *CephManifestsMaster) GetClusterRoles(namespace, systemNamespace string) string {
|
||||
return `apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
|
||||
@@ -434,7 +434,7 @@ func (k8sh *K8sHelper) WaitForPodCount(label, namespace string, count int) error
|
||||
return fmt.Errorf("failed to find pod with label %s. %+v", label, err)
|
||||
}
|
||||
|
||||
if len(pods.Items) == count {
|
||||
if len(pods.Items) >= count {
|
||||
logger.Infof("found %d pods with label %s", count, label)
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user