diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index cfb730426..b83c5611a 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -40,7 +40,7 @@ var osdCmd = &cobra.Command{ Hidden: true, } var osdConfigCmd = &cobra.Command{ - Use: "config", + Use: "init", Short: "Updates ceph.conf for the osd", Hidden: true, } @@ -60,6 +60,7 @@ var ( mountSourcePath string mountPath string osdID int + copyBinariesPath string ) func addOSDFlags(command *cobra.Command) { @@ -74,8 +75,9 @@ func addOSDFlags(command *cobra.Command) { provisionCmd.Flags().BoolVar(&cfg.forceFormat, "force-format", false, "true to force the format of any specified devices, even if they already have a filesystem. BE CAREFUL!") - // flags for generating the osd config file + // flags for generating the osd config osdConfigCmd.Flags().IntVar(&osdID, "osd-id", -1, "osd id for which to generate config") + osdConfigCmd.Flags().StringVar(©BinariesPath, "copy-binaries-path", "", "If specified, copy the rook binaries to this path for use by the daemon container") // flags for running filestore on a device filestoreDeviceCmd.Flags().StringVar(&mountSourcePath, "source-path", "", "the source path of the device to mount") @@ -172,6 +174,13 @@ func writeOSDConfig(cmd *cobra.Command, args []string) error { if err := osddaemon.WriteConfigFile(context, &clusterInfo, kv, osdID, cfg.storeConfig, cfg.nodeName, crushLocation); err != nil { logger.Errorf("failed to write osd config file. %+v", err) } + if copyBinariesPath != "" { + if err := osddaemon.CopyBinariesForDaemon(copyBinariesPath); err != nil { + logger.Errorf("failed to copy rook binaries for filestore device. %+v", err) + } else { + logger.Infof("successfully copied rook binaries") + } + } return nil } diff --git a/pkg/daemon/ceph/osd/config.go b/pkg/daemon/ceph/osd/init.go similarity index 70% rename from pkg/daemon/ceph/osd/config.go rename to pkg/daemon/ceph/osd/init.go index e2bface70..700b14c8f 100644 --- a/pkg/daemon/ceph/osd/config.go +++ b/pkg/daemon/ceph/osd/init.go @@ -18,6 +18,8 @@ package osd import ( "fmt" + "io" + "os" "path" "path/filepath" @@ -75,3 +77,36 @@ func createOSDBootstrapKeyring(context *clusterd.Context, clusterName string) er return cephconfig.CreateKeyring(context, clusterName, username, keyringPath, access, keyringEval) } + +// CopyBinariesForDaemon copies the "tini" and "rook" binaries to a shared volume at the target path. +// This is necessary for the filestore on a device scenario when rook needs to mount a directory +// in the same container as the ceph process so it can be unmounted upon exit. +func CopyBinariesForDaemon(target string) error { + if err := copyBinary("/usr/local/bin", target, "rook"); err != nil { + return err + } + return copyBinary("/", target, "tini") +} + +func copyBinary(sourceDir, targetDir, filename string) error { + sourcePath := path.Join(sourceDir, filename) + targetPath := path.Join(targetDir, filename) + logger.Infof("copying %s to %s", sourcePath, targetPath) + + sourceFile, err := os.Open(sourcePath) + if err != nil { + return err + } + defer sourceFile.Close() + + destinationFile, err := os.Create(targetPath) + if err != nil { + return err + } + defer destinationFile.Close() + if _, err := io.Copy(destinationFile, sourceFile); err != nil { + return err + } + + return os.Chmod(targetPath, 0755) +} diff --git a/pkg/operator/ceph/cluster/osd/pod.go b/pkg/operator/ceph/cluster/osd/spec.go similarity index 88% rename from pkg/operator/ceph/cluster/osd/pod.go rename to pkg/operator/ceph/cluster/osd/spec.go index b314c02f2..8f70b8bc6 100644 --- a/pkg/operator/ceph/cluster/osd/pod.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -28,12 +28,11 @@ import ( rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" + opspec "github.com/rook/rook/pkg/operator/ceph/spec" "github.com/rook/rook/pkg/operator/k8sutil" - batch "k8s.io/api/batch/v1" "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/pkg/kubelet/apis" ) @@ -77,19 +76,9 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se storeConfig config.StoreConfig, metadataDevice, location string, osd OSDInfo) (*extensions.Deployment, error) { replicaCount := int32(1) - var volumeMounts []v1.VolumeMount - configVolumeMounts := []v1.VolumeMount{ - {Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir}, - k8sutil.ConfigOverrideMount(), - } - volumes := []v1.Volume{k8sutil.ConfigOverrideVolume()} - if c.dataDirHostPath != "" { - // the user has specified a host path to use for the data dir, use that instead - dataDirSource := v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: c.dataDirHostPath}} - volumes = append(volumes, v1.Volume{Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource}) - } else { - logger.Infof("no data dir provided") - } + volumeMounts := opspec.CephVolumeMounts() + configVolumeMounts := opspec.RookVolumeMounts() + volumes := opspec.PodVolumes(c.dataDirHostPath) var dataDir string if osd.IsDirectory { @@ -104,17 +93,16 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se dataDirSource := v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: parentDir}} volumes = append(volumes, v1.Volume{Name: volumeName, VolumeSource: dataDirSource}) configVolumeMounts = append(configVolumeMounts, v1.VolumeMount{Name: volumeName, MountPath: parentDir}) + volumeMounts = append(volumeMounts, v1.VolumeMount{Name: volumeName, MountPath: parentDir}) } - volumeMounts = configVolumeMounts } else { - // Mount the required device dataDir = k8sutil.DataDir - // create volume config for the data dir and /dev so the pod can access devices on the host + // Create volume config for /dev so the pod can access devices on the host devVolume := v1.Volume{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}} volumes = append(volumes, devVolume) devMount := v1.VolumeMount{Name: "devices", MountPath: "/dev"} - volumeMounts = append(configVolumeMounts, devMount) + volumeMounts = append(volumeMounts, devMount) } if len(volumes) == 0 { @@ -151,20 +139,37 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se var command []string var args []string if !osd.IsDirectory && osd.IsFileStore { - // filestore on a device requires indirection through the rook entrypoint so we can mount the image + // All scenarios except one can call the ceph-osd daemon directly. The one different scenario is when + // filestore is running on a device. Rook needs to mount the device, run the ceph-osd daemon, and then + // when the daemon exits, rook needs to unmount the device. Since rook needs to be in the container + // for this scenario, we will copy the binaries necessary to a mount, which will then be mounted + // to the daemon container. sourcePath := path.Join("/dev/disk/by-partuuid", osd.DevicePartUUID) + mountPath := "/rook" + command = []string{path.Join(mountPath, "tini")} args = append([]string{ + "--", path.Join(mountPath, "rook"), "ceph", "osd", "filestore-device", "--source-path", sourcePath, "--mount-path", osd.DataPath, - "--", - }, commonArgs...) + "--"}, + commonArgs...) + + // To get rook inside the container, the config init container needs to copy "tini" and "rook" binaries into a volume. + // Set the config flag so rook will copy the binaries. + configEnvVars = append(configEnvVars, v1.EnvVar{Name: "ROOK_COPY_BINARIES_PATH", Value: mountPath}) + + // Create the volume and mount that will be shared between the init container and the daemon container + volumeName := "rookbinaries" + binariesVolume := v1.Volume{Name: volumeName, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}} + volumes = append(volumes, binariesVolume) + binariesMount := v1.VolumeMount{Name: volumeName, MountPath: mountPath} + volumeMounts = append(volumeMounts, binariesMount) + configVolumeMounts = append(configVolumeMounts, binariesMount) } else { // other osds can launch the osd daemon directly - command = append([]string{"/tini", "--", "ceph-osd", - fmt.Sprintf("--public-addr=$(%s)", k8sutil.PublicIPEnvVar), - fmt.Sprintf("--cluster-addr=$(%s)", k8sutil.PrivateIPEnvVar), - }, commonArgs...) + command = []string{"ceph-osd"} + args = commonArgs } privileged := true @@ -213,8 +218,8 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se DNSPolicy: DNSPolicy, InitContainers: []v1.Container{ { - Args: []string{"ceph", "osd", "config"}, - Name: "osd-init-config", + Args: []string{"ceph", "osd", "init"}, + Name: opspec.ConfigInitContainerName, Image: k8sutil.MakeRookImage(c.Version), VolumeMounts: configVolumeMounts, Env: configEnvVars, @@ -225,7 +230,7 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se { Command: command, Args: args, - Name: appName, + Name: "osd", Image: k8sutil.MakeRookImage(c.Version), VolumeMounts: volumeMounts, Env: envVars, @@ -246,13 +251,8 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se func (c *Cluster) provisionPodTemplateSpec(devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements, storeConfig config.StoreConfig, metadataDevice, location string, restart v1.RestartPolicy) (*v1.PodTemplateSpec, error) { - volumes := []v1.Volume{k8sutil.ConfigOverrideVolume()} - if c.dataDirHostPath != "" { - // the user has specified a host path to use for the data dir, use that instead - dataDirSource := v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: c.dataDirHostPath}} - volumes = append(volumes, v1.Volume{Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource}) - } + volumes := opspec.PodVolumes(c.dataDirHostPath) // by default, don't define any volume config unless it is required if len(devices) > 0 || selection.DeviceFilter != "" || selection.GetUseAllDevices() || metadataDevice != "" { @@ -366,10 +366,7 @@ func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection ro devMountNeeded = true } - volumeMounts := []v1.VolumeMount{ - {Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir}, - k8sutil.ConfigOverrideMount(), - } + volumeMounts := opspec.CephVolumeMounts() if devMountNeeded { devMount := v1.VolumeMount{Name: "devices", MountPath: "/dev"} volumeMounts = append(volumeMounts, devMount) diff --git a/pkg/operator/ceph/cluster/osd/pod_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go similarity index 91% rename from pkg/operator/ceph/cluster/osd/pod_test.go rename to pkg/operator/ceph/cluster/osd/spec_test.go index 45db4af29..405b33b33 100644 --- a/pkg/operator/ceph/cluster/osd/pod_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -85,17 +85,17 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, "node1", deployment.Spec.Template.Spec.NodeSelector[apis.LabelHostname]) assert.Equal(t, v1.RestartPolicyAlways, deployment.Spec.Template.Spec.RestartPolicy) if devMountNeeded && len(dataDir) > 0 { - assert.Equal(t, 3, len(deployment.Spec.Template.Spec.Volumes)) + assert.Equal(t, 4, len(deployment.Spec.Template.Spec.Volumes)) } if devMountNeeded && len(dataDir) == 0 { - assert.Equal(t, 2, len(deployment.Spec.Template.Spec.Volumes)) + assert.Equal(t, 4, len(deployment.Spec.Template.Spec.Volumes)) } if !devMountNeeded && len(dataDir) > 0 { assert.Equal(t, 2, len(deployment.Spec.Template.Spec.Volumes)) } - //assert.Equal(t, "rook-data", deployment.Spec.Template.Spec.Volumes[0].Name) - assert.Equal(t, "rook-config-override", deployment.Spec.Template.Spec.Volumes[0].Name) + assert.Equal(t, "rook-data", deployment.Spec.Template.Spec.Volumes[0].Name) + assert.Equal(t, "ceph-default-config-dir", deployment.Spec.Template.Spec.Volumes[1].Name) assert.Equal(t, appName, deployment.Spec.Template.ObjectMeta.Name) assert.Equal(t, appName, deployment.Spec.Template.ObjectMeta.Labels["app"]) @@ -105,8 +105,8 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, 1, len(deployment.Spec.Template.Spec.InitContainers)) initCont := deployment.Spec.Template.Spec.InitContainers[0] assert.Equal(t, "rook/rook:myversion", initCont.Image) - assert.Equal(t, "osd-init-config", initCont.Name) - assert.Equal(t, 2, len(initCont.VolumeMounts)) + assert.Equal(t, "config-init", initCont.Name) + assert.Equal(t, 3, len(initCont.VolumeMounts)) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont := deployment.Spec.Template.Spec.Containers[0] @@ -116,7 +116,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { } else { assert.Equal(t, 3, len(cont.VolumeMounts)) } - assert.Equal(t, "/tini", cont.Command[0]) + assert.Equal(t, "ceph-osd", cont.Command[0]) } func verifyEnvVar(t *testing.T, envVars []v1.EnvVar, expectedName, expectedValue string, expectedFound bool) { @@ -163,20 +163,21 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) { assert.Nil(t, err) // pod spec should have a volume for the given dir in the main container and the init container podSpec := deployment.Spec.Template.Spec - assert.Equal(t, 3, len(podSpec.Volumes)) + assert.Equal(t, 4, len(podSpec.Volumes)) require.Equal(t, 1, len(podSpec.Containers)) cont := podSpec.Containers[0] assert.Equal(t, 3, len(cont.VolumeMounts)) assert.Equal(t, "/var/lib/rook", cont.VolumeMounts[0].MountPath) - assert.Equal(t, "/etc/rook/config", cont.VolumeMounts[1].MountPath) + assert.Equal(t, "/etc/ceph", cont.VolumeMounts[1].MountPath) assert.Equal(t, "/my/root/path", cont.VolumeMounts[2].MountPath) require.Equal(t, 1, len(podSpec.InitContainers)) initCont := podSpec.InitContainers[0] - assert.Equal(t, 3, len(initCont.VolumeMounts)) + assert.Equal(t, 4, len(initCont.VolumeMounts)) assert.Equal(t, "/var/lib/rook", initCont.VolumeMounts[0].MountPath) - assert.Equal(t, "/etc/rook/config", initCont.VolumeMounts[1].MountPath) - assert.Equal(t, "/my/root/path", initCont.VolumeMounts[2].MountPath) + assert.Equal(t, "/etc/ceph", initCont.VolumeMounts[1].MountPath) + assert.Equal(t, "/etc/rook/config", initCont.VolumeMounts[2].MountPath) + assert.Equal(t, "/my/root/path", initCont.VolumeMounts[3].MountPath) // the default osd created on a node will be under /var/lib/rook, which won't need an extra mount osd = OSDInfo{ @@ -189,20 +190,21 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) { assert.Nil(t, err) // pod spec should have a volume for the given dir in the main container and the init container podSpec = deployment.Spec.Template.Spec - assert.Equal(t, 2, len(podSpec.Volumes)) + assert.Equal(t, 3, len(podSpec.Volumes)) require.Equal(t, 1, len(podSpec.Containers)) cont = podSpec.Containers[0] require.Equal(t, 2, len(cont.VolumeMounts)) assert.Equal(t, "/var/lib/rook", cont.VolumeMounts[0].MountPath) - assert.Equal(t, "/etc/rook/config", cont.VolumeMounts[1].MountPath) + assert.Equal(t, "/etc/ceph", cont.VolumeMounts[1].MountPath) assert.Equal(t, (4 + len(k8sutil.ClusterDaemonEnvVars())), len(cont.Env)) require.Equal(t, 1, len(podSpec.InitContainers)) initCont = podSpec.InitContainers[0] - require.Equal(t, 2, len(initCont.VolumeMounts)) + require.Equal(t, 3, len(initCont.VolumeMounts)) assert.Equal(t, "/var/lib/rook", initCont.VolumeMounts[0].MountPath) - assert.Equal(t, "/etc/rook/config", initCont.VolumeMounts[1].MountPath) + assert.Equal(t, "/etc/ceph", initCont.VolumeMounts[1].MountPath) + assert.Equal(t, "/etc/rook/config", initCont.VolumeMounts[2].MountPath) } func TestStorageSpecConfig(t *testing.T) { diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index f9ba67947..43cc3f638 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -362,7 +362,7 @@ spec: fieldPath: metadata.namespace` } -//GetRookCluster returns rook-cluster manifest +//GetClusterRoles returns rook-cluster manifest func (m *CephManifestsMaster) GetClusterRoles(namespace, systemNamespace string) string { return `apiVersion: v1 kind: ServiceAccount diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index f4f2c2eda..54a459e4f 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -434,7 +434,7 @@ func (k8sh *K8sHelper) WaitForPodCount(label, namespace string, count int) error return fmt.Errorf("failed to find pod with label %s. %+v", label, err) } - if len(pods.Items) == count { + if len(pods.Items) >= count { logger.Infof("found %d pods with label %s", count, label) return nil }