From 1d928ecd3678d487c854a21b83929a604b7eeb02 Mon Sep 17 00:00:00 2001 From: travisn Date: Tue, 31 Jul 2018 14:39:53 -0600 Subject: [PATCH] osd: init container to initialize config at startup Signed-off-by: travisn --- Documentation/upgrade-patch.md | 93 ++++++++++++++ Documentation/upgrade.md | 4 + PendingReleaseNotes.md | 1 + cmd/rook/ceph/osd.go | 83 +++++++++--- pkg/daemon/ceph/osd/agent.go | 8 +- pkg/daemon/ceph/osd/agent_test.go | 2 +- pkg/daemon/ceph/osd/daemon.go | 6 +- pkg/daemon/ceph/osd/device.go | 60 +++++++-- pkg/operator/ceph/cluster/osd/osd.go | 30 +++-- pkg/operator/ceph/cluster/osd/pod.go | 148 +++++++++++++--------- pkg/operator/ceph/cluster/osd/pod_test.go | 55 +++++++- pkg/operator/ceph/cluster/osd/status.go | 7 +- pkg/operator/k8sutil/deployment.go | 61 +++++++++ pkg/operator/k8sutil/pod.go | 9 +- 14 files changed, 455 insertions(+), 112 deletions(-) create mode 100644 Documentation/upgrade-patch.md create mode 100644 pkg/operator/k8sutil/deployment.go diff --git a/Documentation/upgrade-patch.md b/Documentation/upgrade-patch.md new file mode 100644 index 000000000..9d0f2be89 --- /dev/null +++ b/Documentation/upgrade-patch.md @@ -0,0 +1,93 @@ +--- +title: Patch Upgrades +weight: 61 +indent: true +--- + +# 0.8 Patch Upgrades + +The changes in patch releases are scoped to the minimal changes necessary and are expected to be straight forward to upgrade. +This guide will walk you through the manual steps to upgrade the software in a Rook cluster from an 0.8 version to another patched version of 0.8. +For example, upgrade from `v0.8.0` to `v0.8.1`. + +After each component is upgraded, it is important to verify that the cluster returns to a healthy and fully functional state. + +## Considerations +With this manual upgrade guide, there are a few notes to consider: +* **WARNING:** Upgrading a Rook cluster is a manual process in its very early stages. There may be unexpected issues or obstacles that damage the integrity and health of your storage cluster, including data loss. +* This guide assumes that your Rook operator and its agents are running in the `rook-ceph-system` namespace. It also assumes that your Rook cluster is in the `rook-ceph` namespace. If any of these components is in a different namespace, search/replace all instances of `-n rook-ceph-system` and `-n rook-ceph` in this guide with `-n `. + +## Prerequisites +In order to successfully upgrade a Rook cluster, the following prerequisites must be met: +* The cluster should be in a healthy state with full functionality. +Review the [health verification section](upgrade.md#health-verification) in the main upgrade guide in order to verify your cluster is in a good starting state. +* All pods consuming Rook storage should be created, running, and in a steady state. No Rook persistent volumes should be in the act of being created or deleted. + +## Upgrade Process +The general flow of the upgrade process will be to upgrade the version of a Rook pod, verify the pod is running with the new version, then verify that the overall cluster health is still in a good state. + +In this guide, we will be upgrading a live Rook cluster running `v0.8.0` to version `v0.8.1`. + +### Operator +The operator controls upgrading all the components and is generally the first component to be updated. +The operator can be updated by setting the deployment version. + +```bash +kubectl -n rook-ceph-system set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v0.8.1 +``` + +The operator pod will automatically be restarted by Kubernetes with the new version. + +Once you've verified the operator is `Running` and on the new version, verify the health of the cluster is still OK. +Instructions for verifying cluster health can be found in the [health verification section](upgrade.md#health-verification). + +### System Daemons +The pods in the rook-ceph-system namespace will all be updated automatically when the operator is updated. After the operator is updated, you will see the `rook-ceph-agent` and `rook-discover` pods restarted on the new version. + +### Monitors +There are multiple monitor pods to upgrade and they are each individually managed by their own replica set. +**For each** monitor's replica set, you will need to update the pod template spec's image version field to `rook/ceph:v0.8.1`. +For example, we can update the replica set for `mon0` with: +```bash +kubectl -n rook-ceph set image replicaset/rook-ceph-mon0 rook-ceph-mon=rook/ceph:v0.8.1 +``` + +Once the replica set has been updated, we need to manually terminate the old pod which will trigger the replica set to create a new pod using the new version. +```bash +kubectl -n rook-ceph delete pod -l mon=rook-ceph-mon0 +``` + +At this point, it's very important to ensure that all monitors are `OK` and `in quorum`. +Refer to the [status output section](upgrade.md#status-output) for instructions. +If all of the monitors (and the cluster health overall) look good, then we can move on and repeat the same upgrade steps for the next monitor until all are completed. + +### Object Storage Daemons (OSDs) +The automatic upgrade of OSD pods has been implemented by the operator. Within a minute of starting the operator on the new version, you should see the osd pods automatically started on the new version. +Going forward, if there is any change needed to the OSD deployment as determined by the operator, the OSD pods will automatically be updated and restarted. +For example, the OSDs will be automatically updated when: +- The version of the operator container changes +- The `resources` or `placement` elements are changed in the cluster CRD + +One by one, as each of the OSDs are updated the operator will wait for the OSD pod to be running again before continuing with the next OSD. +In some scenarios, the operator will need to be restarted in order to apply the changes to the OSD deployment specs. + +### Ceph Manager +To update the Ceph mgrs, edit the deployment image version: +```bash +kubectl -n rook-ceph set image deploy/rook-ceph-mgr-a rook-ceph-mgr-a=rook/ceph:v0.8.1 +``` + +#### Object Storage (RGW) +If you have object storage installed, edit the RGW deployment to use the new image version: +```bash +kubectl -n rook-ceph set image deploy/rook-ceph-rgw-my-store rook-ceph-rgw-my-store=rook/ceph:v0.8.1 +``` + +#### Shared File System (MDS) +If you have a shared file system installed, edit the MDS deployment to use the new image version: +```bash +kubectl -n rook-ceph set image deploy/rook-ceph-mds-myfs rook-ceph-mds-myfs=rook/ceph:v0.8.1 +``` + +## Completion +At this point, your Rook cluster should be fully upgraded to running version `rook/ceph:v0.8.1` and the cluster should be healthy according to the steps in the [health verification section](upgrade.md#health-verification). diff --git a/Documentation/upgrade.md b/Documentation/upgrade.md index d9f4683d6..9fca24279 100644 --- a/Documentation/upgrade.md +++ b/Documentation/upgrade.md @@ -22,6 +22,10 @@ For a guide to upgrade previous versions of Rook, please refer to the version of - [Upgrade 0.6 to 0.7](https://rook.io/docs/rook/v0.7/upgrade.html) - [Upgrade 0.5 to 0.6](https://rook.io/docs/rook/v0.6/upgrade.html) +### Patch Release Upgrades +The changes in patch releases are scoped to the minimal changes necessary and are expected to be straight forward to upgrade. +For upgrades of 0.8 patch releases such as `0.8.0` to `0.8.1`, see the [patch release upgrade guide](upgrade-patch.md). + ## Considerations With this manual upgrade guide, there are a few notes to consider: * **WARNING:** Upgrading a Rook cluster is a manual process in its very early stages. There may be unexpected issues or obstacles that damage the integrity and health of your storage cluster, including data loss. Only proceed with this guide if you are comfortable with that. diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 488f264ae..fc7f10724 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -5,6 +5,7 @@ ## Notable Features - The `fsType` default for StorageClass examples are now using XFS to bring it in line with Ceph recommendations. - Ceph is updated from Luminous 12.2.5 to 12.2.7. +- Ceph OSDs will be automatically updated by the operator when there is a change to the operator version or when the OSD configuration changes. See the [OSD upgrade notes](Documentation/upgrade-patch.md#object-storage-daemons-osds). ## Breaking Changes diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index f626e15ed..4f8e840b6 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -30,6 +30,7 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/flags" "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var osdCmd = &cobra.Command{ @@ -37,6 +38,11 @@ var osdCmd = &cobra.Command{ Short: "Provisions and runs the osd daemon", Hidden: true, } +var osdConfigCmd = &cobra.Command{ + Use: "config", + Short: "Updates ceph.conf for the osd", + Hidden: true, +} var provisionCmd = &cobra.Command{ Use: "provision", Short: "Generates osd config and prepares an osd for runtime", @@ -50,48 +56,57 @@ var filestoreDeviceCmd = &cobra.Command{ var ( osdDataDeviceFilter string ownerRefID string - prepareOnly bool mountSourcePath string mountPath string + osdID int ) func addOSDFlags(command *cobra.Command) { + addOSDConfigFlags(osdConfigCmd) + addOSDConfigFlags(provisionCmd) + + // flags specific to provisioning provisionCmd.Flags().StringVar(&cfg.devices, "data-devices", "", "comma separated list of devices to use for storage") - provisionCmd.Flags().StringVar(&ownerRefID, "cluster-id", "", "the UID of the cluster CRD that owns this cluster") provisionCmd.Flags().StringVar(&osdDataDeviceFilter, "data-device-filter", "", "a regex filter for the device names to use, or \"all\"") provisionCmd.Flags().StringVar(&cfg.directories, "data-directories", "", "comma separated list of directory paths to use for storage") provisionCmd.Flags().StringVar(&cfg.metadataDevice, "metadata-device", "", "device to use for metadata (e.g. a high performance SSD/NVMe device)") - provisionCmd.Flags().StringVar(&cfg.location, "location", "", "location of this node for CRUSH placement") provisionCmd.Flags().BoolVar(&cfg.forceFormat, "force-format", false, "true to force the format of any specified devices, even if they already have a filesystem. BE CAREFUL!") - provisionCmd.Flags().StringVar(&cfg.nodeName, "node-name", os.Getenv("HOSTNAME"), "the host name of the node") - // OSD store config flags - provisionCmd.Flags().IntVar(&cfg.storeConfig.WalSizeMB, "osd-wal-size", osdcfg.WalDefaultSizeMB, "default size (MB) for OSD write ahead log (WAL) (bluestore)") - provisionCmd.Flags().IntVar(&cfg.storeConfig.DatabaseSizeMB, "osd-database-size", osdcfg.DBDefaultSizeMB, "default size (MB) for OSD database (bluestore)") - provisionCmd.Flags().IntVar(&cfg.storeConfig.JournalSizeMB, "osd-journal-size", osdcfg.JournalDefaultSizeMB, "default size (MB) for OSD journal (filestore)") - provisionCmd.Flags().StringVar(&cfg.storeConfig.StoreType, "osd-store", "", "type of backing OSD store to use (bluestore or filestore)") - - // only prepare devices but not start ceph-osd daemon - provisionCmd.Flags().BoolVar(&prepareOnly, "osd-prepare-only", true, "true to only prepare ceph osd directories or devices but not start ceph-osd daemon") + // flags for generating the osd config file + osdConfigCmd.Flags().IntVar(&osdID, "osd-id", -1, "osd id for which to generate config") // flags for running filestore on a device filestoreDeviceCmd.Flags().StringVar(&mountSourcePath, "source-path", "", "the source path of the device to mount") filestoreDeviceCmd.Flags().StringVar(&mountPath, "mount-path", "", "the path where the device should be mounted") // add the subcommands to the parent osd command + osdCmd.AddCommand(osdConfigCmd) osdCmd.AddCommand(provisionCmd) osdCmd.AddCommand(filestoreDeviceCmd) +} +func addOSDConfigFlags(command *cobra.Command) { + command.Flags().StringVar(&ownerRefID, "cluster-id", "", "the UID of the cluster CRD that owns this cluster") + command.Flags().StringVar(&cfg.location, "location", "", "location of this node for CRUSH placement") + command.Flags().StringVar(&cfg.nodeName, "node-name", os.Getenv("HOSTNAME"), "the host name of the node") + + // OSD store config flags + command.Flags().IntVar(&cfg.storeConfig.WalSizeMB, "osd-wal-size", osdcfg.WalDefaultSizeMB, "default size (MB) for OSD write ahead log (WAL) (bluestore)") + command.Flags().IntVar(&cfg.storeConfig.DatabaseSizeMB, "osd-database-size", osdcfg.DBDefaultSizeMB, "default size (MB) for OSD database (bluestore)") + command.Flags().IntVar(&cfg.storeConfig.JournalSizeMB, "osd-journal-size", osdcfg.JournalDefaultSizeMB, "default size (MB) for OSD journal (filestore)") + command.Flags().StringVar(&cfg.storeConfig.StoreType, "osd-store", "", "type of backing OSD store to use (bluestore or filestore)") } func init() { addOSDFlags(osdCmd) addCephFlags(osdCmd) flags.SetFlagsFromEnv(osdCmd.Flags(), rook.RookEnvVarPrefix) + flags.SetFlagsFromEnv(osdConfigCmd.Flags(), rook.RookEnvVarPrefix) flags.SetFlagsFromEnv(provisionCmd.Flags(), rook.RookEnvVarPrefix) flags.SetFlagsFromEnv(filestoreDeviceCmd.Flags(), rook.RookEnvVarPrefix) + osdConfigCmd.RunE = writeOSDConfig provisionCmd.RunE = prepareOSD filestoreDeviceCmd.RunE = runFilestoreDeviceOSD } @@ -118,16 +133,52 @@ func runFilestoreDeviceOSD(cmd *cobra.Command, args []string) error { return nil } -// Provision a device or directory for an OSD -func prepareOSD(cmd *cobra.Command, args []string) error { +func verifyConfigFlags(configCmd *cobra.Command) error { required := []string{"cluster-id", "node-name"} - if err := flags.VerifyRequiredFlags(provisionCmd, required); err != nil { + if err := flags.VerifyRequiredFlags(configCmd, required); err != nil { return err } required = []string{"cluster-name", "mon-endpoints", "mon-secret", "admin-secret"} if err := flags.VerifyRequiredFlags(osdCmd, required); err != nil { return err } + return nil +} + +func writeOSDConfig(cmd *cobra.Command, args []string) error { + if err := verifyConfigFlags(osdConfigCmd); err != nil { + return err + } + if osdID == -1 { + return fmt.Errorf("osd id not specified") + } + + clientset, _, _, err := rook.GetClientset() + if err != nil { + rook.TerminateFatal(fmt.Errorf("failed to init k8s client. %+v\n", err)) + } + + context := createContext() + context.Clientset = clientset + commonOSDInit(osdConfigCmd) + locArgs, err := client.FormatLocation(cfg.location, cfg.nodeName) + if err != nil { + rook.TerminateFatal(fmt.Errorf("invalid location %s. %+v\n", cfg.location, err)) + } + crushLocation := strings.Join(locArgs, " ") + kv := k8sutil.NewConfigMapKVStore(clusterInfo.Name, clientset, metav1.OwnerReference{}) + + if err := osd.WriteConfigFile(context, &clusterInfo, kv, osdID, cfg.storeConfig, cfg.nodeName, crushLocation); err != nil { + logger.Errorf("failed to write osd config file. %+v", err) + } + return nil +} + +// Provision a device or directory for an OSD +func prepareOSD(cmd *cobra.Command, args []string) error { + if err := verifyConfigFlags(provisionCmd); err != nil { + return err + } if err := verifyRenamedFlags(osdCmd); err != nil { return err @@ -166,7 +217,7 @@ func prepareOSD(cmd *cobra.Command, args []string) error { ownerRef := cluster.ClusterOwnerRef(clusterInfo.Name, ownerRefID) kv := k8sutil.NewConfigMapKVStore(clusterInfo.Name, clientset, ownerRef) agent := osd.NewAgent(context, dataDevices, usingDeviceFilter, cfg.metadataDevice, cfg.directories, forceFormat, - crushLocation, cfg.storeConfig, &clusterInfo, cfg.nodeName, kv, prepareOnly) + crushLocation, cfg.storeConfig, &clusterInfo, cfg.nodeName, kv) err = osd.Provision(context, agent) if err != nil { diff --git a/pkg/daemon/ceph/osd/agent.go b/pkg/daemon/ceph/osd/agent.go index 6685da7c0..eef21dc6a 100644 --- a/pkg/daemon/ceph/osd/agent.go +++ b/pkg/daemon/ceph/osd/agent.go @@ -58,11 +58,10 @@ type OsdAgent struct { kv *k8sutil.ConfigMapKVStore configCounter int32 osdsCompleted chan struct{} - prepareOnly bool } func NewAgent(context *clusterd.Context, devices string, usingDeviceFilter bool, metadataDevice, directories string, forceFormat bool, - location string, storeConfig config.StoreConfig, cluster *mon.ClusterInfo, nodeName string, kv *k8sutil.ConfigMapKVStore, prepareOnly bool) *OsdAgent { + location string, storeConfig config.StoreConfig, cluster *mon.ClusterInfo, nodeName string, kv *k8sutil.ConfigMapKVStore) *OsdAgent { return &OsdAgent{ devices: devices, @@ -77,7 +76,6 @@ func NewAgent(context *clusterd.Context, devices string, usingDeviceFilter bool, kv: kv, procMan: proc.New(context.Executor), osdProc: make(map[int]*proc.MonitoredProc), - prepareOnly: prepareOnly, } } @@ -425,13 +423,13 @@ func (a *OsdAgent) prepareOSD(context *clusterd.Context, cfg *osdConfig) (*oposd } // osd_data_dir/ready does not exist yet, create/initialize the OSD - err := initializeOSD(cfg, context, a.cluster, a.location, a.prepareOnly) + err := initializeOSD(cfg, context, a.cluster, a.location) if err != nil { return nil, fmt.Errorf("failed to initialize OSD at %s: %+v", cfg.rootPath, err) } } else { // update the osd config file - err := writeConfigFile(cfg, context, a.cluster, a.location, a.prepareOnly) + err := writeConfigFile(cfg, context, a.cluster, a.location) if err != nil { logger.Warningf("failed to update config file. %+v", err) } diff --git a/pkg/daemon/ceph/osd/agent_test.go b/pkg/daemon/ceph/osd/agent_test.go index 3508ae6ea..682e28c67 100644 --- a/pkg/daemon/ceph/osd/agent_test.go +++ b/pkg/daemon/ceph/osd/agent_test.go @@ -353,7 +353,7 @@ func createTestAgent(t *testing.T, devices, configDir, nodeName string, storeCon cluster := &mon.ClusterInfo{Name: "myclust"} context := &clusterd.Context{ConfigDir: configDir, Executor: executor, Clientset: testop.New(1)} agent := NewAgent(context, devices, false, "", "", forceFormat, location, *storeConfig, - cluster, nodeName, mockKVStore(), false /* prepareOnly */) + cluster, nodeName, mockKVStore()) return agent, executor, context } diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index 25e51fcf1..47d0dda03 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -66,11 +66,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent) error { // set the crush location in the osd config file cephConfig := mon.CreateDefaultCephConfig(context, agent.cluster, path.Join(context.ConfigDir, agent.cluster.Name)) cephConfig.GlobalConfig.CrushLocation = agent.location - // don't set public or cluster addr if prepare only - if agent.prepareOnly { - cephConfig.PublicAddr = "" - cephConfig.ClusterAddr = "" - } + // write the latest config to the config dir if err := mon.GenerateAdminConnectionConfigWithSettings(context, agent.cluster, cephConfig); err != nil { return fmt.Errorf("failed to write connection config. %+v", err) diff --git a/pkg/daemon/ceph/osd/device.go b/pkg/daemon/ceph/osd/device.go index 6c9336b95..d6dfafc5f 100644 --- a/pkg/daemon/ceph/osd/device.go +++ b/pkg/daemon/ceph/osd/device.go @@ -31,6 +31,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/mon" "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util" "github.com/rook/rook/pkg/util/display" "github.com/rook/rook/pkg/util/exec" "github.com/rook/rook/pkg/util/sys" @@ -438,7 +439,55 @@ func getStoreSettings(cfg *osdConfig) (map[string]string, error) { return settings, nil } -func writeConfigFile(cfg *osdConfig, context *clusterd.Context, cluster *mon.ClusterInfo, location string, prepareOnly bool) error { +func WriteConfigFile(context *clusterd.Context, cluster *mon.ClusterInfo, kv *k8sutil.ConfigMapKVStore, osdID int, storeConfig config.StoreConfig, nodeName, location string) error { + scheme, err := config.LoadScheme(kv, config.GetConfigStoreName(nodeName)) + if err != nil { + return fmt.Errorf("failed to load partition scheme: %+v", err) + } + + cfg := &osdConfig{id: osdID, configRoot: context.ConfigDir, rootPath: getOSDRootDir(context.ConfigDir, osdID), + storeConfig: storeConfig, kv: kv, storeName: config.GetConfigStoreName(nodeName)} + + // if a device, search the osd scheme for the requested osd id + device := false + for _, entry := range scheme.Entries { + if entry.ID == osdID { + device = true + cfg.partitionScheme = entry + cfg.uuid = entry.OsdUUID + logger.Infof("found osd %d in device map for uuid %s", osdID, cfg.uuid.String()) + break + } + } + // if not identified as a device, confirm that it is found in the map of directories + if !device { + cfg.dir = true + dirMap, err := config.LoadOSDDirMap(kv, nodeName) + if err != nil { + return fmt.Errorf("failed to load osd dir map. %+v", err) + } + + id, ok := dirMap[context.ConfigDir] + if !ok { + return fmt.Errorf("dir %s was not found in the dir map. %+v", context.ConfigDir, dirMap) + } + if id != osdID { + return fmt.Errorf("dir found in dirMap, but desired osd ID %d does not match dirMap id %d", osdID, id) + } + logger.Infof("found osd %d in dir map for path %s", osdID, context.ConfigDir) + } + + logger.Infof("updating config for osd %d", osdID) + err = writeConfigFile(cfg, context, cluster, location) + if err != nil { + return err + } + confFile := getOSDConfFilePath(cfg.rootPath, cluster.Name) + util.WriteFileToLog(logger, confFile) + return nil +} + +func writeConfigFile(cfg *osdConfig, context *clusterd.Context, cluster *mon.ClusterInfo, location string) error { cephConfig := mon.CreateDefaultCephConfig(context, cluster, cfg.rootPath) if isBluestore(cfg) { cephConfig.GlobalConfig.OsdObjectStore = config.Bluestore @@ -446,11 +495,6 @@ func writeConfigFile(cfg *osdConfig, context *clusterd.Context, cluster *mon.Clu cephConfig.GlobalConfig.OsdObjectStore = config.Filestore } cephConfig.CrushLocation = location - // don't set public or cluster addr if prepare only - if prepareOnly { - cephConfig.PublicAddr = "" - cephConfig.ClusterAddr = "" - } if cfg.dir || isFilestoreDevice(cfg) { // using the local file system requires some config overrides @@ -475,8 +519,8 @@ func writeConfigFile(cfg *osdConfig, context *clusterd.Context, cluster *mon.Clu return nil } -func initializeOSD(config *osdConfig, context *clusterd.Context, cluster *mon.ClusterInfo, location string, prepareOnly bool) error { - err := writeConfigFile(config, context, cluster, location, prepareOnly) +func initializeOSD(config *osdConfig, context *clusterd.Context, cluster *mon.ClusterInfo, location string) error { + err := writeConfigFile(config, context, cluster, location) if err != nil { return fmt.Errorf("failed to write config file: %+v", err) } diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 1521dbe17..7460057cb 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -308,7 +308,7 @@ func (c *Cluster) deleteBatchJob(name string) error { return nil } -func (c *Cluster) startOSDDaemon(nodeName string, config *provisionConfig, configMap *v1.ConfigMap, status *OrchestrationStatus) bool { +func (c *Cluster) startOSDDaemonsOnNode(nodeName string, config *provisionConfig, configMap *v1.ConfigMap, status *OrchestrationStatus) { osds := status.OSDs logger.Infof("starting %d osd daemons on node %s", len(osds), nodeName) @@ -317,14 +317,13 @@ func (c *Cluster) startOSDDaemon(nodeName string, config *provisionConfig, confi n := c.resolveNode(nodeName) if n == nil { config.addError("node %s did not resolve to start osds", nodeName) - return false + return } storeConfig := osdconfig.ToStoreConfig(n.Config) metadataDevice := osdconfig.MetadataDevice(n.Config) // start osds - succeeded := 0 for _, osd := range osds { logger.Debugf("start osd %v", osd) dp, err := c.makeDeployment(n.Name, config.devicesToUse[n.Name], n.Selection, n.Resources, storeConfig, metadataDevice, n.Location, osd) @@ -337,7 +336,7 @@ func (c *Cluster) startOSDDaemon(nodeName string, config *provisionConfig, confi } continue } - dp, err = c.context.Clientset.Extensions().Deployments(c.Namespace).Create(dp) + _, err = c.context.Clientset.Extensions().Deployments(c.Namespace).Create(dp) if err != nil { if !errors.IsAlreadyExists(err) { // we failed to create job, update the orchestration status for this node @@ -348,13 +347,14 @@ func (c *Cluster) startOSDDaemon(nodeName string, config *provisionConfig, confi } continue } - logger.Infof("deployment for osd %d already exists", osd.ID) + logger.Infof("deployment for osd %d already exists. updating if needed", osd.ID) + if err = k8sutil.UpdateDeploymentAndWait(c.context, dp, c.Namespace); err != nil { + config.addError(fmt.Sprintf("failed to update osd deployment %d. %+v", osd.ID, err)) + } } - logger.Infof("started deployment for osd %d (dir=%t, type=%s)", osd.ID, osd.IsDirectory, storeConfig.StoreType) - succeeded++ - } - return succeeded == len(osds) + logger.Infof("started deployment for osd %d (dir=%t, type=%s)", osd.ID, osd.IsDirectory, storeConfig.StoreType) + } } func (c *Cluster) handleRemovedNodes(config *provisionConfig) { @@ -561,5 +561,17 @@ func (c *Cluster) resolveNode(nodeName string) *rookalpha.Node { return nil } rookNode.Resources = k8sutil.MergeResourceRequirements(rookNode.Resources, c.resources) + + // ensure no invalid dirs are specified + var validDirs []rookalpha.Directory + for _, dir := range rookNode.Directories { + if dir.Path == k8sutil.DataDir || dir.Path == c.dataDirHostPath { + logger.Warningf("skipping directory %s that would conflict with the dataDirHostPath", dir.Path) + continue + } + validDirs = append(validDirs, dir) + } + rookNode.Directories = validDirs + return rookNode } diff --git a/pkg/operator/ceph/cluster/osd/pod.go b/pkg/operator/ceph/cluster/osd/pod.go index 62767600e..668d5411a 100644 --- a/pkg/operator/ceph/cluster/osd/pod.go +++ b/pkg/operator/ceph/cluster/osd/pod.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "path" + "path/filepath" "strconv" "strings" @@ -44,7 +45,6 @@ const ( osdWalSizeEnvVarName = "ROOK_OSD_WAL_SIZE" osdJournalSizeEnvVarName = "ROOK_OSD_JOURNAL_SIZE" osdMetadataDeviceEnvVarName = "ROOK_METADATA_DEVICE" - osdPrepareOnlyEnvVarName = "ROOK_OSD_PREPARE_ONLY" ) func (c *Cluster) makeJob(nodeName string, devices []rookalpha.Device, @@ -77,7 +77,8 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se storeConfig config.StoreConfig, metadataDevice, location string, osd OSDInfo) (*extensions.Deployment, error) { replicaCount := int32(1) - volumeMounts := []v1.VolumeMount{ + var volumeMounts []v1.VolumeMount + configVolumeMounts := []v1.VolumeMount{ {Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir}, k8sutil.ConfigOverrideMount(), } @@ -90,37 +91,52 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se logger.Infof("no data dir provided") } - // Mount the path to the directory-based osd unless it is already available under the dataDirHostPath - if osd.IsDirectory && !strings.HasPrefix(osd.DataPath, k8sutil.DataDir) { - volumeName := k8sutil.PathToVolumeName(osd.DataPath) - dataDirSource := v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: osd.DataPath}} - volumes = append(volumes, v1.Volume{Name: volumeName, VolumeSource: dataDirSource}) - volumeMounts = append(volumeMounts, v1.VolumeMount{Name: volumeName, MountPath: osd.DataPath}) - } + var dataDir string + if osd.IsDirectory { + // Mount the path to the directory-based osd + // osd.DataPath includes the osd subdirectory, so we want to mount the parent directory + parentDir := filepath.Dir(osd.DataPath) + dataDir = parentDir + // Skip the mount if this is the default directory being mounted. Inside the container, the path + // will be mounted at "/var/lib/rook" even if the dataDirHostPath is a different path on the host. + if parentDir != k8sutil.DataDir { + volumeName := k8sutil.PathToVolumeName(parentDir) + dataDirSource := v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: parentDir}} + volumes = append(volumes, v1.Volume{Name: volumeName, VolumeSource: dataDirSource}) + configVolumeMounts = append(configVolumeMounts, v1.VolumeMount{Name: volumeName, MountPath: parentDir}) + } + volumeMounts = configVolumeMounts + } else { + // Mount the required device + dataDir = k8sutil.DataDir - // Mount the required device - if !osd.IsDirectory { // create volume config for the data dir and /dev so the pod can access devices on the host devVolume := v1.Volume{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}} volumes = append(volumes, devVolume) devMount := v1.VolumeMount{Name: "devices", MountPath: "/dev"} - volumeMounts = append(volumeMounts, devMount) + volumeMounts = append(configVolumeMounts, devMount) } if len(volumes) == 0 { return nil, fmt.Errorf("empty volumes") } + osdID := strconv.Itoa(osd.ID) + tiniEnvVar := v1.EnvVar{Name: "TINI_SUBREAPER", Value: ""} envVars := []v1.EnvVar{ nodeNameEnvVar(), k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), - {Name: "TINI_SUBREAPER", Value: ""}, + tiniEnvVar, } + configEnvVars := append(c.getConfigEnvVars(storeConfig, dataDir, location), []v1.EnvVar{ + tiniEnvVar, + {Name: "ROOK_OSD_ID", Value: osdID}, + }...) commonArgs := []string{ "--foreground", - "--id", strconv.Itoa(osd.ID), + "--id", osdID, "--conf", osd.Config, "--osd-data", osd.DataPath, "--keyring", osd.KeyringPath, @@ -153,6 +169,12 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se privileged := true runAsUser := int64(0) readOnlyRootFilesystem := false + securityContext := &v1.SecurityContext{ + Privileged: &privileged, + RunAsUser: &runAsUser, + ReadOnlyRootFilesystem: &readOnlyRootFilesystem, + } + DNSPolicy := v1.DNSClusterFirst if c.HostNetwork { DNSPolicy = v1.DNSClusterFirstWithHostNet @@ -182,25 +204,32 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se Annotations: map[string]string{}, }, Spec: v1.PodSpec{ - NodeSelector: map[string]string{apis.LabelHostname: nodeName}, - RestartPolicy: v1.RestartPolicyAlways, - HostNetwork: c.HostNetwork, - HostPID: true, - DNSPolicy: DNSPolicy, + NodeSelector: map[string]string{apis.LabelHostname: nodeName}, + RestartPolicy: v1.RestartPolicyAlways, + ServiceAccountName: c.serviceAccount, + HostNetwork: c.HostNetwork, + HostPID: true, + DNSPolicy: DNSPolicy, + InitContainers: []v1.Container{ + { + Args: []string{"ceph", "osd", "config"}, + Name: "osd-init-config", + Image: k8sutil.MakeRookImage(c.Version), + VolumeMounts: configVolumeMounts, + Env: configEnvVars, + SecurityContext: securityContext, + }, + }, Containers: []v1.Container{ { - Command: command, - Args: args, - Name: appName, - Image: k8sutil.MakeRookImage(c.Version), - VolumeMounts: volumeMounts, - Env: envVars, - Resources: resources, - SecurityContext: &v1.SecurityContext{ - Privileged: &privileged, - RunAsUser: &runAsUser, - ReadOnlyRootFilesystem: &readOnlyRootFilesystem, - }, + Command: command, + Args: args, + Name: appName, + Image: k8sutil.MakeRookImage(c.Version), + VolumeMounts: volumeMounts, + Env: envVars, + Resources: resources, + SecurityContext: securityContext, }, }, Volumes: volumes, @@ -271,9 +300,7 @@ func (c *Cluster) provisionPodTemplateSpec(devices []rookalpha.Device, selection }, nil } -func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements, - storeConfig config.StoreConfig, metadataDevice, location string) v1.Container { - +func (c *Cluster) getConfigEnvVars(storeConfig config.StoreConfig, dataDir, location string) []v1.EnvVar { envVars := []v1.EnvVar{ nodeNameEnvVar(), {Name: "ROOK_CLUSTER_ID", Value: string(c.ownerRef.UID)}, @@ -283,11 +310,37 @@ func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection ro opmon.EndpointEnvVar(), opmon.SecretEnvVar(), opmon.AdminSecretEnvVar(), - k8sutil.ConfigDirEnvVar(), + k8sutil.ConfigDirEnvVar(dataDir), k8sutil.ConfigOverrideEnvVar(), - {Name: osdPrepareOnlyEnvVarName, Value: strconv.FormatBool(true)}, } + if storeConfig.StoreType != "" { + envVars = append(envVars, osdStoreEnvVar(storeConfig.StoreType)) + } + + if storeConfig.DatabaseSizeMB != 0 { + envVars = append(envVars, osdDatabaseSizeEnvVar(storeConfig.DatabaseSizeMB)) + } + + if storeConfig.WalSizeMB != 0 { + envVars = append(envVars, osdWalSizeEnvVar(storeConfig.WalSizeMB)) + } + + if storeConfig.JournalSizeMB != 0 { + envVars = append(envVars, osdJournalSizeEnvVar(storeConfig.JournalSizeMB)) + } + + if location != "" { + envVars = append(envVars, rookalpha.LocationEnvVar(location)) + } + + return envVars +} + +func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements, + storeConfig config.StoreConfig, metadataDevice, location string) v1.Container { + + envVars := c.getConfigEnvVars(storeConfig, k8sutil.DataDir, location) devMountNeeded := false privileged := false @@ -312,9 +365,8 @@ func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection ro devMountNeeded = true } - dataVolume := v1.VolumeMount{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir} volumeMounts := []v1.VolumeMount{ - dataVolume, + {Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir}, k8sutil.ConfigOverrideMount(), } if devMountNeeded { @@ -338,26 +390,6 @@ func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection ro } } - if storeConfig.StoreType != "" { - envVars = append(envVars, osdStoreEnvVar(storeConfig.StoreType)) - } - - if storeConfig.DatabaseSizeMB != 0 { - envVars = append(envVars, osdDatabaseSizeEnvVar(storeConfig.DatabaseSizeMB)) - } - - if storeConfig.WalSizeMB != 0 { - envVars = append(envVars, osdWalSizeEnvVar(storeConfig.WalSizeMB)) - } - - if storeConfig.JournalSizeMB != 0 { - envVars = append(envVars, osdJournalSizeEnvVar(storeConfig.JournalSizeMB)) - } - - if location != "" { - envVars = append(envVars, rookalpha.LocationEnvVar(location)) - } - // elevate to be privileged if it is going to mount devices or if running in a restricted environment such as openshift if devMountNeeded || os.Getenv("ROOK_HOSTPATH_REQUIRES_PRIVILEGED") == "true" { privileged = true diff --git a/pkg/operator/ceph/cluster/osd/pod_test.go b/pkg/operator/ceph/cluster/osd/pod_test.go index b72484bd7..b2876a475 100644 --- a/pkg/operator/ceph/cluster/osd/pod_test.go +++ b/pkg/operator/ceph/cluster/osd/pod_test.go @@ -25,6 +25,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -78,7 +79,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.NotNil(t, deployment) assert.Equal(t, "rook-ceph-osd-id-0", deployment.Name) assert.Equal(t, c.Namespace, deployment.Namespace) - assert.Equal(t, "", deployment.Spec.Template.Spec.ServiceAccountName) + assert.Equal(t, "mysa", deployment.Spec.Template.Spec.ServiceAccountName) assert.Equal(t, int32(1), *(deployment.Spec.Replicas)) assert.Equal(t, "node1", deployment.Spec.Template.Spec.NodeSelector[apis.LabelHostname]) assert.Equal(t, v1.RestartPolicyAlways, deployment.Spec.Template.Spec.RestartPolicy) @@ -100,6 +101,13 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, c.Namespace, deployment.Spec.Template.ObjectMeta.Labels["rook_cluster"]) assert.Equal(t, 0, len(deployment.Spec.Template.ObjectMeta.Annotations)) + assert.Equal(t, 1, len(deployment.Spec.Template.Spec.InitContainers)) + initCont := deployment.Spec.Template.Spec.InitContainers[0] + assert.Equal(t, "rook/rook:myversion", initCont.Image) + assert.Equal(t, "osd-init-config", initCont.Name) + assert.Equal(t, 2, len(initCont.VolumeMounts)) + + assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont := deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, "rook/rook:myversion", cont.Image) if devMountNeeded { @@ -141,18 +149,57 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) { clientset := fake.NewSimpleClientset() c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "", - storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) + storageSpec, "/var/lib/rook", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) n := c.Storage.ResolveNode(storageSpec.Nodes[0].Name) osd := OSDInfo{ - ID: 0, + ID: 0, + IsDirectory: true, + DataPath: "/my/root/path/osd1", } deployment, err := c.makeDeployment(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) assert.NotNil(t, deployment) assert.Nil(t, err) - // pod spec should have a volume for the given dir + // pod spec should have a volume for the given dir in the main container and the init container podSpec := deployment.Spec.Template.Spec + assert.Equal(t, 3, len(podSpec.Volumes)) + require.Equal(t, 1, len(podSpec.Containers)) + cont := podSpec.Containers[0] + assert.Equal(t, 3, len(cont.VolumeMounts)) + assert.Equal(t, "/var/lib/rook", cont.VolumeMounts[0].MountPath) + assert.Equal(t, "/etc/rook/config", cont.VolumeMounts[1].MountPath) + assert.Equal(t, "/my/root/path", cont.VolumeMounts[2].MountPath) + + require.Equal(t, 1, len(podSpec.InitContainers)) + initCont := podSpec.InitContainers[0] + assert.Equal(t, 3, len(initCont.VolumeMounts)) + assert.Equal(t, "/var/lib/rook", initCont.VolumeMounts[0].MountPath) + assert.Equal(t, "/etc/rook/config", initCont.VolumeMounts[1].MountPath) + assert.Equal(t, "/my/root/path", initCont.VolumeMounts[2].MountPath) + + // the default osd created on a node will be under /var/lib/rook, which won't need an extra mount + osd = OSDInfo{ + ID: 1, + IsDirectory: true, + DataPath: "/var/lib/rook/osd1", + } + deployment, err = c.makeDeployment(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) + assert.NotNil(t, deployment) + assert.Nil(t, err) + // pod spec should have a volume for the given dir in the main container and the init container + podSpec = deployment.Spec.Template.Spec assert.Equal(t, 2, len(podSpec.Volumes)) + require.Equal(t, 1, len(podSpec.Containers)) + cont = podSpec.Containers[0] + require.Equal(t, 2, len(cont.VolumeMounts)) + assert.Equal(t, "/var/lib/rook", cont.VolumeMounts[0].MountPath) + assert.Equal(t, "/etc/rook/config", cont.VolumeMounts[1].MountPath) + + require.Equal(t, 1, len(podSpec.InitContainers)) + initCont = podSpec.InitContainers[0] + require.Equal(t, 2, len(initCont.VolumeMounts)) + assert.Equal(t, "/var/lib/rook", initCont.VolumeMounts[0].MountPath) + assert.Equal(t, "/etc/rook/config", initCont.VolumeMounts[1].MountPath) } func TestStorageSpecConfig(t *testing.T) { diff --git a/pkg/operator/ceph/cluster/osd/status.go b/pkg/operator/ceph/cluster/osd/status.go index 450b904fe..35918596c 100644 --- a/pkg/operator/ceph/cluster/osd/status.go +++ b/pkg/operator/ceph/cluster/osd/status.go @@ -231,10 +231,11 @@ func (c *Cluster) handleStatusConfigMapStatus(nodeName string, config *provision logger.Infof("osd orchestration status for node %s is %s", nodeName, status.Status) if status.Status == OrchestrationStatusCompleted { - if !configOSDs || c.startOSDDaemon(nodeName, config, configMap, status) { - // upon success provisioning or removing OSDs on the node, remove the status configmap - c.kv.ClearStore(fmt.Sprintf(orchestrationStatusMapName, nodeName)) + if configOSDs { + c.startOSDDaemonsOnNode(nodeName, config, configMap, status) } + // remove the status configmap that indicated the progress + c.kv.ClearStore(fmt.Sprintf(orchestrationStatusMapName, nodeName)) return true } diff --git a/pkg/operator/k8sutil/deployment.go b/pkg/operator/k8sutil/deployment.go new file mode 100644 index 000000000..503ee8645 --- /dev/null +++ b/pkg/operator/k8sutil/deployment.go @@ -0,0 +1,61 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package k8sutil + +import ( + "fmt" + "time" + + "github.com/rook/rook/pkg/clusterd" + "k8s.io/api/extensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func UpdateDeploymentAndWait(context *clusterd.Context, deployment *v1beta1.Deployment, namespace string) error { + original, err := context.Clientset.Extensions().Deployments(namespace).Get(deployment.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get deployment %s. %+v", deployment.Name, err) + } + + logger.Infof("updating deployment %s", deployment.Name) + if _, err := context.Clientset.Extensions().Deployments(namespace).Update(deployment); err != nil { + return fmt.Errorf("failed to update deployment %s. %+v", deployment.Name, err) + } + + // wait for the deployment to be restarted + sleepTime := 2 + attempts := 30 + if original.Spec.ProgressDeadlineSeconds != nil { + // make the attempts double the progress deadline since the pod is both stopping and starting + attempts = 2 * (int(*original.Spec.ProgressDeadlineSeconds) / sleepTime) + } + for i := 0; i < attempts; i++ { + // check for the status of the deployment + d, err := context.Clientset.Extensions().Deployments(namespace).Get(deployment.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get deployment %s. %+v", deployment.Name, err) + } + if d.Status.ObservedGeneration != original.Status.ObservedGeneration && d.Status.UpdatedReplicas > 0 && d.Status.ReadyReplicas > 0 { + logger.Infof("finished waiting for updated deployment %s", d.Name) + return nil + } + + logger.Debugf("deployment %s status=%v", d.Name, d.Status) + time.Sleep(time.Duration(sleepTime) * time.Second) + } + + return fmt.Errorf("gave up waiting for deployment %s to update", deployment.Name) +} diff --git a/pkg/operator/k8sutil/pod.go b/pkg/operator/k8sutil/pod.go index db2cd6d98..75204ec54 100644 --- a/pkg/operator/k8sutil/pod.go +++ b/pkg/operator/k8sutil/pod.go @@ -89,13 +89,16 @@ func NodeEnvVar() v1.EnvVar { } // ConfigDirEnvVar config dir env var -func ConfigDirEnvVar() v1.EnvVar { - return v1.EnvVar{Name: "ROOK_CONFIG_DIR", Value: DataDir} +func ConfigDirEnvVar(dataDir string) v1.EnvVar { + return v1.EnvVar{Name: "ROOK_CONFIG_DIR", Value: dataDir} } func GetContainerImage(pod *v1.Pod, name string) (string, error) { + return GetSpecContainerImage(pod.Spec, name) +} - image, err := GetMatchingContainer(pod.Spec.Containers, name) +func GetSpecContainerImage(spec v1.PodSpec, name string) (string, error) { + image, err := GetMatchingContainer(spec.Containers, name) if err != nil { return "", err }