diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index a0fe81eb6..036851ea2 100644 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -17,6 +17,9 @@ metadata: name: rook-ceph namespace: rook-ceph spec: + cephVersion: + # see the "Cluster Settings" section below for more details on which image of ceph to run + image: ceph/ceph:v13.2.2-20181023 dataDirHostPath: /var/lib/rook serviceAccount: rook-ceph-cluster storage: @@ -36,6 +39,13 @@ Settings can be specified at the global level to apply to the cluster as a whole ### Cluster Settings +- `cephVersion`: The version information for launching the ceph daemons. + - `image`: The image used for running the ceph daemons. For example, `ceph/ceph:v12.2.9-20181026` or `ceph/ceph:v13.2.2-20181023`. + For the latest ceph images, see the [Ceph DockerHub](https://hub.docker.com/r/ceph/ceph/tags/). + To ensure a consistent version of the image is running across all nodes in the cluster, it is recommended to use a very specific image version. + Tags also exist that would give the latest version, but they are only recommended for test environments. For example, the tag `v13` will be updated each time a new mimic build is released. + Using the `v13` or similar tag is not recommended in production because it may lead to inconsistent versions of the image running across different nodes in the cluster. + - `allowUnsupported`: If `true`, allow an unsupported major version of the Ceph release. Currently only `luminous` and `mimic` are supported, so `nautilus` would require this to be set to `true`. Should be set to `false` in production. - `dataDirHostPath`: The path on the host ([hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath)) where config and data should be stored for each of the services. If the directory does not exist, it will be created. Because this directory persists on the host, it will remain after pods are deleted. - On **Minikube** environments, use `/data/rook`. Minikube boots into a tmpfs but it provides some [directories](https://github.com/kubernetes/minikube/blob/master/docs/persistent_volumes.md) where files can be persisted across reboots. Using one of these directories will ensure that Rook's data and configuration files are persisted and that enough storage space is available. - **WARNING**: For test scenarios, if you delete a cluster and start a new cluster on the same hosts, the path used by `dataDirHostPath` must be deleted. Otherwise, stale keys and other config will remain from the previous cluster and the new mons will fail to start. @@ -155,6 +165,8 @@ metadata: name: rook-ceph namespace: rook-ceph spec: + cephVersion: + image: ceph/ceph:v13.2.2-20181023 dataDirHostPath: /var/lib/rook serviceAccount: rook-ceph-cluster network: @@ -184,6 +196,8 @@ metadata: name: rook-ceph namespace: rook-ceph spec: + cephVersion: + image: ceph/ceph:v13.2.2-20181023 dataDirHostPath: /var/lib/rook serviceAccount: rook-ceph-cluster network: @@ -225,6 +239,8 @@ metadata: name: rook-ceph namespace: rook-ceph spec: + cephVersion: + image: ceph/ceph:v13.2.2-20181023 dataDirHostPath: /var/lib/rook serviceAccount: rook-ceph-cluster network: @@ -260,6 +276,8 @@ metadata: name: rook-ceph namespace: rook-ceph spec: + cephVersion: + image: ceph/ceph:v13.2.2-20181023 dataDirHostPath: /var/lib/rook serviceAccount: rook-ceph-cluster network: @@ -303,6 +321,8 @@ metadata: name: rook-ceph namespace: rook-ceph spec: + cephVersion: + image: ceph/ceph:v13.2.2-20181023 dataDirHostPath: /var/lib/rook serviceAccount: rook-ceph-cluster # cluster level resource requests/limits configuration diff --git a/Documentation/ceph-dashboard.md b/Documentation/ceph-dashboard.md index 141179217..7363bc8f9 100644 --- a/Documentation/ceph-dashboard.md +++ b/Documentation/ceph-dashboard.md @@ -14,7 +14,7 @@ and more. Rook makes it simple to enable the dashboard. ## Enable the Dashboard -The [dashboard](http://docs.ceph.com/docs/luminous/mgr/dashboard/) can be enabled with settings in the cluster CRD. The cluster CRD must have the dashboard `enabled` setting set to `true`. +The [dashboard](http://docs.ceph.com/docs/mimic/mgr/dashboard/) can be enabled with settings in the cluster CRD. The cluster CRD must have the dashboard `enabled` setting set to `true`. This is the default setting in the example manifests. ```yaml spec: @@ -22,19 +22,32 @@ This is the default setting in the example manifests. enabled: true ``` -The Rook operator will enable the ceph-mgr dashboard module to listen on the default port 7000. -A K8s service will also be created to expose that port inside the cluster. +The Rook operator will enable the ceph-mgr dashboard module. A K8s service will be created to expose that port inside the cluster. The ports enabled by Rook will depend +on the version of Ceph that is running: +- Luminous: Port 7000 on http +- Mimic and newer: Port 8443 on https + +This example shows that port 8443 was configured for Mimic or newer. ```bash kubectl -n rook-ceph get service -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -rook-ceph-mgr ClusterIP 10.108.111.192 9283/TCP 3h -rook-ceph-mgr-dashboard ClusterIP 10.110.113.240 7000/TCP 3h +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +rook-ceph-mgr ClusterIP 10.108.111.192 9283/TCP 3h +rook-ceph-mgr-dashboard ClusterIP 10.110.113.240 8443/TCP 3h ``` The first service is for reporting the [Prometheus metrics](monitoring.md), while the latter service is for the dashboard. If you are on a node in the cluster, you will be able to connect to the dashboard by using either the -DNS name of the service at `http://rook-ceph-mgr-dashboard:7000` or by connecting to the cluster IP, -in this example at `http://10.110.113.240:7000`. +DNS name of the service at `https://rook-ceph-mgr-dashboard-https:8443` or by connecting to the cluster IP, +in this example at `https://10.110.113.240:8443`. + +### Credentials + +After you connect to the dashboard you will need to login for secure access. Rook creates a default user named +`admin` and generates a secret called `rook-ceph-dashboard-admin-password` in the namespace where rook is running. +To retrieve the generated password, you can run the following: +``` +kubectl -n rook-ceph get secret rook-ceph-dashboard-password -o yaml | grep "password:" | awk '{print $2}' | base64 --decode +``` ## Viewing the Dashboard External to the Cluster @@ -46,12 +59,14 @@ You can use an [Ingress Controller](https://kubernetes.io/docs/concepts/services NodePort, LoadBalancer, or ExternalIPs. The simplest way to expose the service in minikube or similar environment is using the NodePort to open a port on the -VM that can be accessed by the host. To create a service with the NodePort, save this yaml as `dashboard-external.yaml`: +VM that can be accessed by the host. To create a service with the NodePort, save this yaml as `dashboard-external-https.yaml`. +(For Luminous you will need to set the `port` and `targetPort` to 7000 and connect via `http`.) + ```yaml apiVersion: v1 kind: Service metadata: - name: rook-ceph-mgr-dashboard-external + name: rook-ceph-mgr-dashboard-external-https namespace: rook-ceph labels: app: rook-ceph-mgr @@ -59,9 +74,9 @@ metadata: spec: ports: - name: dashboard - port: 7000 + port: 8443 protocol: TCP - targetPort: 7000 + targetPort: 8443 selector: app: rook-ceph-mgr rook_cluster: rook-ceph @@ -77,12 +92,12 @@ $ kubectl create -f dashboard-external.yaml You will see the new service `rook-ceph-mgr-dashboard-external` created: ```bash $ kubectl -n rook-ceph get service -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -rook-ceph-mgr ClusterIP 10.108.111.192 9283/TCP 4h -rook-ceph-mgr-dashboard ClusterIP 10.110.113.240 7000/TCP 4h -rook-ceph-mgr-dashboard-external NodePort 10.101.209.6 7000:31176/TCP 4h +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +rook-ceph-mgr ClusterIP 10.108.111.192 9283/TCP 4h +rook-ceph-mgr-dashboard ClusterIP 10.110.113.240 8443/TCP 4h +rook-ceph-mgr-dashboard-external-https NodePort 10.101.209.6 8443:31176/TCP 4h ``` -In this example, port `31176` will be opened to expose port `7000` from the ceph-mgr pod. Find the ip address -of the VM. If using minikube, you can `minikube ssh` to the machine and `ifconfig` to find the ip address. -Now you can enter the URL in your browser such as `http://192.168.99.110:31176` and the dashboard will appear. +In this example, port `31176` will be opened to expose port `8443` from the ceph-mgr pod. Find the ip address +of the VM. If using minikube, you can run `minikube ip` to find the ip address. +Now you can enter the URL in your browser such as `https://192.168.99.110:31176` and the dashboard will appear. diff --git a/Documentation/ceph-quickstart.md b/Documentation/ceph-quickstart.md index da05fe1e6..dded97d31 100644 --- a/Documentation/ceph-quickstart.md +++ b/Documentation/ceph-quickstart.md @@ -117,6 +117,9 @@ metadata: name: rook-ceph namespace: rook-ceph spec: + cephVersion: + # For the latest ceph images, see https://hub.docker.com/r/ceph/ceph/tags + image: ceph/ceph:v13.2.2-20181023 dataDirHostPath: /var/lib/rook dashboard: enabled: true diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index f55921eab..326e4c694 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -4,8 +4,10 @@ ## Notable Features +- Different versions of Ceph can be orchestrated by Rook. Both Luminous and Mimic are now supported, with Nautilus coming soon. +The version of Ceph is specified in the cluster CRD with the cephVersion.image property. For example, to run Mimic you could use image `ceph/ceph:v13.2.2-20181023` +or any other image found on the [Ceph DockerHub](https://hub.docker.com/r/ceph/ceph/tags). - The `fsType` default for StorageClass examples are now using XFS to bring it in line with Ceph recommendations. -- Ceph is updated from Luminous 12.2.5 to 12.2.7. - Ceph OSDs will be automatically updated by the operator when there is a change to the operator version or when the OSD configuration changes. See the [OSD upgrade notes](Documentation/upgrade-patch.md#object-storage-daemons-osds). - Rook Ceph block storage provisioner can now correctly create erasure coded block images. See [Advanced Example: Erasure Coded Block Storage](Documentation/block.md#advanced-example-erasure-coded-block-storage) for an example usage. - [Network File System (NFS)](https://github.com/nfs-ganesha/nfs-ganesha/wiki) is now supported by Rook with a new operator to deploy and manage this widely used server. NFS servers can be automatically deployed by creating an instance of the new `nfsservers.nfs.rook.io` custom resource. See the [NFS server user guide](Documentation/nfs.md) to get started with NFS. @@ -19,7 +21,7 @@ ## Breaking Changes - Ceph mons are [named consistently](https://github.com/rook/rook/issues/1751) with other daemons with the letters a, b, c, etc. - Ceph mons are now created with Deployments instead of ReplicaSets to improve the upgrade implementation. -- Ceph mon, mgr, and mds container names in pods have changed with the refactors to initialize the +- Ceph mon, mgr, mds, and rgw container names in pods have changed with the refactors to initialize the daemon environments via pod **InitContainers** and run the Ceph daemons directly from the container entrypoint. diff --git a/cluster/charts/rook-ceph/templates/clusterrole.yaml b/cluster/charts/rook-ceph/templates/clusterrole.yaml index 8f876f90f..9016649a6 100644 --- a/cluster/charts/rook-ceph/templates/clusterrole.yaml +++ b/cluster/charts/rook-ceph/templates/clusterrole.yaml @@ -13,6 +13,7 @@ rules: resources: - secrets - pods + - pods/log - services - configmaps verbs: diff --git a/cluster/examples/kubernetes/ceph/cluster.yaml b/cluster/examples/kubernetes/ceph/cluster.yaml index 33c012bf2..8e7049782 100644 --- a/cluster/examples/kubernetes/ceph/cluster.yaml +++ b/cluster/examples/kubernetes/ceph/cluster.yaml @@ -55,6 +55,16 @@ metadata: name: rook-ceph namespace: rook-ceph spec: + cephVersion: + # The container image used to launch the Ceph daemon pods (mon, mgr, osd, mds, rgw). + # v12 is luminous, v13 is mimic, and v14 is nautilus. + # RECOMMENDATION: In production, use a specific version tag instead of the general v13 flag, which pulls the latest release and could result in different + # versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/. + image: ceph/ceph:v13 + # Whether to allow unsupported versions of Ceph. Currently only luminous and mimic are supported. + # After nautilus is released, Rook will be updated to support nautilus. + # Do not set to true in production. + allowUnsupported: false # The path on the host where configuration files will be persisted. If not specified, a kubernetes emptyDir will be created (not recommended). # Important: if you reinstall the cluster, make sure you delete this directory from each host or else the mons will fail to start on the new cluster. # In Minikube, the '/data' directory is configured to persist across reboots. Use "/data/rook" in Minikube environment. diff --git a/cluster/examples/kubernetes/ceph/dashboard-external.yaml b/cluster/examples/kubernetes/ceph/dashboard-external-http.yaml similarity index 87% rename from cluster/examples/kubernetes/ceph/dashboard-external.yaml rename to cluster/examples/kubernetes/ceph/dashboard-external-http.yaml index 70c4d25a8..c67ac4c5e 100644 --- a/cluster/examples/kubernetes/ceph/dashboard-external.yaml +++ b/cluster/examples/kubernetes/ceph/dashboard-external-http.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: rook-ceph-mgr-dashboard-external + name: rook-ceph-mgr-dashboard-external-http namespace: rook-ceph labels: app: rook-ceph-mgr diff --git a/cluster/examples/kubernetes/ceph/dashboard-external-https.yaml b/cluster/examples/kubernetes/ceph/dashboard-external-https.yaml new file mode 100644 index 000000000..4b94a3778 --- /dev/null +++ b/cluster/examples/kubernetes/ceph/dashboard-external-https.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: rook-ceph-mgr-dashboard-external-https + namespace: rook-ceph + labels: + app: rook-ceph-mgr + rook_cluster: rook-ceph +spec: + ports: + - name: dashboard + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + app: rook-ceph-mgr + rook_cluster: rook-ceph + sessionAffinity: None + type: NodePort diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index 7f9cb834a..9eb17551f 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -18,6 +18,51 @@ spec: - rcc scope: Namespaced version: v1beta1 + validation: + openAPIV3Schema: + properties: + spec: + properties: + cephVersion: + properties: + allowUnsupported: + type: boolean + image: + type: string + name: + pattern: ^(luminous|mimic|nautilus)$ + type: string + dashboard: + properties: + enabled: + type: boolean + dataDirHostPath: + pattern: ^/(\S+) + type: string + mon: + properties: + allowMultiplePerNode: + type: boolean + count: + maximum: 9 + minimum: 1 + type: integer + required: + - count + network: + properties: + hostNetwork: + type: boolean + storage: + properties: + nodes: + items: {} + type: array + useAllDevices: {} + useAllNodes: + type: boolean + required: + - mon --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -97,6 +142,7 @@ rules: resources: - secrets - pods + - pods/log - services - configmaps verbs: diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index b83c5611a..28498f6af 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -44,6 +44,11 @@ var osdConfigCmd = &cobra.Command{ Short: "Updates ceph.conf for the osd", Hidden: true, } +var copyBinariesCmd = &cobra.Command{ + Use: "copybins", + Short: "Copies rook binaries for use by a ceph container", + Hidden: true, +} var provisionCmd = &cobra.Command{ Use: "provision", Short: "Generates osd config and prepares an osd for runtime", @@ -77,7 +82,9 @@ func addOSDFlags(command *cobra.Command) { // flags for generating the osd config osdConfigCmd.Flags().IntVar(&osdID, "osd-id", -1, "osd id for which to generate config") - osdConfigCmd.Flags().StringVar(©BinariesPath, "copy-binaries-path", "", "If specified, copy the rook binaries to this path for use by the daemon container") + + // flag for copying the rook binaries for use by a ceph container + copyBinariesCmd.Flags().StringVar(©BinariesPath, "path", "", "Copy the rook binaries to this path for use by a ceph container") // flags for running filestore on a device filestoreDeviceCmd.Flags().StringVar(&mountSourcePath, "source-path", "", "the source path of the device to mount") @@ -85,6 +92,7 @@ func addOSDFlags(command *cobra.Command) { // add the subcommands to the parent osd command osdCmd.AddCommand(osdConfigCmd) + osdCmd.AddCommand(copyBinariesCmd) osdCmd.AddCommand(provisionCmd) osdCmd.AddCommand(filestoreDeviceCmd) } @@ -106,10 +114,12 @@ func init() { addCephFlags(osdCmd) flags.SetFlagsFromEnv(osdCmd.Flags(), rook.RookEnvVarPrefix) flags.SetFlagsFromEnv(osdConfigCmd.Flags(), rook.RookEnvVarPrefix) + flags.SetFlagsFromEnv(copyBinariesCmd.Flags(), rook.RookEnvVarPrefix) flags.SetFlagsFromEnv(provisionCmd.Flags(), rook.RookEnvVarPrefix) flags.SetFlagsFromEnv(filestoreDeviceCmd.Flags(), rook.RookEnvVarPrefix) osdConfigCmd.RunE = writeOSDConfig + copyBinariesCmd.RunE = copyRookBinaries provisionCmd.RunE = prepareOSD filestoreDeviceCmd.RunE = runFilestoreDeviceOSD } @@ -174,12 +184,17 @@ func writeOSDConfig(cmd *cobra.Command, args []string) error { if err := osddaemon.WriteConfigFile(context, &clusterInfo, kv, osdID, cfg.storeConfig, cfg.nodeName, crushLocation); err != nil { logger.Errorf("failed to write osd config file. %+v", err) } - if copyBinariesPath != "" { - if err := osddaemon.CopyBinariesForDaemon(copyBinariesPath); err != nil { - logger.Errorf("failed to copy rook binaries for filestore device. %+v", err) - } else { - logger.Infof("successfully copied rook binaries") - } + return nil +} + +func copyRookBinaries(cmd *cobra.Command, args []string) error { + if err := flags.VerifyRequiredFlags(copyBinariesCmd, []string{"path"}); err != nil { + return err + } + if err := osddaemon.CopyBinariesForDaemon(copyBinariesPath); err != nil { + logger.Errorf("failed to copy rook binaries for filestore device. %+v", err) + } else { + logger.Infof("successfully copied rook binaries") } return nil } diff --git a/design/decouple-ceph-version.md b/design/decouple-ceph-version.md index fae35a3a4..8bb6f527b 100644 --- a/design/decouple-ceph-version.md +++ b/design/decouple-ceph-version.md @@ -44,10 +44,10 @@ spec: ### Ceph Version -The Ceph version is defined as the property `cephImage` in the Cluster CRD. All Ceph daemon containers launched by the Rook operator will use this image, including the mon, mgr, +The Ceph version is defined under the property `cephVersion` in the Cluster CRD. All Ceph daemon containers launched by the Rook operator will use this image, including the mon, mgr, osd, rgw, and mds pods. The significance of this approach is that the Rook binary is not included in the daemon containers. All initialization performed by Rook to generate the Ceph config and prepare the daemons must be completed in an [init container](https://github.com/rook/rook/issues/2003). Once the Rook init containers complete their execution, the daemon container will run the Ceph image. The daemon container will no longer have Rook running. -In the following Cluster CRD example, the Ceph version is Mimic 13.2.1. +In the following Cluster CRD example, the Ceph version is Mimic `13.2.2` built on 23 Oct 2018. ```yaml apiVersion: ceph.rook.io/v1beta1 @@ -56,14 +56,15 @@ metadata: name: rook-ceph namespace: rook-ceph spec: - cephImage: ceph/ceph:v13.2.1 + cephVersion: + image: ceph/ceph:v13.2.2-20181023 ``` ### Operator Requirements The operator needs to run the Ceph client tools to manage the cluster. For example, the `ceph` tool is needed for general Ceph configuration and status, while `radosgw-admin` is required for managing an object store. Therefore, all the necessary client tools will still be included in the Rook image. -The client tools are tested by the Ceph team to be backward and forward compatible by two versions. Therefore, the operator can support a version of Ceph up to two versions older than the client tools it contains. +The client tools are tested by the Ceph team to be backward and forward compatible by two versions. This means the operator can support a version of Ceph up to two versions older than the client tools it contains. With each Rook release, the tools will be included from the latest release of Ceph. For example, in 0.9 Rook will likely include the Mimic tools. Upgrades would be supported from Luminous to Mimic. Rook 0.9 can also be tested to support upgrades to Nautilus since they may be released in the same time frame. Since the Ceph tools are forward compatible, the Mimic tools will be sufficient to support upgrading to Nautilus. If Nautilus is released after Rook 0.9, a patch release can be made to 0.9 so that Rook can officially support the upgrade at that point. The changes in the patch release should be minimal since upgrading to Nautilus could have been mostly planned for in 0.9. @@ -76,8 +77,8 @@ The operator will be made to understand differences in the Ceph versions that ar Rook will support a very specific list of major versions. Outside these versions, Rook will not be aware of the needs for configuring and upgrading the cluster. In v0.9, the supported versions will be: -- Luminous (ceph/ceph:v12.2.x) -- Mimic (ceph/ceph:v13.2.x) +- luminous (ceph/ceph:v12.2.x) +- mimic (ceph/ceph:v13.2.x) Depending on the timing of the 0.9 and Nautilus releases, Nautilus will likely be supported either in 0.9 or a patch release. Versions not yet officially supported can be tested with settings in the CRD to be mentioned below. @@ -89,7 +90,7 @@ various Mimic patch releases. The flexibility during upgrades will now be improved since the upgrade of Rook will be independent from the upgrade to the Ceph version. - To upgrade Rook, update the version of the Rook operator container -- To upgrade Ceph, make sure Rook is running the latest release, then update the `cephImage` in the cluster CRD +- To upgrade Ceph, make sure Rook is running the latest release, then update the `cephVersion.image` in the cluster CRD The versions to be supported during upgrade will be a specific set for each version of Rook. In 0.9, it is anticipated that the only upgrade of Ceph supported would only be Luminous to Mimic. When Rook officially adds support for a release of Ceph (ie. Nautilus), the upgrade path will also be supported from one previous version. @@ -117,23 +118,26 @@ To allow more control over the upgrade, we define `upgradePolicy` settings. They - Allow for testing of future versions that are not officially supported The settings in the CRD to accommodate the design include: -- `newCephImage`: The version of the image to start applying to the daemons specified in the `upgrade` list. If `newCephImage` is not set, the operator will ensure that all daemons are running the version specified by the `cephImage` setting. -- `upgrade`: A list of daemons that should be upgraded to the version `newCephImage`. The daemons include `mon`, `osd`, `mgr`, `rgw`, and `mds`. The ordering of the list will be ignored as Rook will only support ordering as it determines necessary for a version. If there are special upgrade actions in the future, they could be named and added to this list. -- `allowUnrecognizedVersion`: If `false`, the operator would refuse to upgrade the Ceph version if it doesn't support or recognize that version. If `true`, Rook would go ahead and blindly set the image version and assume the pod specs should match `unrecognizedMajorVersion`. This would allow testing of upgrade to unreleased versions. The default is `false`. -- `unrecognizedMajorVersion`: The version of pod spec to use when orchestrating an unrecognized version. Possible values are the names of versions that Rook understands: `luminous`, `mimic`, or `nautilus`. +- `upgradePolicy.cephVersion`: The version of the image to start applying to the daemons specified in the `components` list. + - `allowUnsupported`: If `false`, the operator would refuse to upgrade the Ceph version if it doesn't support or recognize that version. This would allow testing of upgrade to unreleased versions. The default is `false`. +- `upgradePolicy.components`: A list of daemons or other components that should be upgraded to the version `newCephVersion`. The daemons include `mon`, `osd`, `mgr`, `rgw`, and `mds`. The ordering of the list will be ignored as Rook will only support ordering as it determines necessary for a version. If there are special upgrade actions in the future, they could be named and added to this list. For example, with the settings below the operator would only upgrade the mons to mimic, while other daemons would remain on luminous. When the admin is ready, he would add more daemons to the list. ```yaml spec: - cephImage: ceph/ceph:v12.2.7 + cephVersion: + image: ceph/ceph:v12.2.9-20181026 + allowUnsupported: false upgradePolicy: - newCephImage: ceph/ceph:v13.2.1 - upgrade: + cephVersion: + image: ceph/ceph:v13.2.2-20181023 + allowUnsupported: false + components: - mon ``` -When the admin is completed with the upgrade or he is ready to allow Rook to complete the full upgrade for all daemons, he would set `cephImage: ceph/ceph:v13.2.1`, and the operator would ignore the `upgradePolicy` since the `cephImage` and `newCephImage` match. +When the admin is completed with the upgrade or he is ready to allow Rook to complete the full upgrade for all daemons, he would set `cephVersion.image: ceph/ceph:v13.2.2`, and the operator would ignore the `upgradePolicy` since the `cephVersion` and `upgradePolicy.cephVersion` match. If the admin wants to pause or otherwise control the upgrade closely, there are a couple of natural back doors: - Deleting the operator pod will effectively pause the upgrade. Starting the operator pod up again would resume the upgrade. @@ -141,18 +145,19 @@ If the admin wants to pause or otherwise control the upgrade closely, there are #### Developer controls -If a developer wants to test the upgrade from mimic to nautilus, he would first create the cluster based on mimic. Then he would update the crd with the "unrecognized version" attributes in the CRD to specify nautilus such as: +If a developer wants to test the upgrade from mimic to nautilus, he would first create the cluster based on mimic. Then he would update the crd with the "unrecognized version" attribute in the CRD to specify nautilus such as: ```yaml spec: - cephImage: ceph/ceph:v14.1.1 - upgradePolicy: - allowUnrecognizedVersion: true - unrecognizedMajorVersion: nautilus + cephVersion: + image: ceph/ceph:v14.1.1 + allowUnsupported: true ``` +Until Nautilus builds are released, the latest Nautilus build can be tested by using the image `ceph/daemon-base:latest-master`. + ### Default Version -For backward compatibility, if the `cephImage` property is empty, the operator will need to internally set a default version of Ceph. +For backward compatibility, if the `cephVersion` property is not set, the operator will need to internally set a default version of Ceph. The operator will assume the desired Ceph version is Luminous 12.2.7, which was shipped with Rook v0.8. This default will allow the Rook upgrade from v0.8 to v0.9 to only impact the Rook version and hold the Ceph version at Luminous. -After the Rook upgrade to v0.9, the user can choose to set the `cephImage` property to some newer version of Ceph such as Mimic. +After the Rook upgrade to v0.9, the user can choose to set the `cephVersion` property to some newer version of Ceph such as mimic. diff --git a/images/ceph/Dockerfile b/images/ceph/Dockerfile index 3e95fef9d..e1e7621ce 100644 --- a/images/ceph/Dockerfile +++ b/images/ceph/Dockerfile @@ -12,12 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM BASEIMAGE - -RUN yum --assumeyes install \ - net-tools \ - nmap-ncat && \ - yum clean all && rm -rf /tmp/* /var/tmp/* +# the latest mimic release is the base image +FROM ceph/ceph:v13 ARG ARCH ARG TINI_VERSION diff --git a/images/ceph/Makefile b/images/ceph/Makefile index 0edfa7bac..ee19dc350 100755 --- a/images/ceph/Makefile +++ b/images/ceph/Makefile @@ -19,8 +19,7 @@ include ../image.mk CEPH_IMAGE = $(BUILD_REGISTRY)/ceph-$(GOARCH) -CEPH_VERSION = $(shell cat ceph_version) -BASEIMAGE = $(CEPH_VERSION)-$(PLATFORM_ARCH) +BASEIMAGE = ubuntu:18.04 TEMP := $(shell mktemp -d) diff --git a/images/ceph/ceph_version b/images/ceph/ceph_version deleted file mode 100644 index 23d15c34b..000000000 --- a/images/ceph/ceph_version +++ /dev/null @@ -1 +0,0 @@ -ceph/daemon-base:v3.0.7-stable-3.0-luminous-centos-7 \ No newline at end of file diff --git a/pkg/apis/ceph.rook.io/v1beta1/types.go b/pkg/apis/ceph.rook.io/v1beta1/types.go index 88b84e48a..c7224680f 100644 --- a/pkg/apis/ceph.rook.io/v1beta1/types.go +++ b/pkg/apis/ceph.rook.io/v1beta1/types.go @@ -48,6 +48,9 @@ type ClusterList struct { } type ClusterSpec struct { + // The version information that instructs Rook to orchestrate a particular version of Ceph. + CephVersion CephVersionSpec `json:"cephVersion,omitempty"` + // A spec for available storage in the cluster and how it should be used Storage rook.StorageScopeSpec `json:"storage,omitempty"` @@ -73,6 +76,18 @@ type ClusterSpec struct { Dashboard DashboardSpec `json:"dashboard,omitempty"` } +// VersionSpec represents the settings for the Ceph version that Rook is orchestrating. +type CephVersionSpec struct { + // Image is the container image used to launch the ceph daemons, such as ceph/ceph:v12.2.7 or ceph/ceph:v13.2.1 + Image string `json:"image,omitempty"` + + // The name of the major release of Ceph: luminous, mimic, or nautilus + Name string `json:"name,omitempty"` + + // Whether to allow unsupported versions (do not set to true in production) + AllowUnsupported bool `json:"allowUnsupported,omitempty"` +} + // DashboardSpec represents the settings for the Ceph dashboard type DashboardSpec struct { // Whether to enable the dashboard diff --git a/pkg/apis/ceph.rook.io/v1beta1/version.go b/pkg/apis/ceph.rook.io/v1beta1/version.go new file mode 100644 index 000000000..9699272f8 --- /dev/null +++ b/pkg/apis/ceph.rook.io/v1beta1/version.go @@ -0,0 +1,38 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +    http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package v1beta1 + +const ( + Luminous = "luminous" + Mimic = "mimic" + Nautilus = "nautilus" + DefaultLuminousImage = "ceph/ceph:v12.2.9-20181026" +) + +func VersionAtLeast(version, minimumVersion string) bool { + orderedVersions := []string{Luminous, Mimic, Nautilus} + found := false + for _, v := range orderedVersions { + if v == minimumVersion { + found = true + } + if v == version { + // if we found the matching version, the version is at least the minimum version if found==true + return found + } + } + return false +} diff --git a/pkg/apis/ceph.rook.io/v1beta1/version_test.go b/pkg/apis/ceph.rook.io/v1beta1/version_test.go new file mode 100644 index 000000000..12ac562de --- /dev/null +++ b/pkg/apis/ceph.rook.io/v1beta1/version_test.go @@ -0,0 +1,39 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package v1beta1 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestVersionAtLeast(t *testing.T) { + // Valid combinations + assert.True(t, VersionAtLeast(Luminous, Luminous)) + assert.False(t, VersionAtLeast(Luminous, Mimic)) + assert.False(t, VersionAtLeast(Luminous, Nautilus)) + assert.True(t, VersionAtLeast(Mimic, Luminous)) + assert.True(t, VersionAtLeast(Mimic, Mimic)) + assert.False(t, VersionAtLeast(Mimic, Nautilus)) + assert.True(t, VersionAtLeast(Nautilus, Luminous)) + assert.True(t, VersionAtLeast(Nautilus, Mimic)) + assert.True(t, VersionAtLeast(Nautilus, Nautilus)) + + // Invalid combinations + assert.False(t, VersionAtLeast(Mimic, "foo")) + assert.False(t, VersionAtLeast("foo", Luminous)) +} diff --git a/pkg/apis/ceph.rook.io/v1beta1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1beta1/zz_generated.deepcopy.go index d1c0f4889..ddb38375f 100644 --- a/pkg/apis/ceph.rook.io/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1beta1/zz_generated.deepcopy.go @@ -25,6 +25,22 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CephVersionSpec) DeepCopyInto(out *CephVersionSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CephVersionSpec. +func (in *CephVersionSpec) DeepCopy() *CephVersionSpec { + if in == nil { + return nil + } + out := new(CephVersionSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Cluster) DeepCopyInto(out *Cluster) { *out = *in @@ -89,6 +105,7 @@ func (in *ClusterList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = *in + out.CephVersion = in.CephVersion in.Storage.DeepCopyInto(&out.Storage) if in.Placement != nil { in, out := &in.Placement, &out.Placement diff --git a/pkg/daemon/ceph/client/command.go b/pkg/daemon/ceph/client/command.go index a7262a759..20c95074c 100644 --- a/pkg/daemon/ceph/client/command.go +++ b/pkg/daemon/ceph/client/command.go @@ -59,6 +59,10 @@ func FinalizeCephCommandArgs(command string, args []string, configDir, clusterNa return command, append(args, configArgs...) } +func ExecuteCephCommandDebugLog(context *clusterd.Context, clusterName string, args []string) ([]byte, error) { + return executeCephCommandWithOutputFile(context, clusterName, true, args) +} + func ExecuteCephCommand(context *clusterd.Context, clusterName string, args []string) ([]byte, error) { return executeCephCommandWithOutputFile(context, clusterName, false, args) } diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 94a009941..b8094d419 100644 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -21,6 +21,8 @@ import ( "fmt" "reflect" "sort" + "strings" + "time" cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" rookv1alpha2 "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" @@ -30,11 +32,23 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/cluster/osd" "github.com/rook/rook/pkg/operator/k8sutil" + batch "k8s.io/api/batch/v1" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + detectVersionName = "rook-ceph-detect-version" +) + +var ( + // supportedVersions are production-ready versions that rook supports + supportedVersions = []string{cephv1beta1.Luminous, cephv1beta1.Mimic} + // allVersions includes all supportedVersions as well as unreleased versions that are being tested with rook + allVersions = append(supportedVersions, cephv1beta1.Nautilus) +) + type cluster struct { context *clusterd.Context Namespace string @@ -52,6 +66,79 @@ func newCluster(c *cephv1beta1.Cluster, context *clusterd.Context) *cluster { ownerRef: ClusterOwnerRef(c.Namespace, string(c.UID))} } +func (c *cluster) setCephMajorVersion(timeout time.Duration) error { + version, err := c.detectCephMajorVersion(timeout) + if err != nil { + // Don't return the err yet here so we can override the failure with a setting in the crd below. + logger.Errorf("failed to detect ceph version. %+v", err) + } else { + logger.Infof("detected ceph version %s for image %s", version, c.Spec.CephVersion.Image) + } + + if c.Spec.CephVersion.Name != "" { + // if the cephVersion.name is already set, override the detected version + logger.Warningf("overriding ceph version to %s specified in the CRD", c.Spec.CephVersion.Name) + return nil + } + + c.Spec.CephVersion.Name = version + return err +} + +func (c *cluster) detectCephMajorVersion(timeout time.Duration) (string, error) { + // get the major ceph version by running "ceph --version" in the ceph image + job := &batch.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: detectVersionName, + Namespace: c.Namespace, + }, + Spec: batch.JobSpec{ + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "job": detectVersionName, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Command: []string{"ceph"}, + Args: []string{"--version"}, + Name: "version", + Image: c.Spec.CephVersion.Image, + }, + }, + RestartPolicy: v1.RestartPolicyOnFailure, + }, + }, + }, + } + k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &job.ObjectMeta, &c.ownerRef) + + // run the job to detect the version + if err := k8sutil.RunReplaceableJob(c.context.Clientset, job); err != nil { + return "", fmt.Errorf("failed to start version job. %+v", err) + } + + if err := k8sutil.WaitForJobCompletion(c.context.Clientset, job, timeout); err != nil { + return "", fmt.Errorf("failed to complete version job. %+v", err) + } + + log, err := k8sutil.GetPodLog(c.context.Clientset, c.Namespace, "job="+detectVersionName) + if err != nil { + return "", fmt.Errorf("failed to get version job log to detect version. %+v", err) + } + + version, err := extractCephVersion(log) + if err != nil { + return "", fmt.Errorf("failed to extract ceph version. %+v", err) + } + + // delete the job since we're done with it + k8sutil.DeleteBatchJob(c.context.Clientset, c.Namespace, job.Name, false) + return version, nil +} + func (c *cluster) createInstance(rookImage string) error { // Create a configmap for overriding ceph config settings @@ -73,7 +160,7 @@ func (c *cluster) createInstance(rookImage string) error { } // Start the mon pods - c.mons = mon.New(c.context, c.Namespace, c.Spec.DataDirHostPath, rookImage, c.Spec.Mon, cephv1beta1.GetMonPlacement(c.Spec.Placement), + c.mons = mon.New(c.context, c.Namespace, c.Spec.DataDirHostPath, rookImage, c.Spec.CephVersion, c.Spec.Mon, cephv1beta1.GetMonPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork, cephv1beta1.GetMonResources(c.Spec.Resources), c.ownerRef) err = c.mons.Start() if err != nil { @@ -85,7 +172,7 @@ func (c *cluster) createInstance(rookImage string) error { return fmt.Errorf("failed to create initial crushmap: %+v", err) } - c.mgrs = mgr.New(c.context, c.Namespace, rookImage, cephv1beta1.GetMgrPlacement(c.Spec.Placement), + c.mgrs = mgr.New(c.context, c.Namespace, rookImage, c.Spec.CephVersion, cephv1beta1.GetMgrPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork, c.Spec.Dashboard, cephv1beta1.GetMgrResources(c.Spec.Resources), c.ownerRef) err = c.mgrs.Start() if err != nil { @@ -93,7 +180,7 @@ func (c *cluster) createInstance(rookImage string) error { } // Start the OSDs - c.osds = osd.New(c.context, c.Namespace, rookImage, c.Spec.ServiceAccount, c.Spec.Storage, c.Spec.DataDirHostPath, + c.osds = osd.New(c.context, c.Namespace, rookImage, c.Spec.CephVersion, c.Spec.ServiceAccount, c.Spec.Storage, c.Spec.DataDirHostPath, cephv1beta1.GetOSDPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork, cephv1beta1.GetOSDResources(c.Spec.Resources), c.ownerRef) err = c.osds.Start() if err != nil { @@ -195,5 +282,28 @@ func clusterChanged(oldCluster, newCluster cephv1beta1.ClusterSpec, clusterRef * clusterRef.mons.MonCountMutex.Unlock() } + if oldCluster.CephVersion.AllowUnsupported != newCluster.CephVersion.AllowUnsupported { + logger.Infof("ceph version allowUnsupported has changed from %t to %t", oldCluster.CephVersion.AllowUnsupported, newCluster.CephVersion.AllowUnsupported) + changeFound = true + } + return changeFound } + +func extractCephVersion(version string) (string, error) { + for _, v := range allVersions { + if strings.Contains(version, v) { + return v, nil + } + } + return "", fmt.Errorf("failed to parse version from: %s", version) +} + +func versionSupported(version string) bool { + for _, v := range supportedVersions { + if v == version { + return true + } + } + return false +} diff --git a/pkg/operator/ceph/cluster/cluster_test.go b/pkg/operator/ceph/cluster/cluster_test.go new file mode 100644 index 000000000..1dbe6f480 --- /dev/null +++ b/pkg/operator/ceph/cluster/cluster_test.go @@ -0,0 +1,48 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "testing" + "time" + + cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" + "github.com/rook/rook/pkg/clusterd" + testop "github.com/rook/rook/pkg/operator/test" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" +) + +func TestSetCephVersion(t *testing.T) { + clientset := testop.New(3) + executor := &exectest.MockExecutor{} + context := &clusterd.Context{Clientset: clientset, Executor: executor} + cluster := &cephv1beta1.Cluster{} + c := newCluster(cluster, context) + c.Namespace = "myns" + + // expect a failure with no default and no version from the batch job + err := c.setCephMajorVersion(time.Millisecond) + assert.NotNil(t, err) + assert.Equal(t, "", cluster.Spec.CephVersion.Name) + + // expect success when the default is set from the crd + cluster.Spec.CephVersion.Name = "mimic" + err = c.setCephMajorVersion(time.Millisecond) + assert.Nil(t, err) + assert.Equal(t, "mimic", cluster.Spec.CephVersion.Name) +} diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 92ce6c80c..a4f1c4b43 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -182,6 +182,19 @@ func (c *ClusterController) onAdd(obj interface{}) { logger.Warningf("mon count is even (given: %d), should be uneven, continuing", cluster.Spec.Mon.Count) } + err = cluster.setCephMajorVersion(15 * time.Minute) + if err != nil { + logger.Errorf("unknown ceph major version. %+v", err) + return + } + + if !cluster.Spec.CephVersion.AllowUnsupported { + if !versionSupported(cluster.Spec.CephVersion.Name) { + logger.Errorf("unsupported ceph version detected: %s. allowUnupported must be set to true to run with this version.", cluster.Spec.CephVersion.Name) + return + } + } + // Start the Rook cluster components. Retry several times in case of failure. err = wait.Poll(clusterCreateInterval, clusterCreateTimeout, func() (bool, error) { if err := c.updateClusterStatus(clusterObj.Namespace, clusterObj.Name, cephv1beta1.ClusterStateCreating, ""); err != nil { @@ -217,11 +230,11 @@ func (c *ClusterController) onAdd(obj interface{}) { poolController.StartWatch(cluster.Namespace, cluster.stopCh) // Start object store CRD watcher - objectStoreController := object.NewObjectStoreController(c.context, c.rookImage, cluster.Spec.Network.HostNetwork, cluster.ownerRef) + objectStoreController := object.NewObjectStoreController(c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef) objectStoreController.StartWatch(cluster.Namespace, cluster.stopCh) // Start file system CRD watcher - fileController := file.NewFilesystemController(c.context, c.rookImage, cluster.Spec.Network.HostNetwork, cluster.ownerRef) + fileController := file.NewFilesystemController(c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef) fileController.StartWatch(cluster.Namespace, cluster.stopCh) // Start mon health checker diff --git a/pkg/operator/ceph/cluster/mgr/dashboard.go b/pkg/operator/ceph/cluster/mgr/dashboard.go new file mode 100644 index 000000000..3d3572e6a --- /dev/null +++ b/pkg/operator/ceph/cluster/mgr/dashboard.go @@ -0,0 +1,215 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package mgr for the Ceph manager. +package mgr + +import ( + "fmt" + "math/rand" + "os/exec" + "syscall" + "time" + + cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" + "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/k8sutil" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + dashboardModuleName = "dashboard" + dashboardPortHttps = 8443 + dashboardPortHttp = 7000 + dashboardUsername = "admin" + dashboardPasswordName = "rook-ceph-dashboard-password" + passwordLength = 10 + passwordKeyName = "password" + certAlreadyConfiguredErrorCode = 5 +) + +var ( + dashboardInitWaitTime = 5 * time.Second +) + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +func (c *Cluster) configureDashboard(port int) error { + // enable or disable the dashboard module + if err := c.configureDashboardModule(); err != nil { + return fmt.Errorf("failed to enable mgr dashboard module. %+v", err) + } + + dashboardService := c.makeDashboardService(appName, port) + if c.dashboard.Enabled { + // expose the dashboard service + if _, err := c.context.Clientset.CoreV1().Services(c.Namespace).Create(dashboardService); err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create dashboard mgr service. %+v", err) + } + logger.Infof("dashboard service already exists") + } else { + logger.Infof("dashboard service started") + } + } else { + // delete the dashboard service if it exists + err := c.context.Clientset.CoreV1().Services(c.Namespace).Delete(dashboardService.Name, &metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete dashboard service. %+v", err) + } + } + + return nil +} + +// Ceph docs about the dashboard module: http://docs.ceph.com/docs/luminous/mgr/dashboard/ +func (c *Cluster) configureDashboardModule() error { + if c.dashboard.Enabled { + if err := client.MgrEnableModule(c.context, c.Namespace, dashboardModuleName, true); err != nil { + return fmt.Errorf("failed to enable mgr dashboard module. %+v", err) + } + + if err := c.initializeDashboard(); err != nil { + return fmt.Errorf("failed to initialize dashboard. %+v", err) + } + } else { + if err := client.MgrDisableModule(c.context, c.Namespace, dashboardModuleName); err != nil { + logger.Errorf("failed to disable mgr dashboard module. %+v", err) + } + } + return nil +} + +func (c *Cluster) initializeDashboard() error { + if c.cephVersion.Name == cephv1beta1.Luminous || c.cephVersion.Name == "" { + logger.Infof("skipping cert and user configuration on luminous") + return nil + } + + // we need to wait a short period after enabling the module before we can call the `ceph dashboard` commands. + time.Sleep(dashboardInitWaitTime) + + password, err := c.getOrGenerateDashboardPassword() + if err != nil { + return fmt.Errorf("failed to generate a password. %+v", err) + } + + alreadyCreated, err := c.createSelfSignedCert() + if err != nil { + return fmt.Errorf("failed to create a self signed cert. %+v", err) + } + if alreadyCreated { + return nil + } + + if err := c.setLoginCredentials(password); err != nil { + return fmt.Errorf("failed to set login creds. %+v", err) + } + + return c.restartDashboard() +} + +func (c *Cluster) createSelfSignedCert() (bool, error) { + // create a self-signed cert for the https connections required in mimic + args := []string{"dashboard", "create-self-signed-cert"} + _, err := client.ExecuteCephCommand(c.context, c.Namespace, args) + if err != nil { + if exiterr, ok := err.(*exec.ExitError); ok { + if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { + exitCode := status.ExitStatus() + if exitCode == certAlreadyConfiguredErrorCode { + logger.Infof("dashboard is already initialized with a cert") + return true, nil + } + } + } + return false, fmt.Errorf("failed to create self signed cert on mgr. %+v", err) + } + return false, nil +} + +func (c *Cluster) setLoginCredentials(password string) error { + // Set the login credentials. Write the command/args to the debug log so we don't write the password by default to the log. + logger.Infof("Running command: ceph dashboard set-login-credentials admin *******") + args := []string{"dashboard", "set-login-credentials", dashboardUsername, password} + _, err := client.ExecuteCephCommandDebugLog(c.context, c.Namespace, args) + if err != nil { + return fmt.Errorf("failed to set login creds on mgr. %+v", err) + } + return nil +} + +func (c *Cluster) getOrGenerateDashboardPassword() (string, error) { + secret, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(dashboardPasswordName, metav1.GetOptions{}) + if err == nil { + logger.Infof("the dashboard secret was already generated") + return decodeSecret(secret) + } + if !errors.IsNotFound(err) { + return "", fmt.Errorf("failed to get dashboard secret. %+v", err) + } + + // Generate a password + password := generatePassword(passwordLength) + + // Store the keyring in a secret + secrets := map[string][]byte{ + passwordKeyName: []byte(password), + } + secret = &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: dashboardPasswordName, + Namespace: c.Namespace, + }, + Data: secrets, + Type: k8sutil.RookType, + } + k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &secret.ObjectMeta, &c.ownerRef) + + _, err = c.context.Clientset.CoreV1().Secrets(c.Namespace).Create(secret) + if err != nil { + return "", fmt.Errorf("failed to save dashboard secret. %+v", err) + } + return password, nil +} + +func generatePassword(length int) string { + const passwordChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + passwd := make([]byte, length) + for i := range passwd { + passwd[i] = passwordChars[rand.Intn(len(passwordChars))] + } + return string(passwd) +} + +func decodeSecret(secret *v1.Secret) (string, error) { + password, ok := secret.Data[passwordKeyName] + if !ok { + return "", fmt.Errorf("password not found in secret") + } + return string(password), nil +} + +func (c *Cluster) restartDashboard() error { + logger.Infof("restarting the mgr module") + client.MgrDisableModule(c.context, c.Namespace, dashboardModuleName) + client.MgrEnableModule(c.context, c.Namespace, dashboardModuleName, true) + return nil +} diff --git a/pkg/operator/ceph/cluster/mgr/dashboard_test.go b/pkg/operator/ceph/cluster/mgr/dashboard_test.go new file mode 100644 index 000000000..d0418e160 --- /dev/null +++ b/pkg/operator/ceph/cluster/mgr/dashboard_test.go @@ -0,0 +1,106 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package mgr + +import ( + "testing" + + cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/test" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestGeneratePassword(t *testing.T) { + password := generatePassword(0) + assert.Equal(t, "", password) + + password = generatePassword(1) + assert.Equal(t, 1, len(password)) + logger.Infof("password: %s", password) + + password = generatePassword(10) + assert.Equal(t, 10, len(password)) + logger.Infof("password: %s", password) +} + +func TestGetOrGeneratePassword(t *testing.T) { + c := &Cluster{context: &clusterd.Context{Clientset: test.New(3)}, Namespace: "myns"} + _, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(dashboardPasswordName, metav1.GetOptions{}) + assert.True(t, errors.IsNotFound(err)) + + // Generate a password + password, err := c.getOrGenerateDashboardPassword() + require.Nil(t, err) + assert.Equal(t, passwordLength, len(password)) + + secret, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(dashboardPasswordName, metav1.GetOptions{}) + assert.Nil(t, err) + assert.Equal(t, 1, len(secret.Data)) + passwordFromSecret, err := decodeSecret(secret) + assert.Equal(t, password, passwordFromSecret) + + // We should retrieve the same password on the second call + retrievedPassword, err := c.getOrGenerateDashboardPassword() + assert.Nil(t, err) + assert.Equal(t, password, retrievedPassword) +} + +func TestStartSecureDashboard(t *testing.T) { + enables := 0 + disables := 0 + executor := &exectest.MockExecutor{ + MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) { + logger.Infof("command: %s %v", command, args) + if args[1] == "module" { + if args[2] == "enable" { + enables++ + } else if args[2] == "disable" { + disables++ + } + } + return "", nil + }, + } + c := &Cluster{context: &clusterd.Context{Clientset: test.New(3), Executor: executor}, Namespace: "myns", + dashboard: cephv1beta1.DashboardSpec{Enabled: true}, cephVersion: cephv1beta1.CephVersionSpec{Name: cephv1beta1.Mimic, Image: "ceph/ceph:v13.2.2"}} + dashboardInitWaitTime = 0 + err := c.configureDashboard(dashboardPortHttp) + assert.Nil(t, err) + // the dashboard is enabled, then disabled and enabled again to restart it with the cert + assert.Equal(t, 2, enables) + assert.Equal(t, 1, disables) + + svc, err := c.context.Clientset.CoreV1().Services(c.Namespace).Get("rook-ceph-mgr-dashboard", metav1.GetOptions{}) + assert.Nil(t, err) + assert.NotNil(t, svc) + + // disable the dashboard + c.dashboard.Enabled = false + err = c.configureDashboard(dashboardPortHttp) + assert.Nil(t, err) + assert.Equal(t, 2, enables) + assert.Equal(t, 2, disables) + + svc, err = c.context.Clientset.CoreV1().Services(c.Namespace).Get("rook-ceph-mgr-dashboard", metav1.GetOptions{}) + assert.NotNil(t, err) + assert.True(t, errors.IsNotFound(err)) + assert.Nil(t, svc) +} diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go index e55604bdf..b4dafe384 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr.go +++ b/pkg/operator/ceph/cluster/mgr/mgr.go @@ -25,7 +25,6 @@ import ( rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" - opspec "github.com/rook/rook/pkg/operator/ceph/spec" "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -38,9 +37,7 @@ const ( appName = "rook-ceph-mgr" keyringSecretKeyName = "keyring" prometheusModuleName = "prometheus" - dashboardModuleName = "dashboard" metricsPort = 9283 - dashboardPort = 7000 ) var mgrNames = []string{"a", "b"} @@ -48,7 +45,6 @@ var mgrNames = []string{"a", "b"} // Cluster represents the Rook and environment configuration settings needed to set up Ceph mgrs. type Cluster struct { Namespace string - Version string Replicas int placement rookalpha.Placement context *clusterd.Context @@ -57,6 +53,8 @@ type Cluster struct { resources v1.ResourceRequirements ownerRef metav1.OwnerReference dashboard cephv1beta1.DashboardSpec + cephVersion cephv1beta1.CephVersionSpec + rookVersion string } // mgrConfig for a single mgr @@ -66,13 +64,14 @@ type mgrConfig struct { } // New creates an instance of the mgr -func New(context *clusterd.Context, namespace, version string, placement rookalpha.Placement, hostNetwork bool, dashboard cephv1beta1.DashboardSpec, +func New(context *clusterd.Context, namespace, rookVersion string, cephVersion cephv1beta1.CephVersionSpec, placement rookalpha.Placement, hostNetwork bool, dashboard cephv1beta1.DashboardSpec, resources v1.ResourceRequirements, ownerRef metav1.OwnerReference) *Cluster { return &Cluster{ context: context, Namespace: namespace, placement: placement, - Version: version, + rookVersion: rookVersion, + cephVersion: cephVersion, Replicas: 1, dataDir: k8sutil.DataDir, dashboard: dashboard, @@ -86,6 +85,11 @@ func New(context *clusterd.Context, namespace, version string, placement rookalp func (c *Cluster) Start() error { logger.Infof("start running mgr") + dashboardPort := dashboardPortHttps + if c.cephVersion.Name == cephv1beta1.Luminous { + dashboardPort = dashboardPortHttp + } + for i := 0; i < c.Replicas; i++ { if i >= len(mgrNames) { logger.Errorf("cannot have more than %d mgrs", len(mgrNames)) @@ -104,7 +108,7 @@ func (c *Cluster) Start() error { } // start the deployment - deployment := c.makeDeployment(mgrConfig) + deployment := c.makeDeployment(mgrConfig, dashboardPort) if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Create(deployment); err != nil { if !errors.IsAlreadyExists(err) { return fmt.Errorf("failed to create %s deployment. %+v", resourceName, err) @@ -115,8 +119,16 @@ func (c *Cluster) Start() error { } } + if err := c.configureOrchestratorModules(); err != nil { + logger.Errorf("failed to enable orchestrator modules. %+v", err) + } + if err := c.enablePrometheusModule(c.Namespace); err != nil { - return fmt.Errorf("failed to enable mgr prometheus module. %+v", err) + logger.Errorf("failed to enable mgr prometheus module. %+v", err) + } + + if err := c.configureDashboard(dashboardPort); err != nil { + logger.Errorf("failed to enable mgr dashboard. %+v", err) } // create the metrics service @@ -130,86 +142,9 @@ func (c *Cluster) Start() error { logger.Infof("mgr metrics service started") } - return c.configureDashboard() -} - -func (c *Cluster) configureDashboard() error { - // enable or disable the dashboard module - if err := c.configureDashboardModule(c.Namespace, c.dashboard.Enabled); err != nil { - return fmt.Errorf("failed to enable mgr dashboard module. %+v", err) - } - - dashboardService := c.makeDashboardService(appName) - if c.dashboard.Enabled { - // expose the dashboard service - if _, err := c.context.Clientset.CoreV1().Services(c.Namespace).Create(dashboardService); err != nil { - if !errors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create dashboard mgr service. %+v", err) - } - logger.Infof("dashboard service already exists") - } else { - logger.Infof("dashboard service started") - } - } else { - // delete the dashboard service if it exists - err := c.context.Clientset.CoreV1().Services(c.Namespace).Delete(dashboardService.Name, &metav1.DeleteOptions{}) - if err != nil && !errors.IsNotFound(err) { - return fmt.Errorf("failed to delete dashboard service. %+v", err) - } - } - return nil } -func (c *Cluster) makeMetricsService(name string) *v1.Service { - labels := opspec.AppLabels(appName, c.Namespace) - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: c.Namespace, - Labels: labels, - }, - Spec: v1.ServiceSpec{ - Selector: labels, - Type: v1.ServiceTypeClusterIP, - Ports: []v1.ServicePort{ - { - Name: "http-metrics", - Port: int32(metricsPort), - Protocol: v1.ProtocolTCP, - }, - }, - }, - } - - k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &svc.ObjectMeta, &c.ownerRef) - return svc -} - -func (c *Cluster) makeDashboardService(name string) *v1.Service { - labels := opspec.AppLabels(appName, c.Namespace) - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-dashboard", name), - Namespace: c.Namespace, - Labels: labels, - }, - Spec: v1.ServiceSpec{ - Selector: labels, - Type: v1.ServiceTypeClusterIP, - Ports: []v1.ServicePort{ - { - Name: "http-dashboard", - Port: int32(dashboardPort), - Protocol: v1.ProtocolTCP, - }, - }, - }, - } - k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &svc.ObjectMeta, &c.ownerRef) - return svc -} - func (c *Cluster) createKeyring(clusterName, name, daemonName string) error { _, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(name, metav1.GetOptions{}) if err == nil { @@ -256,23 +191,9 @@ func (c *Cluster) enablePrometheusModule(clusterName string) error { return nil } -// Ceph docs about the dashboard module: http://docs.ceph.com/docs/luminous/mgr/dashboard/ -func (c *Cluster) configureDashboardModule(clusterName string, enable bool) error { - if enable { - if err := client.MgrEnableModule(c.context, clusterName, dashboardModuleName, true); err != nil { - return fmt.Errorf("failed to enable mgr dashboard module. %+v", err) - } - } else { - if err := client.MgrDisableModule(c.context, clusterName, dashboardModuleName); err != nil { - return fmt.Errorf("failed to disable mgr dashboard module. %+v", err) - } - } - return nil -} - func getKeyringProperties(name string) (string, []string) { username := fmt.Sprintf("mgr.%s", name) - access := []string{"mon", "allow *"} + access := []string{"mon", "allow profile mgr", "mds", "allow *", "osd", "allow *"} return username, access } diff --git a/pkg/operator/ceph/cluster/mgr/mgr_test.go b/pkg/operator/ceph/cluster/mgr/mgr_test.go index 8b5227ca4..109042d41 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr_test.go +++ b/pkg/operator/ceph/cluster/mgr/mgr_test.go @@ -45,7 +45,7 @@ func TestStartMGR(t *testing.T) { Executor: executor, ConfigDir: configDir, Clientset: testop.New(3)} - c := New(context, "ns", "myversion", rookalpha.Placement{}, false, cephv1beta1.DashboardSpec{Enabled: true}, v1.ResourceRequirements{}, metav1.OwnerReference{}) + c := New(context, "ns", "myversion", cephv1beta1.CephVersionSpec{}, rookalpha.Placement{}, false, cephv1beta1.DashboardSpec{Enabled: true}, v1.ResourceRequirements{}, metav1.OwnerReference{}) defer os.RemoveAll(c.dataDir) // start a basic service diff --git a/pkg/operator/ceph/cluster/mgr/orchestrator.go b/pkg/operator/ceph/cluster/mgr/orchestrator.go new file mode 100644 index 000000000..a2c5a668e --- /dev/null +++ b/pkg/operator/ceph/cluster/mgr/orchestrator.go @@ -0,0 +1,50 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package mgr for the Ceph manager. +package mgr + +import ( + "fmt" + + cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" + "github.com/rook/rook/pkg/daemon/ceph/client" +) + +const ( + orchestratorModuleName = "orchestrator_cli" + rookModuleName = "rook" +) + +// Ceph docs about the orchestrator modules: http://docs.ceph.com/docs/master/mgr/orchestrator_cli/ +func (c *Cluster) configureOrchestratorModules() error { + if !cephv1beta1.VersionAtLeast(c.cephVersion.Name, cephv1beta1.Nautilus) { + logger.Infof("skipping enabling orchestrator modules on releases older than nautilus") + return nil + } + + if err := client.MgrEnableModule(c.context, c.Namespace, orchestratorModuleName, true); err != nil { + return fmt.Errorf("failed to enable mgr orchestrator module. %+v", err) + } + if err := client.MgrEnableModule(c.context, c.Namespace, rookModuleName, true); err != nil { + return fmt.Errorf("failed to enable mgr rook module. %+v", err) + } + if _, err := client.ExecuteCephCommand(c.context, c.Namespace, []string{"orchestrator", "set", "backend", "rook"}); err != nil { + return fmt.Errorf("failed to set rook as the orchestrator backend. %+v", err) + } + + return nil +} diff --git a/pkg/operator/ceph/cluster/mgr/orchestrator_test.go b/pkg/operator/ceph/cluster/mgr/orchestrator_test.go new file mode 100644 index 000000000..fb39467fb --- /dev/null +++ b/pkg/operator/ceph/cluster/mgr/orchestrator_test.go @@ -0,0 +1,78 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package mgr + +import ( + "fmt" + "testing" + + cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" + "github.com/rook/rook/pkg/clusterd" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" +) + +func TestOrchestratorModules(t *testing.T) { + executor := &exectest.MockExecutor{} + context := &clusterd.Context{Executor: executor} + orchestratorModuleEnabled := false + rookModuleEnabled := false + rookBackendSet := false + executor.MockExecuteCommandWithOutputFile = func(debug bool, actionName, command, outputFile string, args ...string) (string, error) { + logger.Infof("Command: %s %v", command, args) + if args[0] == "mgr" && args[1] == "module" && args[2] == "enable" { + if args[3] == "orchestrator_cli" { + orchestratorModuleEnabled = true + return "", nil + } + if args[3] == "rook" { + rookModuleEnabled = true + return "", nil + } + } + if args[0] == "orchestrator" && args[1] == "set" && args[2] == "backend" && args[3] == "rook" { + rookBackendSet = true + return "", nil + } + return "", fmt.Errorf("unexpected ceph command '%v'", args) + } + + c := &Cluster{context: context} + + // the modules are skipped on luminous + c.cephVersion.Name = cephv1beta1.Luminous + err := c.configureOrchestratorModules() + assert.Nil(t, err) + assert.False(t, orchestratorModuleEnabled) + assert.False(t, rookModuleEnabled) + assert.False(t, rookBackendSet) + + // the modules are skipped on mimic + c.cephVersion.Name = cephv1beta1.Mimic + err = c.configureOrchestratorModules() + assert.Nil(t, err) + assert.False(t, orchestratorModuleEnabled) + assert.False(t, rookModuleEnabled) + assert.False(t, rookBackendSet) + + // the modules are configured on nautilus + c.cephVersion.Name = cephv1beta1.Nautilus + err = c.configureOrchestratorModules() + assert.Nil(t, err) + assert.True(t, orchestratorModuleEnabled) + assert.True(t, rookModuleEnabled) + assert.True(t, rookBackendSet) +} diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index a1a2578b9..e9cacfa4a 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -33,7 +33,7 @@ const ( mgrDaemonCommand = "ceph-mgr" ) -func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) *extensions.Deployment { +func (c *Cluster) makeDeployment(mgrConfig *mgrConfig, port int) *extensions.Deployment { podSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: mgrConfig.ResourceName, @@ -47,7 +47,7 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) *extensions.Deployment { c.makeConfigInitContainer(mgrConfig), }, Containers: []v1.Container{ - c.makeMgrDaemonContainer(mgrConfig), + c.makeMgrDaemonContainer(mgrConfig, port), }, RestartPolicy: v1.RestartPolicyAlways, Volumes: opspec.PodVolumes(""), @@ -80,7 +80,7 @@ func (c *Cluster) makeConfigInitContainer(mgrConfig *mgrConfig) v1.Container { fmt.Sprintf("--config-dir=%s", k8sutil.DataDir), fmt.Sprintf("--mgr-name=%s", mgrConfig.DaemonName), }, - Image: k8sutil.MakeRookImage(c.Version), + Image: k8sutil.MakeRookImage(c.rookVersion), Env: []v1.EnvVar{ // Set '--mgr-keyring' flag with an env var sourced from the secret {Name: "ROOK_MGR_KEYRING", @@ -102,7 +102,7 @@ func (c *Cluster) makeConfigInitContainer(mgrConfig *mgrConfig) v1.Container { } } -func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { +func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig, port int) v1.Container { container := v1.Container{ Name: "mgr", Command: []string{ @@ -113,7 +113,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { "--id", mgrConfig.DaemonName, // do not add the '--cluster/--conf/--keyring' flags; rook wants their default values }, - Image: k8sutil.MakeRookImage(c.Version), + Image: c.cephVersion.Image, VolumeMounts: opspec.CephVolumeMounts(), Ports: []v1.ContainerPort{ { @@ -128,7 +128,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { }, { Name: "dashboard", - ContainerPort: int32(dashboardPort), + ContainerPort: int32(port), Protocol: v1.ProtocolTCP, }, }, @@ -139,6 +139,55 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { return container } +func (c *Cluster) makeMetricsService(name string) *v1.Service { + labels := opspec.AppLabels(appName, c.Namespace) + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: c.Namespace, + Labels: labels, + }, + Spec: v1.ServiceSpec{ + Selector: labels, + Type: v1.ServiceTypeClusterIP, + Ports: []v1.ServicePort{ + { + Name: "http-metrics", + Port: int32(metricsPort), + Protocol: v1.ProtocolTCP, + }, + }, + }, + } + + k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &svc.ObjectMeta, &c.ownerRef) + return svc +} + +func (c *Cluster) makeDashboardService(name string, port int) *v1.Service { + labels := opspec.AppLabels(appName, c.Namespace) + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-dashboard", name), + Namespace: c.Namespace, + Labels: labels, + }, + Spec: v1.ServiceSpec{ + Selector: labels, + Type: v1.ServiceTypeClusterIP, + Ports: []v1.ServicePort{ + { + Name: "https-dashboard", + Port: int32(port), + Protocol: v1.ProtocolTCP, + }, + }, + }, + } + k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &svc.ObjectMeta, &c.ownerRef) + return svc +} + func (c *Cluster) getPodLabels(daemonName string) map[string]string { labels := opspec.PodLabels(appName, c.Namespace, "mgr", daemonName) // leave "instance" key for legacy usage diff --git a/pkg/operator/ceph/cluster/mgr/spec_test.go b/pkg/operator/ceph/cluster/mgr/spec_test.go index 689f40b1c..e3ae386e9 100644 --- a/pkg/operator/ceph/cluster/mgr/spec_test.go +++ b/pkg/operator/ceph/cluster/mgr/spec_test.go @@ -41,6 +41,7 @@ func TestPodSpec(t *testing.T) { &clusterd.Context{Clientset: testop.New(1)}, "ns", "rook/rook:myversion", + cephv1beta1.CephVersionSpec{Image: "ceph/ceph:myceph"}, rookalpha.Placement{}, false, cephv1beta1.DashboardSpec{}, @@ -60,7 +61,7 @@ func TestPodSpec(t *testing.T) { ResourceName: "mgr-a", } - d := c.makeDeployment(&mgrTestConfig) + d := c.makeDeployment(&mgrTestConfig, dashboardPortHttp) assert.NotNil(t, d) assert.Equal(t, "mgr-a", d.Name) @@ -106,8 +107,7 @@ func TestPodSpec(t *testing.T) { configContainerDefinition.TestContainer(t, "config init", cont, logger) assert.Equal(t, "100", cont.Resources.Limits.Cpu().String()) assert.Equal(t, "1337", cont.Resources.Requests.Memory().String()) - - daemonImage := "rook/rook:myversion" + daemonImage := "ceph/ceph:myceph" // +1 for $ROOK_CLUSTER_NAME daemonEnvs := len(k8sutil.ClusterDaemonEnvVars()) + 1 daemonContainerDefinition := cephtest.ContainerTestDefinition{ @@ -126,7 +126,7 @@ func TestPodSpec(t *testing.T) { Protocol: v1.ProtocolTCP}, {ContainerPort: int32(metricsPort), Protocol: v1.ProtocolTCP}, - {ContainerPort: int32(dashboardPort), + {ContainerPort: int32(dashboardPortHttp), Protocol: v1.ProtocolTCP}}, IsPrivileged: nil, // not set in spec } @@ -146,7 +146,7 @@ func TestPodSpec(t *testing.T) { } func TestServiceSpec(t *testing.T) { - c := New(&clusterd.Context{}, "ns", "myversion", rookalpha.Placement{}, false, cephv1beta1.DashboardSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{}) + c := New(&clusterd.Context{}, "ns", "myversion", cephv1beta1.CephVersionSpec{}, rookalpha.Placement{}, false, cephv1beta1.DashboardSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{}) s := c.makeMetricsService("rook-mgr") assert.NotNil(t, s) @@ -159,6 +159,7 @@ func TestHostNetwork(t *testing.T) { &clusterd.Context{Clientset: testop.New(1)}, "ns", "myversion", + cephv1beta1.CephVersionSpec{}, rookalpha.Placement{}, true, cephv1beta1.DashboardSpec{}, @@ -171,7 +172,7 @@ func TestHostNetwork(t *testing.T) { ResourceName: "mgr-a", } - d := c.makeDeployment(&mgrTestConfig) + d := c.makeDeployment(&mgrTestConfig, dashboardPortHttp) assert.NotNil(t, d) assert.Equal(t, true, d.Spec.Template.Spec.HostNetwork) diff --git a/pkg/operator/ceph/cluster/migration.go b/pkg/operator/ceph/cluster/migration.go index 6089e7e47..dead0b704 100644 --- a/pkg/operator/ceph/cluster/migration.go +++ b/pkg/operator/ceph/cluster/migration.go @@ -48,7 +48,9 @@ func getClusterObject(obj interface{}) (cluster *cephv1beta1.Cluster, migrationN cluster, ok = obj.(*cephv1beta1.Cluster) if ok { // the cluster object is of the latest type, simply return it - return cluster.DeepCopy(), false, nil + cluster = cluster.DeepCopy() + setClusterDefaults(cluster) + return cluster, false, nil } // type assertion to current cluster type failed, try instead asserting to the legacy cluster types @@ -61,6 +63,16 @@ func getClusterObject(obj interface{}) (cluster *cephv1beta1.Cluster, migrationN return nil, false, fmt.Errorf("not a known cluster object: %+v", obj) } +func setClusterDefaults(cluster *cephv1beta1.Cluster) { + // The ceph version image should be set in the CRD. + // If/when the v1beta1 CRD is converted to v1, we could set this permanently during the conversion instead of + // setting this default in memory every time we run the operator. + if cluster.Spec.CephVersion.Image == "" { + logger.Infof("setting default luminous image: %s", cephv1beta1.DefaultLuminousImage) + cluster.Spec.CephVersion.Image = cephv1beta1.DefaultLuminousImage + } +} + func (c *ClusterController) migrateClusterObject(clusterToMigrate *cephv1beta1.Cluster, legacyObj interface{}) error { logger.Infof("migrating legacy cluster %s in namespace %s", clusterToMigrate.Name, clusterToMigrate.Namespace) diff --git a/pkg/operator/ceph/cluster/migration_test.go b/pkg/operator/ceph/cluster/migration_test.go index 167739d8a..d501e4c1f 100644 --- a/pkg/operator/ceph/cluster/migration_test.go +++ b/pkg/operator/ceph/cluster/migration_test.go @@ -54,6 +54,14 @@ func TestGetClusterObject(t *testing.T) { assert.NotNil(t, err) } +func TestDefaultClusterValues(t *testing.T) { + // the default ceph version should be set + cluster, _, err := getClusterObject(&cephv1beta1.Cluster{}) + assert.NotNil(t, cluster) + assert.Nil(t, err) + assert.Equal(t, cephv1beta1.DefaultLuminousImage, cluster.Spec.CephVersion.Image) +} + func TestMigrateClusterObject(t *testing.T) { // create a legacy cluster that will get migrated legacyCluster := &rookv1alpha1.Cluster{ diff --git a/pkg/operator/ceph/cluster/mon/health_test.go b/pkg/operator/ceph/cluster/mon/health_test.go index faa65b663..effb313f2 100644 --- a/pkg/operator/ceph/cluster/mon/health_test.go +++ b/pkg/operator/ceph/cluster/mon/health_test.go @@ -52,7 +52,7 @@ func TestCheckHealth(t *testing.T) { ConfigDir: configDir, Executor: executor, } - c := New(context, "ns", "", "myversion", cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, + c := New(context, "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(1) logger.Infof("initial mons: %v", c.clusterInfo.Monitors) @@ -96,7 +96,7 @@ func TestCheckHealthNotFound(t *testing.T) { ConfigDir: configDir, Executor: executor, } - c := New(context, "ns", "", "myversion", cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, + c := New(context, "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(2) c.waitForStart = false @@ -165,7 +165,7 @@ func TestCheckHealthTwoMonsOneNode(t *testing.T) { ConfigDir: configDir, Executor: executor, } - c := New(context, "ns", "", "myversion", cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, + c := New(context, "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(2) c.waitForStart = false @@ -276,7 +276,7 @@ func TestCheckMonsValid(t *testing.T) { ConfigDir: configDir, Executor: executor, } - c := New(context, "ns", "", "myversion", cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, + c := New(context, "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(1) c.waitForStart = false @@ -361,7 +361,7 @@ func TestAddRemoveMons(t *testing.T) { ConfigDir: configDir, Executor: executor, } - c := New(context, "ns", "", "myversion", cephv1beta1.MonSpec{Count: 5, AllowMultiplePerNode: true}, + c := New(context, "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 5, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.maxMonID = 0 c.clusterInfo = test.CreateConfigDir(0) diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index a0512419f..95d189258 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -75,7 +75,8 @@ type Cluster struct { context *clusterd.Context Namespace string Keyring string - Version string + rookVersion string + cephVersion cephv1beta1.CephVersionSpec Count int AllowMultiplePerNode bool MonCountMutex sync.Mutex @@ -120,14 +121,15 @@ type NodeInfo struct { } // New creates an instance of a mon cluster -func New(context *clusterd.Context, namespace, dataDirHostPath, version string, mon cephv1beta1.MonSpec, placement rookalpha.Placement, hostNetwork bool, - resources v1.ResourceRequirements, ownerRef metav1.OwnerReference) *Cluster { +func New(context *clusterd.Context, namespace, dataDirHostPath, rookVersion string, cephVersion cephv1beta1.CephVersionSpec, mon cephv1beta1.MonSpec, + placement rookalpha.Placement, hostNetwork bool, resources v1.ResourceRequirements, ownerRef metav1.OwnerReference) *Cluster { return &Cluster{ context: context, placement: placement, dataDirHostPath: dataDirHostPath, Namespace: namespace, - Version: version, + rookVersion: rookVersion, + cephVersion: cephVersion, Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode, maxMonID: -1, diff --git a/pkg/operator/ceph/cluster/mon/mon_test.go b/pkg/operator/ceph/cluster/mon/mon_test.go index 9b3075255..7911a3941 100644 --- a/pkg/operator/ceph/cluster/mon/mon_test.go +++ b/pkg/operator/ceph/cluster/mon/mon_test.go @@ -68,7 +68,7 @@ func newCluster(context *clusterd.Context, namespace string, hostNetwork bool, r HostNetwork: hostNetwork, context: context, Namespace: namespace, - Version: "myversion", + rookVersion: "myversion", Count: 3, AllowMultiplePerNode: true, maxMonID: -1, @@ -166,7 +166,7 @@ func TestSaveMonEndpoints(t *testing.T) { clientset := test.New(1) configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", "", "myversion", + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(1) @@ -240,7 +240,7 @@ func TestNameToIndex(t *testing.T) { func TestAvailableMonNodes(t *testing.T) { clientset := test.New(1) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", + c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(0) @@ -267,7 +267,7 @@ func TestAvailableMonNodes(t *testing.T) { func TestAvailableNodesInUse(t *testing.T) { clientset := test.New(3) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", + c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(0) @@ -308,7 +308,7 @@ func TestAvailableNodesInUse(t *testing.T) { func TestTaintedNodes(t *testing.T) { clientset := test.New(3) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", + c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(0) @@ -343,7 +343,7 @@ func TestTaintedNodes(t *testing.T) { func TestNodeAffinity(t *testing.T) { clientset := test.New(3) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", + c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(0) @@ -385,7 +385,7 @@ func TestNodeAffinity(t *testing.T) { func TestHostNetwork(t *testing.T) { clientset := test.New(3) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", + c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(0) @@ -417,7 +417,7 @@ func TestGetNodeInfoFromNode(t *testing.T) { }, } - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", + c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, true, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(0) @@ -445,10 +445,8 @@ func TestHostNetworkPortIncrease(t *testing.T) { configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) - c := New(&clusterd.Context{ - Clientset: clientset, - ConfigDir: configDir, - }, "ns", "", "myversion", cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, + "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, true, v1.ResourceRequirements{}, metav1.OwnerReference{}) c.clusterInfo = test.CreateConfigDir(0) diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 47c4c3350..d64781020 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -142,7 +142,7 @@ func (c *Cluster) makeConfigInitContainer(monConfig *monConfig) v1.Container { fmt.Sprintf("--port=%d", monConfig.Port), fmt.Sprintf("--fsid=%s", c.clusterInfo.FSID), }, - Image: k8sutil.MakeRookImage(c.Version), + Image: k8sutil.MakeRookImage(c.rookVersion), Env: []v1.EnvVar{ k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), {Name: k8sutil.PublicIPEnvVar, Value: monConfig.PublicIP}, @@ -186,7 +186,7 @@ func (c *Cluster) makeMonmapInitContainer(monConfig *monConfig) v1.Container { }, monmapAddMonArgs..., ), - Image: k8sutil.MakeRookImage(c.Version), // TODO: ceph: image + Image: c.cephVersion.Image, VolumeMounts: opspec.CephVolumeMounts(), SecurityContext: podSecurityContext(), // monmap creation does not require ports to be exposed @@ -215,7 +215,7 @@ func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) v1.Container { }, c.cephMonCommonArgs(monConfig)..., ), - Image: k8sutil.MakeRookImage(c.Version), // TODO: ceph: image + Image: c.cephVersion.Image, VolumeMounts: opspec.CephVolumeMounts(), SecurityContext: podSecurityContext(), // filesystem creation does not require ports to be exposed @@ -246,7 +246,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { }, c.cephMonCommonArgs(monConfig)..., ), - Image: k8sutil.MakeRookImage(c.Version), // TODO: ceph: image + Image: c.cephVersion.Image, VolumeMounts: opspec.CephVolumeMounts(), SecurityContext: podSecurityContext(), Ports: []v1.ContainerPort{ diff --git a/pkg/operator/ceph/cluster/mon/spec_test.go b/pkg/operator/ceph/cluster/mon/spec_test.go index c8a6cc0b3..21b288678 100644 --- a/pkg/operator/ceph/cluster/mon/spec_test.go +++ b/pkg/operator/ceph/cluster/mon/spec_test.go @@ -53,6 +53,7 @@ func testPodSpec(t *testing.T, dataDir string) { "ns", dataDir, "rook/rook:myversion", + cephv1beta1.CephVersionSpec{Image: "ceph/ceph:myceph"}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false, @@ -123,7 +124,7 @@ func testPodSpec(t *testing.T, dataDir string) { assert.Equal(t, "1337", cont.Resources.Requests.Memory().String()) // All ceph images have the same image, no envs, and the same volume mounts - cephImage := "rook/rook:myversion" + cephImage := "ceph/ceph:myceph" cephEnvs := 0 cephVolumeMountNames := []string{ "rook-data", diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 885fbea12..075499adf 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -22,9 +22,9 @@ import ( "os" "strconv" "strings" - "time" "github.com/coreos/pkg/capnslog" + cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" @@ -60,7 +60,8 @@ type Cluster struct { Namespace string placement rookalpha.Placement Keyring string - Version string + rookVersion string + cephVersion cephv1beta1.CephVersionSpec Storage rookalpha.StorageScopeSpec dataDirHostPath string HostNetwork bool @@ -73,8 +74,9 @@ type Cluster struct { // New creates an instance of the OSD manager func New( context *clusterd.Context, - namespace, - version, + namespace string, + rookVersion string, + cephVersion cephv1beta1.CephVersionSpec, serviceAccount string, storageSpec rookalpha.StorageScopeSpec, dataDirHostPath string, @@ -95,7 +97,8 @@ func New( Namespace: namespace, serviceAccount: serviceAccount, placement: placement, - Version: version, + rookVersion: rookVersion, + cephVersion: cephVersion, Storage: storageSpec, dataDirHostPath: dataDirHostPath, HostNetwork: hostNetwork, @@ -255,7 +258,7 @@ func (c *Cluster) startProvisioning(config *provisionConfig) { } } - if !c.updateJob(job, n.Name, config, "provision") { + if !c.runJob(job, n.Name, config, "provision") { if err = discover.FreeDevices(c.context, n.Name, c.Namespace); err != nil { logger.Warningf("failed to free devices: %s", err) } @@ -263,27 +266,8 @@ func (c *Cluster) startProvisioning(config *provisionConfig) { } } -func (c *Cluster) updateJob(job *batch.Job, nodeName string, config *provisionConfig, action string) bool { - // check if the job was already created and what its status is - existingJob, err := c.context.Clientset.Batch().Jobs(c.Namespace).Get(job.Name, metav1.GetOptions{}) - if err != nil && !errors.IsNotFound(err) { - config.addError("failed to detect %s job for node %s. %+v", action, nodeName, err) - } else if err == nil { - // delete the job that already exists from a previous run - if existingJob.Status.Active > 0 { - logger.Infof("Found previous %s job for node %s. Status=%+v", action, nodeName, existingJob.Status) - return true - } - - logger.Infof("Removing previous %s job for node %s to start a new one", action, nodeName) - err := c.deleteBatchJob(existingJob.Name) - if err != nil { - logger.Warningf("failed to remove job %s. %+v", nodeName, err) - } - } - - _, err = c.context.Clientset.Batch().Jobs(c.Namespace).Create(job) - if err != nil { +func (c *Cluster) runJob(job *batch.Job, nodeName string, config *provisionConfig, action string) bool { + if err := k8sutil.RunReplaceableJob(c.context.Clientset, job); err != nil { if !errors.IsAlreadyExists(err) { // we failed to create job, update the orchestration status for this node message := fmt.Sprintf("failed to create %s job for node %s. %+v", action, nodeName, err) @@ -298,32 +282,6 @@ func (c *Cluster) updateJob(job *batch.Job, nodeName string, config *provisionCo return true } -func (c *Cluster) deleteBatchJob(name string) error { - propagation := metav1.DeletePropagationForeground - gracePeriod := int64(0) - options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation} - - if err := c.context.Clientset.Batch().Jobs(c.Namespace).Delete(name, options); err != nil { - return fmt.Errorf("failed to remove previous provisioning job for node %s. %+v", name, err) - } - - retries := 20 - sleepInterval := 2 * time.Second - for i := 0; i < retries; i++ { - _, err := c.context.Clientset.Batch().Jobs(c.Namespace).Get(name, metav1.GetOptions{}) - if err != nil && errors.IsNotFound(err) { - logger.Infof("batch job %s deleted", name) - return nil - } - - logger.Infof("batch job %s still exists", name) - time.Sleep(sleepInterval) - } - - logger.Warningf("gave up waiting for batch job %s to be deleted", name) - return nil -} - func (c *Cluster) startOSDDaemonsOnNode(nodeName string, config *provisionConfig, configMap *v1.ConfigMap, status *OrchestrationStatus) { osds := status.OSDs @@ -461,7 +419,7 @@ func (c *Cluster) cleanupRemovedNode(config *provisionConfig, nodeName, crushNam return } - if !c.updateJob(job, nodeName, config, "remove") { + if !c.runJob(job, nodeName, config, "remove") { status := OrchestrationStatus{Status: OrchestrationStatusCompleted, Message: fmt.Sprintf("failed to cleanup osd config on node %s", nodeName)} if err := c.updateNodeStatus(nodeName, status); err != nil { config.addError("failed to update node %s status. %+v", nodeName, err) diff --git a/pkg/operator/ceph/cluster/osd/osd_test.go b/pkg/operator/ceph/cluster/osd/osd_test.go index f6251f5ca..ec7679e86 100644 --- a/pkg/operator/ceph/cluster/osd/osd_test.go +++ b/pkg/operator/ceph/cluster/osd/osd_test.go @@ -20,6 +20,7 @@ import ( "os" "testing" + cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" discoverDaemon "github.com/rook/rook/pkg/daemon/discover" @@ -40,7 +41,7 @@ import ( func TestStart(t *testing.T) { clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "", + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", cephv1beta1.CephVersionSpec{}, "", rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) // Start the first time @@ -89,7 +90,7 @@ func createNode(nodeName string, condition v1.NodeConditionType, clientset *fake func TestLegacyDeployment(t *testing.T) { clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset}, "ns", "myversion", "", + c := New(&clusterd.Context{Clientset: clientset}, "ns", "myversion", cephv1beta1.CephVersionSpec{}, "", rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) osdID := 23 @@ -144,7 +145,7 @@ func TestAddRemoveNode(t *testing.T) { statusMapWatcher := watch.NewFake() clientset.PrependWatchReactor("configmaps", k8stesting.DefaultWatchReactor(statusMapWatcher, nil)) - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", "", + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", cephv1beta1.CephVersionSpec{}, "", storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) // kick off the start of the orchestration in a goroutine @@ -229,7 +230,7 @@ func TestAddRemoveNode(t *testing.T) { // modify the storage spec to remove the node from the cluster storageSpec.Nodes = []rookalpha.Node{} - c = New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: mockExec}, "ns-add-remove", "myversion", "", + c = New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: mockExec}, "ns-add-remove", "myversion", cephv1beta1.CephVersionSpec{}, "", storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) // reset the orchestration status watcher @@ -268,7 +269,7 @@ func TestGetIDFromDeployment(t *testing.T) { } func TestDiscoverOSDs(t *testing.T) { - c := New(&clusterd.Context{}, "ns", "myversion", "", + c := New(&clusterd.Context{}, "ns", "myversion", cephv1beta1.CephVersionSpec{}, "", rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) node1 := "n1" node2 := "n2" @@ -336,7 +337,7 @@ func TestAddNodeFailure(t *testing.T) { cmErr := createDiscoverConfigmap(nodeName, "rook-system", clientset) assert.Nil(t, cmErr) - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", "", + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", cephv1beta1.CephVersionSpec{}, "", storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) // kick off the start of the orchestration in a goroutine diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index a71dc41fa..fb01a4d95 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -44,6 +44,8 @@ const ( osdWalSizeEnvVarName = "ROOK_OSD_WAL_SIZE" osdJournalSizeEnvVarName = "ROOK_OSD_JOURNAL_SIZE" osdMetadataDeviceEnvVarName = "ROOK_METADATA_DEVICE" + rookBinariesMountPath = "/rook" + rookBinariesVolumeName = "rook-binaries" ) func (c *Cluster) makeJob(nodeName string, devices []rookalpha.Device, @@ -138,6 +140,7 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se var command []string var args []string + var copyBinariesContainer *v1.Container if !osd.IsDirectory && osd.IsFileStore { // All scenarios except one can call the ceph-osd daemon directly. The one different scenario is when // filestore is running on a device. Rook needs to mount the device, run the ceph-osd daemon, and then @@ -145,27 +148,21 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se // for this scenario, we will copy the binaries necessary to a mount, which will then be mounted // to the daemon container. sourcePath := path.Join("/dev/disk/by-partuuid", osd.DevicePartUUID) - mountPath := "/rook" - command = []string{path.Join(mountPath, "tini")} + command = []string{path.Join(rookBinariesMountPath, "tini")} args = append([]string{ - "--", path.Join(mountPath, "rook"), + "--", path.Join(rookBinariesMountPath, "rook"), "ceph", "osd", "filestore-device", "--source-path", sourcePath, "--mount-path", osd.DataPath, "--"}, commonArgs...) - // To get rook inside the container, the config init container needs to copy "tini" and "rook" binaries into a volume. - // Set the config flag so rook will copy the binaries. - configEnvVars = append(configEnvVars, v1.EnvVar{Name: "ROOK_COPY_BINARIES_PATH", Value: mountPath}) + var copyBinariesVolume v1.Volume + copyBinariesVolume, copyBinariesContainer = c.getCopyBinariesContainer() + // Add the volume to the spec and the mount to the daemon container + volumes = append(volumes, copyBinariesVolume) + volumeMounts = append(volumeMounts, copyBinariesContainer.VolumeMounts[0]) - // Create the volume and mount that will be shared between the init container and the daemon container - volumeName := "rookbinaries" - binariesVolume := v1.Volume{Name: volumeName, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}} - volumes = append(volumes, binariesVolume) - binariesMount := v1.VolumeMount{Name: volumeName, MountPath: mountPath} - volumeMounts = append(volumeMounts, binariesMount) - configVolumeMounts = append(configVolumeMounts, binariesMount) } else { // other osds can launch the osd daemon directly command = []string{"ceph-osd"} @@ -220,7 +217,7 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se { Args: []string{"ceph", "osd", "init"}, Name: opspec.ConfigInitContainerName, - Image: k8sutil.MakeRookImage(c.Version), + Image: k8sutil.MakeRookImage(c.rookVersion), VolumeMounts: configVolumeMounts, Env: configEnvVars, SecurityContext: securityContext, @@ -231,7 +228,7 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se Command: command, Args: args, Name: "osd", - Image: k8sutil.MakeRookImage(c.Version), + Image: c.cephVersion.Image, VolumeMounts: volumeMounts, Env: envVars, Resources: resources, @@ -244,15 +241,36 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se Replicas: &replicaCount, }, } + if copyBinariesContainer != nil { + deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, *copyBinariesContainer) + } k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &deployment.ObjectMeta, &c.ownerRef) c.placement.ApplyToPodSpec(&deployment.Spec.Template.Spec) return deployment, nil } +// To get rook inside the container, the config init container needs to copy "tini" and "rook" binaries into a volume. +// Get the config flag so rook will copy the binaries and create the volume and mount that will be shared between +// the init container and the daemon container +func (c *Cluster) getCopyBinariesContainer() (v1.Volume, *v1.Container) { + volume := v1.Volume{Name: rookBinariesVolumeName, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}} + mount := v1.VolumeMount{Name: rookBinariesVolumeName, MountPath: rookBinariesMountPath} + + return volume, &v1.Container{ + Args: []string{"ceph", "osd", "copybins"}, + Name: "copy-bins", + Image: k8sutil.MakeRookImage(c.rookVersion), + VolumeMounts: []v1.VolumeMount{mount}, + Env: []v1.EnvVar{{Name: "ROOK_PATH", Value: rookBinariesMountPath}}, + } +} + func (c *Cluster) provisionPodTemplateSpec(devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements, storeConfig config.StoreConfig, metadataDevice, nodeName, location string, restart v1.RestartPolicy) (*v1.PodTemplateSpec, error) { - volumes := opspec.PodVolumes(c.dataDirHostPath) + copyBinariesVolume, copyBinariesContainer := c.getCopyBinariesContainer() + + volumes := append(opspec.PodVolumes(c.dataDirHostPath), copyBinariesVolume) // by default, don't define any volume config unless it is required if len(devices) > 0 || selection.DeviceFilter != "" || selection.GetUseAllDevices() || metadataDevice != "" { @@ -278,10 +296,13 @@ func (c *Cluster) provisionPodTemplateSpec(devices []rookalpha.Device, selection podSpec := v1.PodSpec{ ServiceAccountName: c.serviceAccount, - Containers: []v1.Container{c.provisionOSDContainer(devices, selection, resources, storeConfig, metadataDevice, nodeName, location)}, - RestartPolicy: restart, - Volumes: volumes, - HostNetwork: c.HostNetwork, + Containers: []v1.Container{ + *copyBinariesContainer, + c.provisionOSDContainer(devices, selection, resources, storeConfig, metadataDevice, nodeName, location, copyBinariesContainer.VolumeMounts[0]), + }, + RestartPolicy: restart, + Volumes: volumes, + HostNetwork: c.HostNetwork, } if c.HostNetwork { podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet @@ -339,7 +360,7 @@ func (c *Cluster) getConfigEnvVars(storeConfig config.StoreConfig, dataDir, node } func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements, - storeConfig config.StoreConfig, metadataDevice, nodeName, location string) v1.Container { + storeConfig config.StoreConfig, metadataDevice, nodeName, location string, copyBinariesMount v1.VolumeMount) v1.Container { envVars := c.getConfigEnvVars(storeConfig, k8sutil.DataDir, nodeName, location) devMountNeeded := false @@ -366,7 +387,7 @@ func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection ro devMountNeeded = true } - volumeMounts := opspec.CephVolumeMounts() + volumeMounts := append(opspec.CephVolumeMounts(), copyBinariesMount) if devMountNeeded { devMount := v1.VolumeMount{Name: "devices", MountPath: "/dev"} volumeMounts = append(volumeMounts, devMount) @@ -395,11 +416,12 @@ func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection ro runAsUser := int64(0) runAsNonRoot := false readOnlyRootFilesystem := false + return v1.Container{ - // Set the hostname so we have the pod's host in the crush map rather than the pod container name - Args: []string{"ceph", "osd", "provision"}, - Name: appName, - Image: k8sutil.MakeRookImage(c.Version), + Command: []string{path.Join(rookBinariesMountPath, "tini")}, + Args: []string{"--", path.Join(rookBinariesMountPath, "rook"), "ceph", "osd", "provision"}, + Name: "provision", + Image: c.cephVersion.Image, VolumeMounts: volumeMounts, Env: envVars, SecurityContext: &v1.SecurityContext{ diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index 6b1eba264..28a1182f8 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -20,6 +20,7 @@ package osd import ( "testing" + cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" @@ -35,14 +36,23 @@ import ( ) func TestPodContainer(t *testing.T) { - cluster := &Cluster{Namespace: "myosd", Version: "23"} + cluster := &Cluster{Namespace: "myosd", rookVersion: "23"} c, err := cluster.provisionPodTemplateSpec([]rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "node", "", v1.RestartPolicyAlways) assert.NotNil(t, c) assert.Nil(t, err) - assert.Equal(t, 1, len(c.Spec.Containers)) + assert.Equal(t, 2, len(c.Spec.Containers)) container := c.Spec.Containers[0] + logger.Infof("container: %+v", container) assert.Equal(t, "ceph", container.Args[0]) assert.Equal(t, "osd", container.Args[1]) + assert.Equal(t, "copybins", container.Args[2]) + container = c.Spec.Containers[1] + assert.Equal(t, "/rook/tini", container.Command[0]) + assert.Equal(t, "--", container.Args[0]) + assert.Equal(t, "/rook/rook", container.Args[1]) + assert.Equal(t, "ceph", container.Args[2]) + assert.Equal(t, "osd", container.Args[3]) + assert.Equal(t, "provision", container.Args[4]) } func TestDaemonset(t *testing.T) { @@ -62,7 +72,8 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { } clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "mysa", + cephVersion := cephv1beta1.CephVersionSpec{Image: "ceph/ceph:v12.2.8"} + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", cephVersion, "mysa", storageSpec, dataDir, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) devMountNeeded := deviceName != "" || allDevices @@ -110,7 +121,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont := deployment.Spec.Template.Spec.Containers[0] - assert.Equal(t, "rook/rook:myversion", cont.Image) + assert.Equal(t, cephVersion.Image, cont.Image) if devMountNeeded { assert.Equal(t, 3, len(cont.VolumeMounts)) } else { @@ -149,7 +160,7 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) { } clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "", + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", cephv1beta1.CephVersionSpec{}, "", storageSpec, "/var/lib/rook", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) n := c.Storage.ResolveNode(storageSpec.Nodes[0].Name) @@ -236,7 +247,7 @@ func TestStorageSpecConfig(t *testing.T) { } clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "", + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", cephv1beta1.CephVersionSpec{}, "", storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) n := c.Storage.ResolveNode(storageSpec.Nodes[0].Name) @@ -249,6 +260,9 @@ func TestStorageSpecConfig(t *testing.T) { assert.Equal(t, "rook-ceph-osd-prepare-node1", job.ObjectMeta.Name) container := job.Spec.Template.Spec.Containers[0] assert.NotNil(t, container) + verifyEnvVar(t, container.Env, "ROOK_PATH", "/rook", true) + container = job.Spec.Template.Spec.Containers[1] + assert.NotNil(t, container) verifyEnvVar(t, container.Env, "ROOK_OSD_STORE", "bluestore", true) verifyEnvVar(t, container.Env, "ROOK_OSD_DATABASE_SIZE", "10", true) verifyEnvVar(t, container.Env, "ROOK_OSD_WAL_SIZE", "20", true) @@ -283,7 +297,7 @@ func TestHostNetwork(t *testing.T) { } clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "", + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", cephv1beta1.CephVersionSpec{}, "", storageSpec, "", rookalpha.Placement{}, true, v1.ResourceRequirements{}, metav1.OwnerReference{}) n := c.Storage.ResolveNode(storageSpec.Nodes[0].Name) diff --git a/pkg/operator/ceph/cluster/osd/status_test.go b/pkg/operator/ceph/cluster/osd/status_test.go index 6d8033046..412f6de22 100644 --- a/pkg/operator/ceph/cluster/osd/status_test.go +++ b/pkg/operator/ceph/cluster/osd/status_test.go @@ -21,6 +21,7 @@ import ( "testing" "time" + cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" @@ -35,7 +36,7 @@ import ( func TestOrchestrationStatus(t *testing.T) { clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "", + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", cephv1beta1.CephVersionSpec{}, "", rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) kv := k8sutil.NewConfigMapKVStore(c.Namespace, clientset, metav1.OwnerReference{}) nodeName := "mynode" diff --git a/pkg/operator/ceph/file/controller.go b/pkg/operator/ceph/file/controller.go index fd895574a..72b9fb6ea 100644 --- a/pkg/operator/ceph/file/controller.go +++ b/pkg/operator/ceph/file/controller.go @@ -64,7 +64,8 @@ var filesystemResourceRookLegacy = opkit.CustomResource{ // FilesystemController represents a controller for file system custom resources type FilesystemController struct { context *clusterd.Context - rookImage string + rookVersion string + cephVersion cephv1beta1.CephVersionSpec hostNetwork bool ownerRef metav1.OwnerReference } @@ -72,13 +73,15 @@ type FilesystemController struct { // NewFilesystemController create controller for watching file system custom resources created func NewFilesystemController( context *clusterd.Context, - rookImage string, + rookVersion string, + cephVersion cephv1beta1.CephVersionSpec, hostNetwork bool, ownerRef metav1.OwnerReference, ) *FilesystemController { return &FilesystemController{ context: context, - rookImage: rookImage, + rookVersion: rookVersion, + cephVersion: cephVersion, hostNetwork: hostNetwork, ownerRef: ownerRef, } @@ -117,7 +120,7 @@ func (c *FilesystemController) onAdd(obj interface{}) { return } - err = createFilesystem(c.context, *filesystem, c.rookImage, c.hostNetwork, c.filesystemOwners(filesystem)) + err = createFilesystem(c.context, *filesystem, c.rookVersion, c.cephVersion, c.hostNetwork, c.filesystemOwners(filesystem)) if err != nil { logger.Errorf("failed to create file system %s: %+v", filesystem.Name, err) } @@ -149,7 +152,7 @@ func (c *FilesystemController) onUpdate(oldObj, newObj interface{}) { // if the file system is modified, allow the file system to be created if it wasn't already logger.Infof("updating filesystem %s", newFS.Name) - err = createFilesystem(c.context, *newFS, c.rookImage, c.hostNetwork, c.filesystemOwners(newFS)) + err = createFilesystem(c.context, *newFS, c.rookVersion, c.cephVersion, c.hostNetwork, c.filesystemOwners(newFS)) if err != nil { logger.Errorf("failed to create (modify) file system %s: %+v", newFS.Name, err) } diff --git a/pkg/operator/ceph/file/controller_test.go b/pkg/operator/ceph/file/controller_test.go index 9e514bc5a..1baf32dbe 100644 --- a/pkg/operator/ceph/file/controller_test.go +++ b/pkg/operator/ceph/file/controller_test.go @@ -83,7 +83,7 @@ func TestMigrateFilesystemObject(t *testing.T) { Clientset: clientset, RookClientset: rookfake.NewSimpleClientset(legacyFilesystem), } - controller := NewFilesystemController(context, "", false, metav1.OwnerReference{}) + controller := NewFilesystemController(context, "", cephv1beta1.CephVersionSpec{}, false, metav1.OwnerReference{}) // convert the legacy filesystem object in memory and assert that a migration is needed convertedFilesystem, migrationNeeded, err := getFilesystemObject(legacyFilesystem) diff --git a/pkg/operator/ceph/file/filesystem.go b/pkg/operator/ceph/file/filesystem.go index b635567a0..f6450ddbc 100644 --- a/pkg/operator/ceph/file/filesystem.go +++ b/pkg/operator/ceph/file/filesystem.go @@ -32,7 +32,8 @@ import ( func createFilesystem( context *clusterd.Context, fs cephv1beta1.Filesystem, - version string, + rookVersion string, + cephVersion cephv1beta1.CephVersionSpec, hostNetwork bool, ownerRefs []metav1.OwnerReference, ) error { @@ -55,7 +56,7 @@ func createFilesystem( } logger.Infof("start running mdses for file system %s", fs.Name) - c := newCluster(context, version, hostNetwork, fs, filesystem, ownerRefs) + c := newCluster(context, rookVersion, cephVersion, hostNetwork, fs, filesystem, ownerRefs) if err := c.start(); err != nil { return err } diff --git a/pkg/operator/ceph/file/filesystem_test.go b/pkg/operator/ceph/file/filesystem_test.go index 4b1ab8aa7..7ced860c8 100644 --- a/pkg/operator/ceph/file/filesystem_test.go +++ b/pkg/operator/ceph/file/filesystem_test.go @@ -106,12 +106,12 @@ func TestCreateFilesystem(t *testing.T) { } // start a basic cluster - err := createFilesystem(context, fs, "v0.1", false, []metav1.OwnerReference{}) + err := createFilesystem(context, fs, "v0.1", cephv1beta1.CephVersionSpec{}, false, []metav1.OwnerReference{}) assert.Nil(t, err) validateStart(t, context, fs) // starting again should be a no-op - err = createFilesystem(context, fs, "v0.1", false, []metav1.OwnerReference{}) + err = createFilesystem(context, fs, "v0.1", cephv1beta1.CephVersionSpec{}, false, []metav1.OwnerReference{}) assert.Nil(t, err) validateStart(t, context, fs) @@ -130,7 +130,7 @@ func TestCreateFilesystem(t *testing.T) { Clientset: testop.New(3)} //Create another filesystem which should fail - err = createFilesystem(context, fs, "v0.1", false, []metav1.OwnerReference{}) + err = createFilesystem(context, fs, "v0.1", cephv1beta1.CephVersionSpec{}, false, []metav1.OwnerReference{}) assert.Equal(t, "failed to create file system myfs: Cannot create multiple filesystems. Enable ROOK_ALLOW_MULTIPLE_FILESYSTEMS env variable to create more than one", err.Error()) } diff --git a/pkg/operator/ceph/file/mds.go b/pkg/operator/ceph/file/mds.go index 8f31dab25..69e5a921b 100644 --- a/pkg/operator/ceph/file/mds.go +++ b/pkg/operator/ceph/file/mds.go @@ -44,7 +44,8 @@ const ( type cluster struct { context *clusterd.Context - Version string + rookVersion string + cephVersion cephv1beta1.CephVersionSpec HostNetwork bool fs cephv1beta1.Filesystem fsID string @@ -53,7 +54,8 @@ type cluster struct { func newCluster( context *clusterd.Context, - version string, + rookVersion string, + cephVersion cephv1beta1.CephVersionSpec, hostNetwork bool, fs cephv1beta1.Filesystem, fsdetails *client.CephFilesystemDetails, @@ -61,7 +63,8 @@ func newCluster( ) *cluster { return &cluster{ context: context, - Version: version, + rookVersion: rookVersion, + cephVersion: cephVersion, HostNetwork: hostNetwork, fs: fs, fsID: strconv.Itoa(fsdetails.ID), diff --git a/pkg/operator/ceph/file/spec.go b/pkg/operator/ceph/file/spec.go index 731c0050f..942403096 100644 --- a/pkg/operator/ceph/file/spec.go +++ b/pkg/operator/ceph/file/spec.go @@ -79,7 +79,7 @@ func (c *cluster) makeConfigInitContainer(mdsConfig *mdsConfig) v1.Container { "--filesystem-id", c.fsID, "--active-standby", strconv.FormatBool(c.fs.Spec.MetadataServer.ActiveStandby), }, - Image: k8sutil.MakeRookImage(c.Version), + Image: k8sutil.MakeRookImage(c.rookVersion), Env: []v1.EnvVar{ // Set '--mds-keyring' flag with an env var sourced from the secret {Name: "ROOK_MDS_KEYRING", @@ -112,7 +112,7 @@ func (c *cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { "--id", mdsConfig.DaemonName, // do not add the '--cluster/--conf/--keyring' flags; rook wants their default values }, - Image: k8sutil.MakeRookImage(c.Version), + Image: c.cephVersion.Image, Env: k8sutil.ClusterDaemonEnvVars(), VolumeMounts: opspec.CephVolumeMounts(), // TODO: mds doesn't need ports? diff --git a/pkg/operator/ceph/file/spec_test.go b/pkg/operator/ceph/file/spec_test.go index 90cdd52da..fb8e82fb8 100644 --- a/pkg/operator/ceph/file/spec_test.go +++ b/pkg/operator/ceph/file/spec_test.go @@ -52,6 +52,7 @@ func testDeploymentObject(hostNetwork bool) *extensions.Deployment { c := newCluster( &clusterd.Context{Clientset: testop.New(1)}, "rook/rook:myversion", + cephv1beta1.CephVersionSpec{Image: "ceph/ceph:testversion"}, hostNetwork, fs, &client.CephFilesystemDetails{ID: 15}, @@ -113,7 +114,7 @@ func TestPodSpecs(t *testing.T) { assert.Equal(t, "100", cont.Resources.Limits.Cpu().String()) assert.Equal(t, "1337", cont.Resources.Requests.Memory().String()) - daemonImage := "rook/rook:myversion" + daemonImage := "ceph/ceph:testversion" daemonEnvs := len(k8sutil.ClusterDaemonEnvVars()) daemonContainerDefinition := cephtest.ContainerTestDefinition{ Image: &daemonImage, diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index f0f6a1d97..e4940eafc 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -64,15 +64,17 @@ var ObjectStoreResourceRookLegacy = opkit.CustomResource{ type ObjectStoreController struct { context *clusterd.Context rookImage string + cephVersion cephv1beta1.CephVersionSpec hostNetwork bool ownerRef metav1.OwnerReference } // NewObjectStoreController create controller for watching object store custom resources created -func NewObjectStoreController(context *clusterd.Context, rookImage string, hostNetwork bool, ownerRef metav1.OwnerReference) *ObjectStoreController { +func NewObjectStoreController(context *clusterd.Context, rookImage string, cephVersion cephv1beta1.CephVersionSpec, hostNetwork bool, ownerRef metav1.OwnerReference) *ObjectStoreController { return &ObjectStoreController{ context: context, rookImage: rookImage, + cephVersion: cephVersion, hostNetwork: hostNetwork, ownerRef: ownerRef, } @@ -111,7 +113,8 @@ func (c *ObjectStoreController) onAdd(obj interface{}) { return } - if err = CreateStore(c.context, *objectstore, c.rookImage, c.hostNetwork, c.storeOwners(objectstore)); err != nil { + cfg := config{c.context, *objectstore, c.rookImage, c.cephVersion, c.hostNetwork, c.storeOwners(objectstore)} + if err = cfg.createStore(); err != nil { logger.Errorf("failed to create object store %s. %+v", objectstore.Name, err) } } @@ -142,7 +145,8 @@ func (c *ObjectStoreController) onUpdate(oldObj, newObj interface{}) { } logger.Infof("applying object store %s changes", newStore.Name) - if err = UpdateStore(c.context, *newStore, c.rookImage, c.hostNetwork, c.storeOwners(newStore)); err != nil { + cfg := config{c.context, *newStore, c.rookImage, c.cephVersion, c.hostNetwork, c.storeOwners(newStore)} + if err = cfg.updateStore(); err != nil { logger.Errorf("failed to create (modify) object store %s. %+v", newStore.Name, err) } } @@ -159,7 +163,8 @@ func (c *ObjectStoreController) onDelete(obj interface{}) { return } - if err = DeleteStore(c.context, *objectstore); err != nil { + cfg := config{context: c.context, store: *objectstore} + if err = cfg.deleteStore(); err != nil { logger.Errorf("failed to delete object store %s. %+v", objectstore.Name, err) } } diff --git a/pkg/operator/ceph/object/controller_test.go b/pkg/operator/ceph/object/controller_test.go index dcff045ed..3bff6a0d7 100644 --- a/pkg/operator/ceph/object/controller_test.go +++ b/pkg/operator/ceph/object/controller_test.go @@ -91,7 +91,7 @@ func TestMigrateObjectStoreObject(t *testing.T) { Clientset: clientset, RookClientset: rookfake.NewSimpleClientset(legacyObjectStore), } - controller := NewObjectStoreController(context, "", false, metav1.OwnerReference{}) + controller := NewObjectStoreController(context, "", cephv1beta1.CephVersionSpec{}, false, metav1.OwnerReference{}) // convert the legacy objectstore object in memory and assert that a migration is needed convertedObjectStore, migrationNeeded, err := getObjectStoreObject(legacyObjectStore) diff --git a/pkg/operator/ceph/object/rgw.go b/pkg/operator/ceph/object/rgw.go index 263a702f1..2331a84ba 100644 --- a/pkg/operator/ceph/object/rgw.go +++ b/pkg/operator/ceph/object/rgw.go @@ -40,67 +40,76 @@ const ( certFilename = "rgw-cert.pem" ) +type config struct { + context *clusterd.Context + store cephv1beta1.ObjectStore + rookVersion string + cephVersion cephv1beta1.CephVersionSpec + hostNetwork bool + ownerRefs []metav1.OwnerReference +} + // Start the rgw manager -func CreateStore(context *clusterd.Context, store cephv1beta1.ObjectStore, version string, hostNetwork bool, ownerRefs []metav1.OwnerReference) error { - return createOrUpdate(context, store, version, hostNetwork, false, ownerRefs) +func (c *config) createStore() error { + return c.createOrUpdate(false) } -func UpdateStore(context *clusterd.Context, store cephv1beta1.ObjectStore, version string, hostNetwork bool, ownerRefs []metav1.OwnerReference) error { - return createOrUpdate(context, store, version, hostNetwork, true, ownerRefs) +func (c *config) updateStore() error { + return c.createOrUpdate(true) } -func createOrUpdate(context *clusterd.Context, store cephv1beta1.ObjectStore, version string, hostNetwork, update bool, ownerRefs []metav1.OwnerReference) error { +func (c *config) createOrUpdate(update bool) error { // validate the object store settings - if err := validateStore(context, store); err != nil { - return fmt.Errorf("invalid object store %s arguments. %+v", store.Name, err) + if err := validateStore(c.context, c.store); err != nil { + return fmt.Errorf("invalid object store %s arguments. %+v", c.store.Name, err) } // check if the object store already exists - exists, err := storeExists(context, store) + exists, err := c.storeExists() if err == nil && exists { if !update { - logger.Infof("object store %s exists in namespace %s", store.Name, store.Namespace) + logger.Infof("object store %s exists in namespace %s", c.store.Name, c.store.Namespace) return nil } - logger.Infof("object store %s exists in namespace %s. checking for updates", store.Name, store.Namespace) + logger.Infof("object store %s exists in namespace %s. checking for updates", c.store.Name, c.store.Namespace) } - logger.Infof("creating object store %s in namespace %s", store.Name, store.Namespace) - err = createKeyring(context, store, ownerRefs) + logger.Infof("creating object store %s in namespace %s", c.store.Name, c.store.Namespace) + err = c.createKeyring() if err != nil { return fmt.Errorf("failed to create rgw keyring. %+v", err) } // start the service - serviceIP, err := startService(context, store, hostNetwork, ownerRefs) + serviceIP, err := c.startService() if err != nil { return fmt.Errorf("failed to start rgw service. %+v", err) } // create the ceph artifacts for the object store - objContext := rgwdaemon.NewContext(context, store.Name, store.Namespace) - err = rgwdaemon.CreateObjectStore(objContext, *store.Spec.MetadataPool.ToModel(""), *store.Spec.DataPool.ToModel(""), serviceIP, store.Spec.Gateway.Port) + objContext := rgwdaemon.NewContext(c.context, c.store.Name, c.store.Namespace) + err = rgwdaemon.CreateObjectStore(objContext, *c.store.Spec.MetadataPool.ToModel(""), *c.store.Spec.DataPool.ToModel(""), serviceIP, c.store.Spec.Gateway.Port) if err != nil { return fmt.Errorf("failed to create pools. %+v", err) } - if err := startRGWPods(context, store, version, hostNetwork, update, ownerRefs); err != nil { + if err := c.startRGWPods(update); err != nil { return fmt.Errorf("failed to start pods. %+v", err) } - logger.Infof("created object store %s", store.Name) + logger.Infof("created object store %s", c.store.Name) return nil } -func startRGWPods(context *clusterd.Context, store cephv1beta1.ObjectStore, version string, hostNetwork, update bool, ownerRefs []metav1.OwnerReference) error { +func (c *config) startRGWPods(update bool) error { // if intended to update, remove the old pods so they can be created with the new spec settings if update { - err := k8sutil.DeleteDeployment(context.Clientset, store.Namespace, instanceName(store)) + err := k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, c.instanceName()) if err != nil { logger.Warning(err.Error()) } - err = k8sutil.DeleteDaemonset(context.Clientset, store.Namespace, instanceName(store)) + err = k8sutil.DeleteDaemonset(c.context.Clientset, c.store.Namespace, c.instanceName()) if err != nil { logger.Warning(err.Error()) } @@ -109,12 +118,12 @@ func startRGWPods(context *clusterd.Context, store cephv1beta1.ObjectStore, vers // start the deployment or daemonset var rgwType string var err error - if store.Spec.Gateway.AllNodes { + if c.store.Spec.Gateway.AllNodes { rgwType = "daemonset" - err = startDaemonset(context, store, version, hostNetwork, ownerRefs) + err = c.startDaemonset() } else { rgwType = "deployment" - err = startDeployment(context, store, version, store.Spec.Gateway.Instances, hostNetwork, ownerRefs) + err = c.startDeployment() } if err != nil { @@ -131,59 +140,59 @@ func startRGWPods(context *clusterd.Context, store cephv1beta1.ObjectStore, vers // Delete the object store. // WARNING: This is a very destructive action that deletes all metadata and data pools. -func DeleteStore(context *clusterd.Context, store cephv1beta1.ObjectStore) error { +func (c *config) deleteStore() error { // check if the object store exists - exists, err := storeExists(context, store) + exists, err := c.storeExists() if err != nil { return fmt.Errorf("failed to detect if there is an object store to delete. %+v", err) } if !exists { - logger.Infof("Object store %s does not exist in namespace %s", store.Name, store.Namespace) + logger.Infof("Object store %s does not exist in namespace %s", c.store.Name, c.store.Namespace) return nil } - logger.Infof("Deleting object store %s from namespace %s", store.Name, store.Namespace) + logger.Infof("Deleting object store %s from namespace %s", c.store.Name, c.store.Namespace) var gracePeriod int64 propagation := metav1.DeletePropagationForeground options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation} // Delete the rgw service - err = context.Clientset.CoreV1().Services(store.Namespace).Delete(instanceName(store), options) + err = c.context.Clientset.CoreV1().Services(c.store.Namespace).Delete(c.instanceName(), options) if err != nil && !errors.IsNotFound(err) { logger.Warningf("failed to delete rgw service. %+v", err) } // Make a best effort to delete the rgw pods - err = k8sutil.DeleteDeployment(context.Clientset, store.Namespace, instanceName(store)) + err = k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, c.instanceName()) if err != nil { logger.Warning(err.Error()) } - err = k8sutil.DeleteDaemonset(context.Clientset, store.Namespace, instanceName(store)) + err = k8sutil.DeleteDaemonset(c.context.Clientset, c.store.Namespace, c.instanceName()) if err != nil { logger.Warning(err.Error()) } // Delete the rgw keyring - err = context.Clientset.CoreV1().Secrets(store.Namespace).Delete(instanceName(store), options) + err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Delete(c.instanceName(), options) if err != nil && !errors.IsNotFound(err) { logger.Warningf("failed to delete rgw secret. %+v", err) } // Delete the realm and pools - objContext := rgwdaemon.NewContext(context, store.Name, store.Namespace) + objContext := rgwdaemon.NewContext(c.context, c.store.Name, c.store.Namespace) err = rgwdaemon.DeleteObjectStore(objContext) if err != nil { return fmt.Errorf("failed to delete the realm and pools. %+v", err) } - logger.Infof("Completed deleting object store %s", store.Name) + logger.Infof("Completed deleting object store %s", c.store.Name) return nil } // Check if the object store exists depending on either the deployment or the daemonset -func storeExists(context *clusterd.Context, store cephv1beta1.ObjectStore) (bool, error) { - _, err := context.Clientset.ExtensionsV1beta1().Deployments(store.Namespace).Get(instanceName(store), metav1.GetOptions{}) +func (c *config) storeExists() (bool, error) { + _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) if err == nil { // the deployment was found return true, nil @@ -192,7 +201,7 @@ func storeExists(context *clusterd.Context, store cephv1beta1.ObjectStore) (bool return false, err } - _, err = context.Clientset.ExtensionsV1beta1().DaemonSets(store.Namespace).Get(instanceName(store), metav1.GetOptions{}) + _, err = c.context.Clientset.ExtensionsV1beta1().DaemonSets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) if err == nil { // the daemonset was found return true, nil @@ -205,8 +214,8 @@ func storeExists(context *clusterd.Context, store cephv1beta1.ObjectStore) (bool return false, nil } -func createKeyring(context *clusterd.Context, store cephv1beta1.ObjectStore, ownerRefs []metav1.OwnerReference) error { - _, err := context.Clientset.CoreV1().Secrets(store.Namespace).Get(instanceName(store), metav1.GetOptions{}) +func (c *config) createKeyring() error { + _, err := c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) if err == nil { logger.Infof("the rgw keyring was already generated") return nil @@ -217,7 +226,7 @@ func createKeyring(context *clusterd.Context, store cephv1beta1.ObjectStore, own // create the keyring logger.Infof("generating rgw keyring") - keyring, err := createRGWKeyring(context, store.Namespace) + keyring, err := c.createRGWKeyring() if err != nil { return fmt.Errorf("failed to create keyring. %+v", err) } @@ -228,14 +237,14 @@ func createKeyring(context *clusterd.Context, store cephv1beta1.ObjectStore, own } secret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: instanceName(store), - Namespace: store.Namespace, + Name: c.instanceName(), + Namespace: c.store.Namespace, }, StringData: secrets, Type: k8sutil.RookType, } - k8sutil.SetOwnerRefs(context.Clientset, store.Namespace, &secret.ObjectMeta, ownerRefs) - _, err = context.Clientset.CoreV1().Secrets(store.Namespace).Create(secret) + k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &secret.ObjectMeta, c.ownerRefs) + _, err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Create(secret) if err != nil { return fmt.Errorf("failed to save rgw secrets. %+v", err) } @@ -243,17 +252,17 @@ func createKeyring(context *clusterd.Context, store cephv1beta1.ObjectStore, own return nil } -func instanceName(store cephv1beta1.ObjectStore) string { - return fmt.Sprintf("%s-%s", appName, store.Name) +func (c *config) instanceName() string { + return fmt.Sprintf("%s-%s", appName, c.store.Name) } // create a keyring for the rgw client with a limited set of privileges -func createRGWKeyring(context *clusterd.Context, clusterName string) (string, error) { +func (c *config) createRGWKeyring() (string, error) { username := "client.radosgw.gateway" access := []string{"osd", "allow rwx", "mon", "allow rw"} // get-or-create-key for the user account - key, err := client.AuthGetOrCreateKey(context, clusterName, username, access) + key, err := client.AuthGetOrCreateKey(c.context, c.store.Namespace, username, access) if err != nil { return "", fmt.Errorf("failed to get or create auth key for %store. %+v", username, err) } diff --git a/pkg/operator/ceph/object/rgw_test.go b/pkg/operator/ceph/object/rgw_test.go index ed948e655..c2627002a 100644 --- a/pkg/operator/ceph/object/rgw_test.go +++ b/pkg/operator/ceph/object/rgw_test.go @@ -49,43 +49,44 @@ func TestStartRGW(t *testing.T) { version := "v1.1.0" // start a basic cluster - err := CreateStore(context, store, version, false, []metav1.OwnerReference{}) + c := &config{context, store, version, cephv1beta1.CephVersionSpec{}, false, []metav1.OwnerReference{}} + err := c.createStore() assert.Nil(t, err) - validateStart(t, store, clientset, false) + validateStart(t, c, clientset, false) // starting again should update the pods with the new settings - store.Spec.Gateway.AllNodes = true - err = UpdateStore(context, store, version, false, []metav1.OwnerReference{}) + c.store.Spec.Gateway.AllNodes = true + err = c.updateStore() assert.Nil(t, err) - validateStart(t, store, clientset, true) + validateStart(t, c, clientset, true) } -func validateStart(t *testing.T, store cephv1beta1.ObjectStore, clientset *fake.Clientset, allNodes bool) { +func validateStart(t *testing.T, c *config, clientset *fake.Clientset, allNodes bool) { if !allNodes { - r, err := clientset.ExtensionsV1beta1().Deployments(store.Namespace).Get(instanceName(store), metav1.GetOptions{}) + r, err := clientset.ExtensionsV1beta1().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) assert.Nil(t, err) - assert.Equal(t, instanceName(store), r.Name) + assert.Equal(t, c.instanceName(), r.Name) - _, err = clientset.ExtensionsV1beta1().DaemonSets(store.Namespace).Get(instanceName(store), metav1.GetOptions{}) + _, err = clientset.ExtensionsV1beta1().DaemonSets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) assert.True(t, errors.IsNotFound(err)) } else { - r, err := clientset.ExtensionsV1beta1().DaemonSets(store.Namespace).Get(instanceName(store), metav1.GetOptions{}) + r, err := clientset.ExtensionsV1beta1().DaemonSets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) assert.Nil(t, err) - assert.Equal(t, instanceName(store), r.Name) + assert.Equal(t, c.instanceName(), r.Name) - _, err = clientset.ExtensionsV1beta1().Deployments(store.Namespace).Get(instanceName(store), metav1.GetOptions{}) + _, err = clientset.ExtensionsV1beta1().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) assert.True(t, errors.IsNotFound(err)) } - s, err := clientset.CoreV1().Services(store.Namespace).Get(instanceName(store), metav1.GetOptions{}) + s, err := clientset.CoreV1().Services(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) assert.Nil(t, err) - assert.Equal(t, instanceName(store), s.Name) + assert.Equal(t, c.instanceName(), s.Name) - secret, err := clientset.CoreV1().Secrets(store.Namespace).Get(instanceName(store), metav1.GetOptions{}) + secret, err := clientset.CoreV1().Secrets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) assert.Nil(t, err) - assert.Equal(t, instanceName(store), secret.Name) + assert.Equal(t, c.instanceName(), secret.Name) assert.Equal(t, 1, len(secret.StringData)) } @@ -115,7 +116,8 @@ func TestCreateObjectStore(t *testing.T) { context := &clusterd.Context{Executor: executor, Clientset: clientset} // create the pools - err := CreateStore(context, store, "1.2.3.4", false, []metav1.OwnerReference{}) + c := &config{context, store, "1.2.3.4", cephv1beta1.CephVersionSpec{}, false, []metav1.OwnerReference{}} + err := c.createStore() assert.Nil(t, err) } diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 76417e6aa..46a008edd 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -21,8 +21,6 @@ import ( "fmt" "path" - cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" - "github.com/rook/rook/pkg/clusterd" rgwdaemon "github.com/rook/rook/pkg/daemon/ceph/rgw" opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon" opspec "github.com/rook/rook/pkg/operator/ceph/spec" @@ -34,103 +32,103 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) -func startDeployment(context *clusterd.Context, store cephv1beta1.ObjectStore, version string, replicas int32, hostNetwork bool, ownerRefs []metav1.OwnerReference) error { +func (c *config) startDeployment() error { deployment := &extensions.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: instanceName(store), - Namespace: store.Namespace, + Name: c.instanceName(), + Namespace: c.store.Namespace, }, - Spec: extensions.DeploymentSpec{Template: makeRGWPodSpec(store, version, hostNetwork), Replicas: &replicas}, + Spec: extensions.DeploymentSpec{Template: c.makeRGWPodSpec(), Replicas: &c.store.Spec.Gateway.Instances}, } - k8sutil.SetOwnerRefs(context.Clientset, store.Namespace, &deployment.ObjectMeta, ownerRefs) - _, err := context.Clientset.ExtensionsV1beta1().Deployments(store.Namespace).Create(deployment) + k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &deployment.ObjectMeta, c.ownerRefs) + _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(c.store.Namespace).Create(deployment) return err } -func startDaemonset(context *clusterd.Context, store cephv1beta1.ObjectStore, version string, hostNetwork bool, ownerRefs []metav1.OwnerReference) error { +func (c *config) startDaemonset() error { daemonset := &extensions.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ - Name: instanceName(store), - Namespace: store.Namespace, + Name: c.instanceName(), + Namespace: c.store.Namespace, }, Spec: extensions.DaemonSetSpec{ UpdateStrategy: extensions.DaemonSetUpdateStrategy{ Type: extensions.RollingUpdateDaemonSetStrategyType, }, - Template: makeRGWPodSpec(store, version, hostNetwork), + Template: c.makeRGWPodSpec(), }, } - k8sutil.SetOwnerRefs(context.Clientset, store.Namespace, &daemonset.ObjectMeta, ownerRefs) + k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &daemonset.ObjectMeta, c.ownerRefs) - _, err := context.Clientset.ExtensionsV1beta1().DaemonSets(store.Namespace).Create(daemonset) + _, err := c.context.Clientset.ExtensionsV1beta1().DaemonSets(c.store.Namespace).Create(daemonset) return err } -func makeRGWPodSpec(store cephv1beta1.ObjectStore, version string, hostNetwork bool) v1.PodTemplateSpec { +func (c *config) makeRGWPodSpec() v1.PodTemplateSpec { podSpec := v1.PodSpec{ InitContainers: []v1.Container{ - makeConfigInitContainer(store, version), + c.makeConfigInitContainer(), }, Containers: []v1.Container{ - makeDaemonContainer(store, version), + c.makeDaemonContainer(), }, RestartPolicy: v1.RestartPolicyAlways, Volumes: opspec.PodVolumes(""), - HostNetwork: hostNetwork, + HostNetwork: c.hostNetwork, } - if hostNetwork { + if c.hostNetwork { podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet } // Set the ssl cert if specified - if store.Spec.Gateway.SSLCertificateRef != "" { + if c.store.Spec.Gateway.SSLCertificateRef != "" { certVol := v1.Volume{Name: certVolumeName, VolumeSource: v1.VolumeSource{Secret: &v1.SecretVolumeSource{ - SecretName: store.Spec.Gateway.SSLCertificateRef, + SecretName: c.store.Spec.Gateway.SSLCertificateRef, Items: []v1.KeyToPath{{Key: certKeyName, Path: certFilename}}, }}} podSpec.Volumes = append(podSpec.Volumes, certVol) } - store.Spec.Gateway.Placement.ApplyToPodSpec(&podSpec) + c.store.Spec.Gateway.Placement.ApplyToPodSpec(&podSpec) return v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Name: instanceName(store), - Labels: getLabels(store), + Name: c.instanceName(), + Labels: c.getLabels(), Annotations: map[string]string{}, }, Spec: podSpec, } } -func makeConfigInitContainer(store cephv1beta1.ObjectStore, version string) v1.Container { +func (c *config) makeConfigInitContainer() v1.Container { container := v1.Container{ Name: opspec.ConfigInitContainerName, - Image: k8sutil.MakeRookImage(version), + Image: k8sutil.MakeRookImage(c.rookVersion), Args: []string{ "ceph", "rgw", fmt.Sprintf("--config-dir=%s", k8sutil.DataDir), - fmt.Sprintf("--rgw-name=%s", store.Name), - fmt.Sprintf("--rgw-port=%d", store.Spec.Gateway.Port), - fmt.Sprintf("--rgw-secure-port=%d", store.Spec.Gateway.SecurePort), + fmt.Sprintf("--rgw-name=%s", c.store.Name), + fmt.Sprintf("--rgw-port=%d", c.store.Spec.Gateway.Port), + fmt.Sprintf("--rgw-secure-port=%d", c.store.Spec.Gateway.SecurePort), }, VolumeMounts: opspec.RookVolumeMounts(), Env: []v1.EnvVar{ - {Name: "ROOK_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: instanceName(store)}, Key: keyringName}}}, + {Name: "ROOK_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: c.instanceName()}, Key: keyringName}}}, k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), - opmon.ClusterNameEnvVar(store.Namespace), + opmon.ClusterNameEnvVar(c.store.Namespace), opmon.EndpointEnvVar(), opmon.SecretEnvVar(), k8sutil.ConfigOverrideEnvVar(), }, - Resources: store.Spec.Gateway.Resources, + Resources: c.store.Spec.Gateway.Resources, } - if store.Spec.Gateway.SSLCertificateRef != "" { + if c.store.Spec.Gateway.SSLCertificateRef != "" { // Add a volume mount for the ssl certificate mount := v1.VolumeMount{Name: certVolumeName, MountPath: certMountPath, ReadOnly: true} container.VolumeMounts = append(container.VolumeMounts, mount) @@ -143,12 +141,12 @@ func makeConfigInitContainer(store cephv1beta1.ObjectStore, version string) v1.C return container } -func makeDaemonContainer(store cephv1beta1.ObjectStore, version string) v1.Container { +func (c *config) makeDaemonContainer() v1.Container { // start the rgw daemon in the foreground container := v1.Container{ Name: "rgw", - Image: k8sutil.MakeRookImage(version), + Image: c.cephVersion.Image, Command: []string{ "radosgw", }, @@ -159,45 +157,45 @@ func makeDaemonContainer(store cephv1beta1.ObjectStore, version string) v1.Conta }, VolumeMounts: opspec.CephVolumeMounts(), Env: k8sutil.ClusterDaemonEnvVars(), - Resources: store.Spec.Gateway.Resources, + Resources: c.store.Spec.Gateway.Resources, } return container } -func startService(context *clusterd.Context, store cephv1beta1.ObjectStore, hostNetwork bool, ownerRefs []metav1.OwnerReference) (string, error) { - labels := getLabels(store) +func (c *config) startService() (string, error) { + labels := c.getLabels() svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: instanceName(store), - Namespace: store.Namespace, + Name: c.instanceName(), + Namespace: c.store.Namespace, Labels: labels, }, Spec: v1.ServiceSpec{ Selector: labels, }, } - k8sutil.SetOwnerRefs(context.Clientset, store.Namespace, &svc.ObjectMeta, ownerRefs) - if hostNetwork { + k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &svc.ObjectMeta, c.ownerRefs) + if c.hostNetwork { svc.Spec.ClusterIP = v1.ClusterIPNone } - addPort(svc, "http", store.Spec.Gateway.Port) - addPort(svc, "https", store.Spec.Gateway.SecurePort) + addPort(svc, "http", c.store.Spec.Gateway.Port) + addPort(svc, "https", c.store.Spec.Gateway.SecurePort) - svc, err := context.Clientset.CoreV1().Services(store.Namespace).Create(svc) + svc, err := c.context.Clientset.CoreV1().Services(c.store.Namespace).Create(svc) if err != nil { if !errors.IsAlreadyExists(err) { return "", fmt.Errorf("failed to create rgw service. %+v", err) } - svc, err = context.Clientset.CoreV1().Services(store.Namespace).Get(instanceName(store), metav1.GetOptions{}) + svc, err = c.context.Clientset.CoreV1().Services(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) if err != nil { return "", fmt.Errorf("failed to get existing service IP. %+v", err) } return svc.Spec.ClusterIP, nil } - logger.Infof("Gateway service running at %s:%d", svc.Spec.ClusterIP, store.Spec.Gateway.Port) + logger.Infof("Gateway service running at %s:%d", svc.Spec.ClusterIP, c.store.Spec.Gateway.Port) return svc.Spec.ClusterIP, nil } @@ -213,10 +211,10 @@ func addPort(service *v1.Service, name string, port int32) { }) } -func getLabels(store cephv1beta1.ObjectStore) map[string]string { +func (c *config) getLabels() map[string]string { return map[string]string{ k8sutil.AppAttr: appName, - k8sutil.ClusterAttr: store.Namespace, - "rook_object_store": store.Name, + k8sutil.ClusterAttr: c.store.Namespace, + "rook_object_store": c.store.Name, } } diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index 90f43a2c6..6d7ab0a53 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -20,6 +20,7 @@ import ( "fmt" "testing" + cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" "github.com/rook/rook/pkg/clusterd" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" "github.com/rook/rook/pkg/operator/k8sutil" @@ -40,7 +41,8 @@ func TestPodSpecs(t *testing.T) { }, } - s := makeRGWPodSpec(store, "rook/rook:myversion", true) + c := &config{store: store, rookVersion: "rook/rook:myversion", cephVersion: cephv1beta1.CephVersionSpec{Image: "ceph/ceph:v13.2.1"}, hostNetwork: true} + s := c.makeRGWPodSpec() assert.NotNil(t, s) //assert.Equal(t, instanceName(store), s.Name) assert.Equal(t, v1.RestartPolicyAlways, s.Spec.RestartPolicy) @@ -48,7 +50,7 @@ func TestPodSpecs(t *testing.T) { assert.Equal(t, "rook-data", s.Spec.Volumes[0].Name) assert.Equal(t, cephconfig.DefaultConfigMountName, s.Spec.Volumes[1].Name) - assert.Equal(t, instanceName(store), s.ObjectMeta.Name) + assert.Equal(t, c.instanceName(), s.ObjectMeta.Name) assert.Equal(t, appName, s.ObjectMeta.Labels["app"]) assert.Equal(t, store.Namespace, s.ObjectMeta.Labels["rook_cluster"]) assert.Equal(t, store.Name, s.ObjectMeta.Labels["rook_object_store"]) @@ -68,7 +70,7 @@ func TestPodSpecs(t *testing.T) { assert.Equal(t, "--rgw-secure-port=0", cont.Args[5]) cont = s.Spec.Containers[0] - assert.Equal(t, "rook/rook:myversion", cont.Image) + assert.Equal(t, "ceph/ceph:v13.2.1", cont.Image) assert.Equal(t, 2, len(cont.VolumeMounts)) assert.Equal(t, 1, len(cont.Command)) @@ -89,9 +91,10 @@ func TestSSLPodSpec(t *testing.T) { store.Spec.Gateway.SSLCertificateRef = "mycert" store.Spec.Gateway.SecurePort = 443 - s := makeRGWPodSpec(store, "v1.0", true) + c := &config{store: store, rookVersion: "v1.0", hostNetwork: true} + s := c.makeRGWPodSpec() assert.NotNil(t, s) - assert.Equal(t, instanceName(store), s.Name) + assert.Equal(t, c.instanceName(), s.Name) assert.Equal(t, 4, len(s.Spec.Volumes)) assert.Equal(t, certVolumeName, s.Spec.Volumes[3].Name) assert.True(t, s.Spec.HostNetwork) diff --git a/pkg/operator/k8sutil/deployment.go b/pkg/operator/k8sutil/deployment.go index 2f84adad5..09b574b3d 100644 --- a/pkg/operator/k8sutil/deployment.go +++ b/pkg/operator/k8sutil/deployment.go @@ -17,7 +17,6 @@ package k8sutil import ( "fmt" - "strings" "time" "github.com/rook/rook/pkg/clusterd" @@ -26,30 +25,25 @@ import ( "k8s.io/client-go/kubernetes" ) -// GetDeploymentVersion returns the version of the image running in the pod spec for the desired container -func GetDeploymentVersion(clientset kubernetes.Interface, namespace, name, container string) (string, error) { +// GetDeploymentImage returns the version of the image running in the pod spec for the desired container +func GetDeploymentImage(clientset kubernetes.Interface, namespace, name, container string) (string, error) { d, err := clientset.Extensions().Deployments(namespace).Get(name, metav1.GetOptions{}) if err != nil { return "", fmt.Errorf("failed to find deployment %s. %v", name, err) } - return GetDeploymentSpecVersion(clientset, *d, container) + return GetDeploymentSpecImage(clientset, *d, container) } -func GetDeploymentSpecVersion(clientset kubernetes.Interface, d extensions.Deployment, container string) (string, error) { +func GetDeploymentSpecImage(clientset kubernetes.Interface, d extensions.Deployment, container string) (string, error) { image, err := GetSpecContainerImage(d.Spec.Template.Spec, container) if err != nil { return "", err } - parts := strings.Split(image, ":") - if len(parts) != 2 { - return "", fmt.Errorf("unexpected version in image: %s", image) - } - - return parts[1], nil + return image, nil } -func WaitForDeploymentVersion(clientset kubernetes.Interface, namespace, label, container, desiredVersion string) error { +func WaitForDeploymentImage(clientset kubernetes.Interface, namespace, label, container, desiredImage string) error { sleepTime := 3 attempts := 30 @@ -61,25 +55,25 @@ func WaitForDeploymentVersion(clientset kubernetes.Interface, namespace, label, matches := 0 for _, d := range deployments.Items { - version, err := GetDeploymentSpecVersion(clientset, d, container) + image, err := GetDeploymentSpecImage(clientset, d, container) if err != nil { - logger.Infof("failed to get version for deployment %s. %+v", d.Name, err) + logger.Infof("failed to get image for deployment %s. %+v", d.Name, err) continue } - if version == desiredVersion { + if image == desiredImage { matches++ } } if matches == len(deployments.Items) && matches > 0 { - logger.Infof("all %d %s deployments are on version %s", matches, container, desiredVersion) + logger.Infof("all %d %s deployments are on image %s", matches, container, desiredImage) break } if len(deployments.Items) == 0 { logger.Infof("waiting for at least one deployment to start to see the version") } else { - logger.Infof("%d/%d %s deployments match version %s", matches, len(deployments.Items), container, desiredVersion) + logger.Infof("%d/%d %s deployments match image %s", matches, len(deployments.Items), container, desiredImage) } time.Sleep(time.Duration(sleepTime) * time.Second) } diff --git a/pkg/operator/k8sutil/job.go b/pkg/operator/k8sutil/job.go new file mode 100644 index 000000000..0ee8faa83 --- /dev/null +++ b/pkg/operator/k8sutil/job.go @@ -0,0 +1,109 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package k8sutil for Kubernetes helpers. +package k8sutil + +import ( + "fmt" + "time" + + batch "k8s.io/api/batch/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" +) + +func RunReplaceableJob(clientset kubernetes.Interface, job *batch.Job) error { + // check if the job was already created and what its status is + existingJob, err := clientset.Batch().Jobs(job.Namespace).Get(job.Name, metav1.GetOptions{}) + if err != nil && !errors.IsNotFound(err) { + logger.Warningf("failed to detect job %s. %+v", job.Name, err) + } else if err == nil { + // if the job is still running, allow it to continue to completion + if existingJob.Status.Active > 0 { + logger.Infof("Found previous job %s. Status=%+v", job.Name, existingJob.Status) + return nil + } + + // delete the job that already exists from a previous run + logger.Infof("Removing previous job %s to start a new one", job.Name) + err := DeleteBatchJob(clientset, job.Namespace, existingJob.Name, true) + if err != nil { + logger.Warningf("failed to remove job %s. %+v", job.Name, err) + } + } + + _, err = clientset.Batch().Jobs(job.Namespace).Create(job) + return err +} + +// WaitForJobCompletion waits for a job to reach the completed state. +// Assumes that only one pod needs to complete. +func WaitForJobCompletion(clientset kubernetes.Interface, job *batch.Job, timeout time.Duration) error { + logger.Infof("waiting for job %s to complete...", job.Name) + return wait.Poll(5*time.Second, timeout, func() (bool, error) { + job, err := clientset.Batch().Jobs(job.Namespace).Get(job.Name, metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("failed to detect job %s. %+v", job.Name, err) + } + + // if the job is still running, allow it to continue to completion + if job.Status.Active > 0 { + logger.Debugf("job is still running. Status=%+v", job.Status) + return false, nil + } + if job.Status.Failed > 0 { + return false, fmt.Errorf("job %s failed.", job.Name) + } + if job.Status.Succeeded > 0 { + return true, nil + } + logger.Debugf("job is still initializing") + return false, nil + }) +} + +func DeleteBatchJob(clientset kubernetes.Interface, namespace, name string, wait bool) error { + propagation := metav1.DeletePropagationForeground + gracePeriod := int64(0) + options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation} + + if err := clientset.Batch().Jobs(namespace).Delete(name, options); err != nil { + return fmt.Errorf("failed to remove previous provisioning job for node %s. %+v", name, err) + } + + if !wait { + return nil + } + + retries := 20 + sleepInterval := 2 * time.Second + for i := 0; i < retries; i++ { + _, err := clientset.Batch().Jobs(namespace).Get(name, metav1.GetOptions{}) + if err != nil && errors.IsNotFound(err) { + logger.Infof("batch job %s deleted", name) + return nil + } + + logger.Infof("batch job %s still exists", name) + time.Sleep(sleepInterval) + } + + logger.Warningf("gave up waiting for batch job %s to be deleted", name) + return nil +} diff --git a/pkg/operator/k8sutil/pod.go b/pkg/operator/k8sutil/pod.go index b5bb12da1..66f64f161 100644 --- a/pkg/operator/k8sutil/pod.go +++ b/pkg/operator/k8sutil/pod.go @@ -19,8 +19,10 @@ package k8sutil import ( "fmt" + "io" "os" "path" + "strings" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -186,6 +188,32 @@ func GetPodPhaseMap(pods *v1.PodList) map[v1.PodPhase][]string { return podPhaseMap } +// GetJobLog gets the logs for the pod. If there is more than one pod with the label selector, the logs from +// the first pod will be returned. +func GetPodLog(clientset kubernetes.Interface, namespace string, labelSelector string) (string, error) { + opts := metav1.ListOptions{ + LabelSelector: labelSelector, + } + pods, err := clientset.CoreV1().Pods(namespace).List(opts) + if err != nil { + return "", fmt.Errorf("failed to get version pod. %+v", err) + } + for _, pod := range pods.Items { + req := clientset.CoreV1().Pods(namespace).GetLogs(pod.Name, &v1.PodLogOptions{}) + readCloser, err := req.Stream() + if err != nil { + return "", fmt.Errorf("failed to read from stream. %+v", err) + } + + builder := &strings.Builder{} + defer readCloser.Close() + _, err = io.Copy(builder, readCloser) + return builder.String(), err + } + + return "", fmt.Errorf("did not find any pods with label %s", labelSelector) +} + // DeleteDeployment makes a best effort at deleting a deployment and its pods, then waits for them to be deleted func DeleteDeployment(clientset kubernetes.Interface, namespace, name string) error { logger.Debugf("removing %s deployment if it exists", name) diff --git a/tests/framework/clients/object.go b/tests/framework/clients/object.go index e42afcc74..41ba154e7 100644 --- a/tests/framework/clients/object.go +++ b/tests/framework/clients/object.go @@ -47,7 +47,8 @@ func (o *ObjectOperation) Create(namespace, storeName string, replicaCount int32 return err } - err := o.k8sh.WaitForLabeledPodsToRun(fmt.Sprintf("rook_object_store=%s", storeName), namespace) + // Starting an object store takes longer than the average operation, so add more retries + err := o.k8sh.WaitForLabeledPodsToRunWithRetries(fmt.Sprintf("rook_object_store=%s", storeName), namespace, 40) if err != nil { return fmt.Errorf("rgw did not start via crd. %+v", err) } diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index a4a2c803e..3babe1145 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -32,9 +32,7 @@ import ( "flag" cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" - rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/daemon/ceph/client" - "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/api/errors" @@ -43,11 +41,20 @@ import ( ) const ( + // test with the latest luminous build + luminousTestImage = "ceph/ceph:v12" + // test with the latest mimic build + mimicTestImage = "ceph/ceph:v13" rookOperatorCreatedCrd = "clusters.ceph.rook.io" helmChartName = "local/rook-ceph" helmDeployName = "rook-ceph" ) +var ( + LuminousVersion = cephv1beta1.CephVersionSpec{Image: luminousTestImage, Name: cephv1beta1.Luminous} + MimicVersion = cephv1beta1.CephVersionSpec{Image: mimicTestImage, Name: cephv1beta1.Mimic} +) + //CephInstaller wraps installing and uninstalling rook on a platform type CephInstaller struct { Manifests CephManifests @@ -56,6 +63,7 @@ type CephInstaller struct { helmHelper *utils.HelmHelper k8sVersion string changeHostnames bool + cephVersion cephv1beta1.CephVersionSpec T func() *testing.T } @@ -180,12 +188,13 @@ func (h *CephInstaller) CreateK8sRookToolbox(namespace string) (err error) { func (h *CephInstaller) CreateK8sRookCluster(namespace, systemNamespace string, storeType string) (err error) { return h.CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType, false, - cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, true /* startWithAllNodes */) + cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, true, /* startWithAllNodes */ + LuminousVersion) } //CreateK8sRookCluster creates rook cluster via kubectl func (h *CephInstaller) CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType string, - useAllDevices bool, mon cephv1beta1.MonSpec, startWithAllNodes bool) error { + useAllDevices bool, mon cephv1beta1.MonSpec, startWithAllNodes bool, cephVersion cephv1beta1.CephVersionSpec) error { dataDirHostPath, err := h.initTestDir(namespace) if err != nil { @@ -207,71 +216,11 @@ func (h *CephInstaller) CreateK8sRookClusterWithHostPathAndDevices(namespace, sy return fmt.Errorf("Failed to create cluster roles. %+v", err) } - // for increasing the test matrix, sometimes use the strongly typed client instead of the yaml - if h.k8shelper.VersionAtLeast("v1.10.0") { - logger.Infof("Starting Rook cluster with strongly typed clientset") - - clust := &cephv1beta1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - Namespace: namespace, - }, - Spec: cephv1beta1.ClusterSpec{ - ServiceAccount: "rook-ceph-cluster", - DataDirHostPath: dataDirHostPath, - Mon: cephv1beta1.MonSpec{ - Count: mon.Count, - AllowMultiplePerNode: mon.AllowMultiplePerNode, - }, - Storage: rookalpha.StorageScopeSpec{ - UseAllNodes: startWithAllNodes, - Selection: rookalpha.Selection{ - UseAllDevices: &useAllDevices, - }, - Config: map[string]string{ - config.StoreTypeKey: storeType, - config.DatabaseSizeMBKey: "1024", - config.JournalSizeMBKey: "1024", - }, - }, - }, - } - _, err := h.k8shelper.RookClientset.CephV1beta1().Clusters(namespace).Create(clust) - if err != nil { - return fmt.Errorf("failed to create cluster %s. %+v", clust.Name, err) - } - - if !startWithAllNodes { - // now that the cluster is created, let's get all the k8s nodes so we can update the cluster CRD with them - logger.Info("cluster was started without all nodes, will update cluster to add nodes now.") - nodeNames, err := h.GetNodeHostnames() - if err != nil { - return fmt.Errorf("failed to get k8s nodes to add to cluster CRD: %+v", err) - } - - // add all discovered k8s nodes to the cluster CRD - rookNodes := make([]rookalpha.Node, len(nodeNames)) - for i, hostname := range nodeNames { - rookNodes[i] = rookalpha.Node{Name: hostname} - } - clust, err = h.k8shelper.RookClientset.CephV1beta1().Clusters(namespace).Get(namespace, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("failed to get rook cluster to add nodes to it: %+v", err) - } - clust.Spec.Storage.Nodes = rookNodes - - // update the cluster CRD now - _, err = h.k8shelper.RookClientset.CephV1beta1().Clusters(namespace).Update(clust) - if err != nil { - return fmt.Errorf("failed to update cluster %s with nodes. %+v", clust.Name, err) - } - } - } else { - logger.Infof("Starting Rook Cluster with yaml") - rookCluster := h.Manifests.GetRookCluster(&ClusterSettings{namespace, storeType, dataDirHostPath, useAllDevices, mon.Count}) - if _, err := h.k8shelper.KubectlWithStdin(rookCluster, createFromStdinArgs...); err != nil { - return fmt.Errorf("Failed to create rook cluster : %v ", err) - } + logger.Infof("Starting Rook Cluster with yaml") + settings := &ClusterSettings{namespace, storeType, dataDirHostPath, useAllDevices, mon.Count, cephVersion} + rookCluster := h.Manifests.GetRookCluster(settings) + if _, err := h.k8shelper.KubectlWithStdin(rookCluster, createFromStdinArgs...); err != nil { + return fmt.Errorf("Failed to create rook cluster : %v ", err) } if err := h.k8shelper.WaitForPodCount("app=rook-ceph-mon", namespace, mon.Count); err != nil { @@ -372,7 +321,8 @@ func (h *CephInstaller) InstallRookOnK8sWithHostPathAndDevices(namespace, storeT //Create rook cluster err = h.CreateK8sRookClusterWithHostPathAndDevices(namespace, onamespace, storeType, - useDevices, cephv1beta1.MonSpec{Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode}, startWithAllNodes) + useDevices, cephv1beta1.MonSpec{Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode}, startWithAllNodes, + h.cephVersion) if err != nil { logger.Errorf("Rook cluster %s not installed, error -> %v", namespace, err) return false, err @@ -488,7 +438,7 @@ func (h *CephInstaller) GatherAllRookLogs(namespace, systemNamespace string, tes } // NewCephInstaller creates new instance of CephInstaller -func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, rookVersion string) *CephInstaller { +func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, rookVersion string, cephVersion cephv1beta1.CephVersionSpec) *CephInstaller { // All e2e tests should run ceph commands in the toolbox since we are not inside a container client.RunAllCephCommandsInToolbox = true @@ -502,11 +452,14 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, rook if err != nil { panic("failed to get kubectl client :" + err.Error()) } + logger.Infof("Rook Version: %s", rookVersion) + logger.Infof("Ceph Version: %s (%s)", cephVersion.Image, cephVersion.Name) h := &CephInstaller{ Manifests: NewCephManifests(rookVersion), k8shelper: k8shelp, helmHelper: utils.NewHelmHelper(Env.Helm), k8sVersion: version.String(), + cephVersion: cephVersion, changeHostnames: rookVersion != Version0_8 && k8shelp.VersionAtLeast("v1.11.0"), T: t, } diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index b85c1b1a4..b3d95b94b 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -21,6 +21,7 @@ import ( "strconv" "github.com/google/uuid" + cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" ) type CephManifests interface { @@ -45,6 +46,7 @@ type ClusterSettings struct { DataDirHostPath string UseAllDevices bool Mons int + CephVersion cephv1beta1.CephVersionSpec } //CephManifestsMaster wraps rook yaml definitions @@ -190,6 +192,7 @@ rules: resources: - secrets - pods + - pods/log - services - configmaps verbs: @@ -419,6 +422,9 @@ metadata: name: ` + settings.Namespace + ` namespace: ` + settings.Namespace + ` spec: + cephVersion: + image: ` + settings.CephVersion.Image + ` + allowUnsupported: ` + strconv.FormatBool(settings.CephVersion.AllowUnsupported) + ` serviceAccount: rook-ceph-cluster dataDirHostPath: ` + settings.DataDirHostPath + ` network: diff --git a/tests/framework/installer/installer.go b/tests/framework/installer/installer.go index 8f07344ed..423952199 100644 --- a/tests/framework/installer/installer.go +++ b/tests/framework/installer/installer.go @@ -32,7 +32,7 @@ const ( // Version tag for the latest manifests VersionMaster = "master" // Version tag for Rook v0.8 - Version0_8 = "v0.8.1" + Version0_8 = "v0.8.3" ) var ( diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index a51506528..cc557a605 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -457,11 +457,16 @@ func (k8sh *K8sHelper) IsPodWithLabelPresent(label string, namespace string) boo return count > 0 } +//WaitForLabelWaitForLabeledPodsToRun calls WaitForLabeledPodsToRunWithRetries with the default number of retries +func (k8sh *K8sHelper) WaitForLabeledPodsToRun(label, namespace string) error { + return k8sh.WaitForLabeledPodsToRunWithRetries(label, namespace, RetryLoop) +} + //WaitForLabeledPodsToRun returns true if a Pod is running status or goes to Running status within 90s else returns false -func (k8sh *K8sHelper) WaitForLabeledPodsToRun(label string, namespace string) error { +func (k8sh *K8sHelper) WaitForLabeledPodsToRunWithRetries(label string, namespace string, retries int) error { options := metav1.ListOptions{LabelSelector: label} var lastPod v1.Pod - for i := 0; i < RetryLoop; i++ { + for i := 0; i < retries; i++ { pods, err := k8sh.Clientset.CoreV1().Pods(namespace).List(options) lastStatus := "" running := 0 diff --git a/tests/integration/base_deploy_test.go b/tests/integration/base_deploy_test.go index 9bc752296..31d9a11b3 100644 --- a/tests/integration/base_deploy_test.go +++ b/tests/integration/base_deploy_test.go @@ -100,11 +100,11 @@ type TestCluster struct { } // StartTestCluster creates new instance of TestCluster struct -func StartTestCluster(t func() *testing.T, namespace, storeType string, helmInstalled, useDevices bool, mons int, rookVersion string) (*TestCluster, *utils.K8sHelper) { +func StartTestCluster(t func() *testing.T, namespace, storeType string, helmInstalled, useDevices bool, mons int, rookVersion string, cephVersion cephv1beta1.CephVersionSpec) (*TestCluster, *utils.K8sHelper) { kh, err := utils.CreateK8sHelper(t) require.NoError(t(), err) - i := installer.NewCephInstaller(t, kh.Clientset, rookVersion) + i := installer.NewCephInstaller(t, kh.Clientset, rookVersion, cephVersion) op := &TestCluster{i, kh, nil, t, namespace, storeType, helmInstalled, useDevices, mons} @@ -113,6 +113,7 @@ func StartTestCluster(t func() *testing.T, namespace, storeType string, helmInst assert.NoError(t(), kh.GetDockerImage("rook/ceph:"+rookVersion)) assert.NoError(t(), kh.GetDockerImage("rook/ceph-toolbox:"+rookVersion)) } + assert.NoError(t(), kh.GetDockerImage(cephVersion.Image)) op.Setup() return op, kh diff --git a/tests/integration/block_create_test.go b/tests/integration/block_create_test.go index 556ceaa25..b65163b7b 100644 --- a/tests/integration/block_create_test.go +++ b/tests/integration/block_create_test.go @@ -54,7 +54,7 @@ func (s *BlockCreateSuite) SetupSuite() { var err error s.namespace = "block-k8s-ns" - s.op, s.kh = StartTestCluster(s.T, s.namespace, "bluestore", false, false, 1, installer.VersionMaster) + s.op, s.kh = StartTestCluster(s.T, s.namespace, "bluestore", false, false, 1, installer.VersionMaster, installer.MimicVersion) s.testClient = clients.CreateTestClient(s.kh, s.op.installer.Manifests) initialBlocks, err := s.testClient.BlockClient.List(s.namespace) assert.Nil(s.T(), err) diff --git a/tests/integration/helm_test.go b/tests/integration/helm_test.go index b7b9c4e62..abae8f2ee 100644 --- a/tests/integration/helm_test.go +++ b/tests/integration/helm_test.go @@ -63,7 +63,7 @@ type HelmSuite struct { func (hs *HelmSuite) SetupSuite() { hs.namespace = "helm-ns" - hs.op, hs.kh = StartTestCluster(hs.T, hs.namespace, "bluestore", true, false, 1, installer.VersionMaster) + hs.op, hs.kh = StartTestCluster(hs.T, hs.namespace, "bluestore", true, false, 1, installer.VersionMaster, installer.MimicVersion) hs.helper = clients.CreateTestClient(hs.kh, hs.op.installer.Manifests) } diff --git a/tests/integration/multi_cluster_test.go b/tests/integration/multi_cluster_test.go index 2a87dd9b3..51d0d0709 100644 --- a/tests/integration/multi_cluster_test.go +++ b/tests/integration/multi_cluster_test.go @@ -135,7 +135,7 @@ func NewMCTestOperations(t func() *testing.T, namespace1 string, namespace2 stri kh, err := utils.CreateK8sHelper(t) require.NoError(t(), err) - i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster) + i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster, installer.LuminousVersion) op := &MCTestOperations{i, kh, t, namespace1, namespace2, installer.SystemNamespace(namespace1)} op.Setup() diff --git a/tests/integration/smoke_test.go b/tests/integration/smoke_test.go index 2b70ed9ab..cf95eaa2a 100644 --- a/tests/integration/smoke_test.go +++ b/tests/integration/smoke_test.go @@ -76,7 +76,7 @@ type SmokeSuite struct { func (suite *SmokeSuite) SetupSuite() { suite.namespace = "smoke-ns" useDevices := true - suite.op, suite.k8sh = StartTestCluster(suite.T, suite.namespace, "bluestore", false, useDevices, 3, installer.VersionMaster) + suite.op, suite.k8sh = StartTestCluster(suite.T, suite.namespace, "bluestore", false, useDevices, 3, installer.VersionMaster, installer.MimicVersion) suite.helper = clients.CreateTestClient(suite.k8sh, suite.op.installer.Manifests) } @@ -135,7 +135,10 @@ func checkOrderedSubstrings(t *testing.T, input string, substrings ...string) { } original := input for i, substring := range substrings { - assert.Contains(t, input, substring, fmt.Sprintf("missing substring %d. original=%s", i, original)) + if !strings.Contains(input, substring) { + assert.Fail(t, fmt.Sprintf("missing substring %d. original=%s", i, original)) + return + } index := strings.Index(input, substring) input = input[index+len(substring):] } diff --git a/tests/integration/upgrade_test.go b/tests/integration/upgrade_test.go index 3dc518201..5a837ded2 100644 --- a/tests/integration/upgrade_test.go +++ b/tests/integration/upgrade_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/installer" @@ -61,7 +62,7 @@ func (s *UpgradeSuite) SetupSuite() { s.namespace = "upgrade-ns" useDevices := true - s.op, s.k8sh = StartTestCluster(s.T, s.namespace, "bluestore", false, useDevices, 3, installer.Version0_8) + s.op, s.k8sh = StartTestCluster(s.T, s.namespace, "bluestore", false, useDevices, 3, installer.Version0_8, installer.LuminousVersion) s.helper = clients.CreateTestClient(s.k8sh, s.op.installer.Manifests) } @@ -94,9 +95,9 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { // verify that we're actually running 0.8 before the upgrade operatorContainer := "rook-ceph-operator" - version, err := k8sutil.GetDeploymentVersion(s.k8sh.Clientset, systemNamespace, operatorContainer, operatorContainer) + version, err := k8sutil.GetDeploymentImage(s.k8sh.Clientset, systemNamespace, operatorContainer, operatorContainer) assert.Nil(s.T(), err) - assert.Equal(s.T(), "v0.8.1", version) + assert.Equal(s.T(), "rook/ceph:v0.8.3", version) message := "my simple message" preFilename := "pre-upgrade-file" @@ -109,22 +110,22 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { require.Nil(s.T(), s.k8sh.SetDeploymentVersion(systemNamespace, operatorContainer, operatorContainer, installer.VersionMaster)) // verify that the operator spec is updated - version, err = k8sutil.GetDeploymentVersion(s.k8sh.Clientset, systemNamespace, operatorContainer, operatorContainer) + version, err = k8sutil.GetDeploymentImage(s.k8sh.Clientset, systemNamespace, operatorContainer, operatorContainer) assert.Nil(s.T(), err) - assert.Equal(s.T(), installer.VersionMaster, version) + assert.Equal(s.T(), "rook/ceph:"+installer.VersionMaster, version) // wait for the legacy mon replicasets to be deleted err = s.waitForLegacyMonReplicaSetDeletion() require.Nil(s.T(), err) // wait for the mon pods to be running - err = k8sutil.WaitForDeploymentVersion(s.k8sh.Clientset, s.namespace, "app=rook-ceph-mon", "rook-ceph-mon", installer.VersionMaster) + err = k8sutil.WaitForDeploymentImage(s.k8sh.Clientset, s.namespace, "app=rook-ceph-mon", "mon", cephv1beta1.DefaultLuminousImage) require.Nil(s.T(), err) s.k8sh.WaitForLabeledPodsToRun("app=rook-ceph-mon", s.namespace) // wait for the osd pods to be updated - err = k8sutil.WaitForDeploymentVersion(s.k8sh.Clientset, s.namespace, "app=rook-ceph-osd", "rook-ceph-osd", installer.VersionMaster) + err = k8sutil.WaitForDeploymentImage(s.k8sh.Clientset, s.namespace, "app=rook-ceph-osd", "osd", cephv1beta1.DefaultLuminousImage) require.Nil(s.T(), err) s.k8sh.WaitForLabeledPodsToRun("app=rook-ceph-osd", s.namespace) @@ -135,15 +136,15 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { // Test writing and reading from the pod with cephfs mounted that was created before the upgrade. postFilename := "post-upgrade-file" - assert.Nil(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, preFilename, message, 3)) - assert.Nil(s.T(), s.k8sh.WriteToPodRetry(s.namespace, filePodName, postFilename, message, 3)) - assert.Nil(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, postFilename, message, 3)) + assert.Nil(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, preFilename, message, 5)) + assert.Nil(s.T(), s.k8sh.WriteToPodRetry(s.namespace, filePodName, postFilename, message, 5)) + assert.Nil(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, postFilename, message, 5)) // Test writing and reading from the pod with rbd mounted that was created before the upgrade. // There is some unreliability right after the upgrade when there is only one osd, so we will retry if needed - assert.Nil(s.T(), s.k8sh.ReadFromPodRetry("", podName, preFilename, message, 3)) - assert.Nil(s.T(), s.k8sh.WriteToPodRetry("", podName, postFilename, message, 3)) - assert.Nil(s.T(), s.k8sh.ReadFromPodRetry("", podName, postFilename, message, 3)) + assert.Nil(s.T(), s.k8sh.ReadFromPodRetry("", podName, preFilename, message, 5)) + assert.Nil(s.T(), s.k8sh.WriteToPodRetry("", podName, postFilename, message, 5)) + assert.Nil(s.T(), s.k8sh.ReadFromPodRetry("", podName, postFilename, message, 5)) } func (s *UpgradeSuite) waitForLegacyMonReplicaSetDeletion() error { diff --git a/tests/integration/zblock_mount_unmount_test.go b/tests/integration/zblock_mount_unmount_test.go index edd46961c..ed79ba780 100644 --- a/tests/integration/zblock_mount_unmount_test.go +++ b/tests/integration/zblock_mount_unmount_test.go @@ -86,7 +86,7 @@ func (s *BlockMountUnMountSuite) SetupSuite() { s.pvcNameRWX = "block-persistent-rwx" useHelm := false useDevices := true - s.op, s.kh = StartTestCluster(s.T, s.namespace, "filestore", useHelm, useDevices, 1, installer.VersionMaster) + s.op, s.kh = StartTestCluster(s.T, s.namespace, "filestore", useHelm, useDevices, 1, installer.VersionMaster, installer.LuminousVersion) s.testClient = clients.CreateTestClient(s.kh, s.op.installer.Manifests) s.bc = s.testClient.BlockClient } diff --git a/tests/longhaul/base_block_test.go b/tests/longhaul/base_block_test.go index 16681988c..83bfd0b73 100644 --- a/tests/longhaul/base_block_test.go +++ b/tests/longhaul/base_block_test.go @@ -147,7 +147,7 @@ func StartLoadTestCluster(t func() *testing.T, namespace string) (LoadTestCluste kh, err := utils.CreateK8sHelper(t) require.NoError(t(), err) - i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster) + i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster, cephv1beta1.CephVersionSpec{Image: "ceph/ceph:v12.2.7", Name: "luminous"}) op := LoadTestCluster{i, kh, nil, t, namespace} op.Setup()