Merge pull request #2187 from travisn/decouple-ceph-version
Decouple ceph version from rook to support luminous and mimic, and testing with nautilus
This commit is contained in:
71 files changed
+1531
-554
No files matched your search
@@ -17,6 +17,9 @@ metadata:
|
||||
name: rook-ceph
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
cephVersion:
|
||||
# see the "Cluster Settings" section below for more details on which image of ceph to run
|
||||
image: ceph/ceph:v13.2.2-20181023
|
||||
dataDirHostPath: /var/lib/rook
|
||||
serviceAccount: rook-ceph-cluster
|
||||
storage:
|
||||
@@ -36,6 +39,13 @@ Settings can be specified at the global level to apply to the cluster as a whole
|
||||
|
||||
### Cluster Settings
|
||||
|
||||
- `cephVersion`: The version information for launching the ceph daemons.
|
||||
- `image`: The image used for running the ceph daemons. For example, `ceph/ceph:v12.2.9-20181026` or `ceph/ceph:v13.2.2-20181023`.
|
||||
For the latest ceph images, see the [Ceph DockerHub](https://hub.docker.com/r/ceph/ceph/tags/).
|
||||
To ensure a consistent version of the image is running across all nodes in the cluster, it is recommended to use a very specific image version.
|
||||
Tags also exist that would give the latest version, but they are only recommended for test environments. For example, the tag `v13` will be updated each time a new mimic build is released.
|
||||
Using the `v13` or similar tag is not recommended in production because it may lead to inconsistent versions of the image running across different nodes in the cluster.
|
||||
- `allowUnsupported`: If `true`, allow an unsupported major version of the Ceph release. Currently only `luminous` and `mimic` are supported, so `nautilus` would require this to be set to `true`. Should be set to `false` in production.
|
||||
- `dataDirHostPath`: The path on the host ([hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath)) where config and data should be stored for each of the services. If the directory does not exist, it will be created. Because this directory persists on the host, it will remain after pods are deleted.
|
||||
- On **Minikube** environments, use `/data/rook`. Minikube boots into a tmpfs but it provides some [directories](https://github.com/kubernetes/minikube/blob/master/docs/persistent_volumes.md) where files can be persisted across reboots. Using one of these directories will ensure that Rook's data and configuration files are persisted and that enough storage space is available.
|
||||
- **WARNING**: For test scenarios, if you delete a cluster and start a new cluster on the same hosts, the path used by `dataDirHostPath` must be deleted. Otherwise, stale keys and other config will remain from the previous cluster and the new mons will fail to start.
|
||||
@@ -155,6 +165,8 @@ metadata:
|
||||
name: rook-ceph
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
cephVersion:
|
||||
image: ceph/ceph:v13.2.2-20181023
|
||||
dataDirHostPath: /var/lib/rook
|
||||
serviceAccount: rook-ceph-cluster
|
||||
network:
|
||||
@@ -184,6 +196,8 @@ metadata:
|
||||
name: rook-ceph
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
cephVersion:
|
||||
image: ceph/ceph:v13.2.2-20181023
|
||||
dataDirHostPath: /var/lib/rook
|
||||
serviceAccount: rook-ceph-cluster
|
||||
network:
|
||||
@@ -225,6 +239,8 @@ metadata:
|
||||
name: rook-ceph
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
cephVersion:
|
||||
image: ceph/ceph:v13.2.2-20181023
|
||||
dataDirHostPath: /var/lib/rook
|
||||
serviceAccount: rook-ceph-cluster
|
||||
network:
|
||||
@@ -260,6 +276,8 @@ metadata:
|
||||
name: rook-ceph
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
cephVersion:
|
||||
image: ceph/ceph:v13.2.2-20181023
|
||||
dataDirHostPath: /var/lib/rook
|
||||
serviceAccount: rook-ceph-cluster
|
||||
network:
|
||||
@@ -303,6 +321,8 @@ metadata:
|
||||
name: rook-ceph
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
cephVersion:
|
||||
image: ceph/ceph:v13.2.2-20181023
|
||||
dataDirHostPath: /var/lib/rook
|
||||
serviceAccount: rook-ceph-cluster
|
||||
# cluster level resource requests/limits configuration
|
||||
|
||||
@@ -14,7 +14,7 @@ and more. Rook makes it simple to enable the dashboard.
|
||||
|
||||
## Enable the Dashboard
|
||||
|
||||
The [dashboard](http://docs.ceph.com/docs/luminous/mgr/dashboard/) can be enabled with settings in the cluster CRD. The cluster CRD must have the dashboard `enabled` setting set to `true`.
|
||||
The [dashboard](http://docs.ceph.com/docs/mimic/mgr/dashboard/) can be enabled with settings in the cluster CRD. The cluster CRD must have the dashboard `enabled` setting set to `true`.
|
||||
This is the default setting in the example manifests.
|
||||
```yaml
|
||||
spec:
|
||||
@@ -22,19 +22,32 @@ This is the default setting in the example manifests.
|
||||
enabled: true
|
||||
```
|
||||
|
||||
The Rook operator will enable the ceph-mgr dashboard module to listen on the default port 7000.
|
||||
A K8s service will also be created to expose that port inside the cluster.
|
||||
The Rook operator will enable the ceph-mgr dashboard module. A K8s service will be created to expose that port inside the cluster. The ports enabled by Rook will depend
|
||||
on the version of Ceph that is running:
|
||||
- Luminous: Port 7000 on http
|
||||
- Mimic and newer: Port 8443 on https
|
||||
|
||||
This example shows that port 8443 was configured for Mimic or newer.
|
||||
```bash
|
||||
kubectl -n rook-ceph get service
|
||||
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
|
||||
rook-ceph-mgr ClusterIP 10.108.111.192 <none> 9283/TCP 3h
|
||||
rook-ceph-mgr-dashboard ClusterIP 10.110.113.240 <none> 7000/TCP 3h
|
||||
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
|
||||
rook-ceph-mgr ClusterIP 10.108.111.192 <none> 9283/TCP 3h
|
||||
rook-ceph-mgr-dashboard ClusterIP 10.110.113.240 <none> 8443/TCP 3h
|
||||
```
|
||||
|
||||
The first service is for reporting the [Prometheus metrics](monitoring.md), while the latter service is for the dashboard.
|
||||
If you are on a node in the cluster, you will be able to connect to the dashboard by using either the
|
||||
DNS name of the service at `http://rook-ceph-mgr-dashboard:7000` or by connecting to the cluster IP,
|
||||
in this example at `http://10.110.113.240:7000`.
|
||||
DNS name of the service at `https://rook-ceph-mgr-dashboard-https:8443` or by connecting to the cluster IP,
|
||||
in this example at `https://10.110.113.240:8443`.
|
||||
|
||||
### Credentials
|
||||
|
||||
After you connect to the dashboard you will need to login for secure access. Rook creates a default user named
|
||||
`admin` and generates a secret called `rook-ceph-dashboard-admin-password` in the namespace where rook is running.
|
||||
To retrieve the generated password, you can run the following:
|
||||
```
|
||||
kubectl -n rook-ceph get secret rook-ceph-dashboard-password -o yaml | grep "password:" | awk '{print $2}' | base64 --decode
|
||||
```
|
||||
|
||||
## Viewing the Dashboard External to the Cluster
|
||||
|
||||
@@ -46,12 +59,14 @@ You can use an [Ingress Controller](https://kubernetes.io/docs/concepts/services
|
||||
NodePort, LoadBalancer, or ExternalIPs.
|
||||
|
||||
The simplest way to expose the service in minikube or similar environment is using the NodePort to open a port on the
|
||||
VM that can be accessed by the host. To create a service with the NodePort, save this yaml as `dashboard-external.yaml`:
|
||||
VM that can be accessed by the host. To create a service with the NodePort, save this yaml as `dashboard-external-https.yaml`.
|
||||
(For Luminous you will need to set the `port` and `targetPort` to 7000 and connect via `http`.)
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: rook-ceph-mgr-dashboard-external
|
||||
name: rook-ceph-mgr-dashboard-external-https
|
||||
namespace: rook-ceph
|
||||
labels:
|
||||
app: rook-ceph-mgr
|
||||
@@ -59,9 +74,9 @@ metadata:
|
||||
spec:
|
||||
ports:
|
||||
- name: dashboard
|
||||
port: 7000
|
||||
port: 8443
|
||||
protocol: TCP
|
||||
targetPort: 7000
|
||||
targetPort: 8443
|
||||
selector:
|
||||
app: rook-ceph-mgr
|
||||
rook_cluster: rook-ceph
|
||||
@@ -77,12 +92,12 @@ $ kubectl create -f dashboard-external.yaml
|
||||
You will see the new service `rook-ceph-mgr-dashboard-external` created:
|
||||
```bash
|
||||
$ kubectl -n rook-ceph get service
|
||||
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
|
||||
rook-ceph-mgr ClusterIP 10.108.111.192 <none> 9283/TCP 4h
|
||||
rook-ceph-mgr-dashboard ClusterIP 10.110.113.240 <none> 7000/TCP 4h
|
||||
rook-ceph-mgr-dashboard-external NodePort 10.101.209.6 <none> 7000:31176/TCP 4h
|
||||
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
|
||||
rook-ceph-mgr ClusterIP 10.108.111.192 <none> 9283/TCP 4h
|
||||
rook-ceph-mgr-dashboard ClusterIP 10.110.113.240 <none> 8443/TCP 4h
|
||||
rook-ceph-mgr-dashboard-external-https NodePort 10.101.209.6 <none> 8443:31176/TCP 4h
|
||||
```
|
||||
|
||||
In this example, port `31176` will be opened to expose port `7000` from the ceph-mgr pod. Find the ip address
|
||||
of the VM. If using minikube, you can `minikube ssh` to the machine and `ifconfig` to find the ip address.
|
||||
Now you can enter the URL in your browser such as `http://192.168.99.110:31176` and the dashboard will appear.
|
||||
In this example, port `31176` will be opened to expose port `8443` from the ceph-mgr pod. Find the ip address
|
||||
of the VM. If using minikube, you can run `minikube ip` to find the ip address.
|
||||
Now you can enter the URL in your browser such as `https://192.168.99.110:31176` and the dashboard will appear.
|
||||
@@ -117,6 +117,9 @@ metadata:
|
||||
name: rook-ceph
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
cephVersion:
|
||||
# For the latest ceph images, see https://hub.docker.com/r/ceph/ceph/tags
|
||||
image: ceph/ceph:v13.2.2-20181023
|
||||
dataDirHostPath: /var/lib/rook
|
||||
dashboard:
|
||||
enabled: true
|
||||
|
||||
@@ -4,8 +4,10 @@
|
||||
|
||||
## Notable Features
|
||||
|
||||
- Different versions of Ceph can be orchestrated by Rook. Both Luminous and Mimic are now supported, with Nautilus coming soon.
|
||||
The version of Ceph is specified in the cluster CRD with the cephVersion.image property. For example, to run Mimic you could use image `ceph/ceph:v13.2.2-20181023`
|
||||
or any other image found on the [Ceph DockerHub](https://hub.docker.com/r/ceph/ceph/tags).
|
||||
- The `fsType` default for StorageClass examples are now using XFS to bring it in line with Ceph recommendations.
|
||||
- Ceph is updated from Luminous 12.2.5 to 12.2.7.
|
||||
- Ceph OSDs will be automatically updated by the operator when there is a change to the operator version or when the OSD configuration changes. See the [OSD upgrade notes](Documentation/upgrade-patch.md#object-storage-daemons-osds).
|
||||
- Rook Ceph block storage provisioner can now correctly create erasure coded block images. See [Advanced Example: Erasure Coded Block Storage](Documentation/block.md#advanced-example-erasure-coded-block-storage) for an example usage.
|
||||
- [Network File System (NFS)](https://github.com/nfs-ganesha/nfs-ganesha/wiki) is now supported by Rook with a new operator to deploy and manage this widely used server. NFS servers can be automatically deployed by creating an instance of the new `nfsservers.nfs.rook.io` custom resource. See the [NFS server user guide](Documentation/nfs.md) to get started with NFS.
|
||||
@@ -19,7 +21,7 @@
|
||||
## Breaking Changes
|
||||
- Ceph mons are [named consistently](https://github.com/rook/rook/issues/1751) with other daemons with the letters a, b, c, etc.
|
||||
- Ceph mons are now created with Deployments instead of ReplicaSets to improve the upgrade implementation.
|
||||
- Ceph mon, mgr, and mds container names in pods have changed with the refactors to initialize the
|
||||
- Ceph mon, mgr, mds, and rgw container names in pods have changed with the refactors to initialize the
|
||||
daemon environments via pod **InitContainers** and run the Ceph daemons directly from the
|
||||
container entrypoint.
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ rules:
|
||||
resources:
|
||||
- secrets
|
||||
- pods
|
||||
- pods/log
|
||||
- services
|
||||
- configmaps
|
||||
verbs:
|
||||
|
||||
@@ -55,6 +55,16 @@ metadata:
|
||||
name: rook-ceph
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
cephVersion:
|
||||
# The container image used to launch the Ceph daemon pods (mon, mgr, osd, mds, rgw).
|
||||
# v12 is luminous, v13 is mimic, and v14 is nautilus.
|
||||
# RECOMMENDATION: In production, use a specific version tag instead of the general v13 flag, which pulls the latest release and could result in different
|
||||
# versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/.
|
||||
image: ceph/ceph:v13
|
||||
# Whether to allow unsupported versions of Ceph. Currently only luminous and mimic are supported.
|
||||
# After nautilus is released, Rook will be updated to support nautilus.
|
||||
# Do not set to true in production.
|
||||
allowUnsupported: false
|
||||
# The path on the host where configuration files will be persisted. If not specified, a kubernetes emptyDir will be created (not recommended).
|
||||
# Important: if you reinstall the cluster, make sure you delete this directory from each host or else the mons will fail to start on the new cluster.
|
||||
# In Minikube, the '/data' directory is configured to persist across reboots. Use "/data/rook" in Minikube environment.
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: rook-ceph-mgr-dashboard-external
|
||||
name: rook-ceph-mgr-dashboard-external-http
|
||||
namespace: rook-ceph
|
||||
labels:
|
||||
app: rook-ceph-mgr
|
||||
@@ -0,0 +1,19 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: rook-ceph-mgr-dashboard-external-https
|
||||
namespace: rook-ceph
|
||||
labels:
|
||||
app: rook-ceph-mgr
|
||||
rook_cluster: rook-ceph
|
||||
spec:
|
||||
ports:
|
||||
- name: dashboard
|
||||
port: 8443
|
||||
protocol: TCP
|
||||
targetPort: 8443
|
||||
selector:
|
||||
app: rook-ceph-mgr
|
||||
rook_cluster: rook-ceph
|
||||
sessionAffinity: None
|
||||
type: NodePort
|
||||
@@ -18,6 +18,51 @@ spec:
|
||||
- rcc
|
||||
scope: Namespaced
|
||||
version: v1beta1
|
||||
validation:
|
||||
openAPIV3Schema:
|
||||
properties:
|
||||
spec:
|
||||
properties:
|
||||
cephVersion:
|
||||
properties:
|
||||
allowUnsupported:
|
||||
type: boolean
|
||||
image:
|
||||
type: string
|
||||
name:
|
||||
pattern: ^(luminous|mimic|nautilus)$
|
||||
type: string
|
||||
dashboard:
|
||||
properties:
|
||||
enabled:
|
||||
type: boolean
|
||||
dataDirHostPath:
|
||||
pattern: ^/(\S+)
|
||||
type: string
|
||||
mon:
|
||||
properties:
|
||||
allowMultiplePerNode:
|
||||
type: boolean
|
||||
count:
|
||||
maximum: 9
|
||||
minimum: 1
|
||||
type: integer
|
||||
required:
|
||||
- count
|
||||
network:
|
||||
properties:
|
||||
hostNetwork:
|
||||
type: boolean
|
||||
storage:
|
||||
properties:
|
||||
nodes:
|
||||
items: {}
|
||||
type: array
|
||||
useAllDevices: {}
|
||||
useAllNodes:
|
||||
type: boolean
|
||||
required:
|
||||
- mon
|
||||
---
|
||||
apiVersion: apiextensions.k8s.io/v1beta1
|
||||
kind: CustomResourceDefinition
|
||||
@@ -97,6 +142,7 @@ rules:
|
||||
resources:
|
||||
- secrets
|
||||
- pods
|
||||
- pods/log
|
||||
- services
|
||||
- configmaps
|
||||
verbs:
|
||||
|
||||
+22
-7
@@ -44,6 +44,11 @@ var osdConfigCmd = &cobra.Command{
|
||||
Short: "Updates ceph.conf for the osd",
|
||||
Hidden: true,
|
||||
}
|
||||
var copyBinariesCmd = &cobra.Command{
|
||||
Use: "copybins",
|
||||
Short: "Copies rook binaries for use by a ceph container",
|
||||
Hidden: true,
|
||||
}
|
||||
var provisionCmd = &cobra.Command{
|
||||
Use: "provision",
|
||||
Short: "Generates osd config and prepares an osd for runtime",
|
||||
@@ -77,7 +82,9 @@ func addOSDFlags(command *cobra.Command) {
|
||||
|
||||
// flags for generating the osd config
|
||||
osdConfigCmd.Flags().IntVar(&osdID, "osd-id", -1, "osd id for which to generate config")
|
||||
osdConfigCmd.Flags().StringVar(©BinariesPath, "copy-binaries-path", "", "If specified, copy the rook binaries to this path for use by the daemon container")
|
||||
|
||||
// flag for copying the rook binaries for use by a ceph container
|
||||
copyBinariesCmd.Flags().StringVar(©BinariesPath, "path", "", "Copy the rook binaries to this path for use by a ceph container")
|
||||
|
||||
// flags for running filestore on a device
|
||||
filestoreDeviceCmd.Flags().StringVar(&mountSourcePath, "source-path", "", "the source path of the device to mount")
|
||||
@@ -85,6 +92,7 @@ func addOSDFlags(command *cobra.Command) {
|
||||
|
||||
// add the subcommands to the parent osd command
|
||||
osdCmd.AddCommand(osdConfigCmd)
|
||||
osdCmd.AddCommand(copyBinariesCmd)
|
||||
osdCmd.AddCommand(provisionCmd)
|
||||
osdCmd.AddCommand(filestoreDeviceCmd)
|
||||
}
|
||||
@@ -106,10 +114,12 @@ func init() {
|
||||
addCephFlags(osdCmd)
|
||||
flags.SetFlagsFromEnv(osdCmd.Flags(), rook.RookEnvVarPrefix)
|
||||
flags.SetFlagsFromEnv(osdConfigCmd.Flags(), rook.RookEnvVarPrefix)
|
||||
flags.SetFlagsFromEnv(copyBinariesCmd.Flags(), rook.RookEnvVarPrefix)
|
||||
flags.SetFlagsFromEnv(provisionCmd.Flags(), rook.RookEnvVarPrefix)
|
||||
flags.SetFlagsFromEnv(filestoreDeviceCmd.Flags(), rook.RookEnvVarPrefix)
|
||||
|
||||
osdConfigCmd.RunE = writeOSDConfig
|
||||
copyBinariesCmd.RunE = copyRookBinaries
|
||||
provisionCmd.RunE = prepareOSD
|
||||
filestoreDeviceCmd.RunE = runFilestoreDeviceOSD
|
||||
}
|
||||
@@ -174,12 +184,17 @@ func writeOSDConfig(cmd *cobra.Command, args []string) error {
|
||||
if err := osddaemon.WriteConfigFile(context, &clusterInfo, kv, osdID, cfg.storeConfig, cfg.nodeName, crushLocation); err != nil {
|
||||
logger.Errorf("failed to write osd config file. %+v", err)
|
||||
}
|
||||
if copyBinariesPath != "" {
|
||||
if err := osddaemon.CopyBinariesForDaemon(copyBinariesPath); err != nil {
|
||||
logger.Errorf("failed to copy rook binaries for filestore device. %+v", err)
|
||||
} else {
|
||||
logger.Infof("successfully copied rook binaries")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyRookBinaries(cmd *cobra.Command, args []string) error {
|
||||
if err := flags.VerifyRequiredFlags(copyBinariesCmd, []string{"path"}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := osddaemon.CopyBinariesForDaemon(copyBinariesPath); err != nil {
|
||||
logger.Errorf("failed to copy rook binaries for filestore device. %+v", err)
|
||||
} else {
|
||||
logger.Infof("successfully copied rook binaries")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -44,10 +44,10 @@ spec:
|
||||
|
||||
### Ceph Version
|
||||
|
||||
The Ceph version is defined as the property `cephImage` in the Cluster CRD. All Ceph daemon containers launched by the Rook operator will use this image, including the mon, mgr,
|
||||
The Ceph version is defined under the property `cephVersion` in the Cluster CRD. All Ceph daemon containers launched by the Rook operator will use this image, including the mon, mgr,
|
||||
osd, rgw, and mds pods. The significance of this approach is that the Rook binary is not included in the daemon containers. All initialization performed by Rook to generate the Ceph config and prepare the daemons must be completed in an [init container](https://github.com/rook/rook/issues/2003). Once the Rook init containers complete their execution, the daemon container will run the Ceph image. The daemon container will no longer have Rook running.
|
||||
|
||||
In the following Cluster CRD example, the Ceph version is Mimic 13.2.1.
|
||||
In the following Cluster CRD example, the Ceph version is Mimic `13.2.2` built on 23 Oct 2018.
|
||||
|
||||
```yaml
|
||||
apiVersion: ceph.rook.io/v1beta1
|
||||
@@ -56,14 +56,15 @@ metadata:
|
||||
name: rook-ceph
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
cephImage: ceph/ceph:v13.2.1
|
||||
cephVersion:
|
||||
image: ceph/ceph:v13.2.2-20181023
|
||||
```
|
||||
|
||||
### Operator Requirements
|
||||
|
||||
The operator needs to run the Ceph client tools to manage the cluster. For example, the `ceph` tool is needed for general Ceph configuration and status, while `radosgw-admin` is required for managing an object store. Therefore, all the necessary client tools will still be included in the Rook image.
|
||||
|
||||
The client tools are tested by the Ceph team to be backward and forward compatible by two versions. Therefore, the operator can support a version of Ceph up to two versions older than the client tools it contains.
|
||||
The client tools are tested by the Ceph team to be backward and forward compatible by two versions. This means the operator can support a version of Ceph up to two versions older than the client tools it contains.
|
||||
With each Rook release, the tools will be included from the latest release of Ceph. For example, in 0.9 Rook will likely include the Mimic tools. Upgrades would be supported from Luminous to Mimic.
|
||||
Rook 0.9 can also be tested to support upgrades to Nautilus since they may be released in the same time frame. Since the Ceph tools are forward compatible, the Mimic tools will be sufficient to support upgrading to Nautilus.
|
||||
If Nautilus is released after Rook 0.9, a patch release can be made to 0.9 so that Rook can officially support the upgrade at that point. The changes in the patch release should be minimal since upgrading to Nautilus could have been mostly planned for in 0.9.
|
||||
@@ -76,8 +77,8 @@ The operator will be made to understand differences in the Ceph versions that ar
|
||||
|
||||
Rook will support a very specific list of major versions. Outside these versions, Rook will not be aware of the needs for configuring and upgrading the cluster.
|
||||
In v0.9, the supported versions will be:
|
||||
- Luminous (ceph/ceph:v12.2.x)
|
||||
- Mimic (ceph/ceph:v13.2.x)
|
||||
- luminous (ceph/ceph:v12.2.x)
|
||||
- mimic (ceph/ceph:v13.2.x)
|
||||
|
||||
Depending on the timing of the 0.9 and Nautilus releases, Nautilus will likely be supported either in 0.9 or a patch release. Versions not yet officially supported
|
||||
can be tested with settings in the CRD to be mentioned below.
|
||||
@@ -89,7 +90,7 @@ various Mimic patch releases.
|
||||
|
||||
The flexibility during upgrades will now be improved since the upgrade of Rook will be independent from the upgrade to the Ceph version.
|
||||
- To upgrade Rook, update the version of the Rook operator container
|
||||
- To upgrade Ceph, make sure Rook is running the latest release, then update the `cephImage` in the cluster CRD
|
||||
- To upgrade Ceph, make sure Rook is running the latest release, then update the `cephVersion.image` in the cluster CRD
|
||||
|
||||
The versions to be supported during upgrade will be a specific set for each version of Rook. In 0.9, it is anticipated that the only upgrade of Ceph
|
||||
supported would only be Luminous to Mimic. When Rook officially adds support for a release of Ceph (ie. Nautilus), the upgrade path will also be supported from one previous version.
|
||||
@@ -117,23 +118,26 @@ To allow more control over the upgrade, we define `upgradePolicy` settings. They
|
||||
- Allow for testing of future versions that are not officially supported
|
||||
|
||||
The settings in the CRD to accommodate the design include:
|
||||
- `newCephImage`: The version of the image to start applying to the daemons specified in the `upgrade` list. If `newCephImage` is not set, the operator will ensure that all daemons are running the version specified by the `cephImage` setting.
|
||||
- `upgrade`: A list of daemons that should be upgraded to the version `newCephImage`. The daemons include `mon`, `osd`, `mgr`, `rgw`, and `mds`. The ordering of the list will be ignored as Rook will only support ordering as it determines necessary for a version. If there are special upgrade actions in the future, they could be named and added to this list.
|
||||
- `allowUnrecognizedVersion`: If `false`, the operator would refuse to upgrade the Ceph version if it doesn't support or recognize that version. If `true`, Rook would go ahead and blindly set the image version and assume the pod specs should match `unrecognizedMajorVersion`. This would allow testing of upgrade to unreleased versions. The default is `false`.
|
||||
- `unrecognizedMajorVersion`: The version of pod spec to use when orchestrating an unrecognized version. Possible values are the names of versions that Rook understands: `luminous`, `mimic`, or `nautilus`.
|
||||
- `upgradePolicy.cephVersion`: The version of the image to start applying to the daemons specified in the `components` list.
|
||||
- `allowUnsupported`: If `false`, the operator would refuse to upgrade the Ceph version if it doesn't support or recognize that version. This would allow testing of upgrade to unreleased versions. The default is `false`.
|
||||
- `upgradePolicy.components`: A list of daemons or other components that should be upgraded to the version `newCephVersion`. The daemons include `mon`, `osd`, `mgr`, `rgw`, and `mds`. The ordering of the list will be ignored as Rook will only support ordering as it determines necessary for a version. If there are special upgrade actions in the future, they could be named and added to this list.
|
||||
|
||||
For example, with the settings below the operator would only upgrade the mons to mimic, while other daemons would remain on luminous. When the admin is ready, he would add more daemons to the list.
|
||||
|
||||
```yaml
|
||||
spec:
|
||||
cephImage: ceph/ceph:v12.2.7
|
||||
cephVersion:
|
||||
image: ceph/ceph:v12.2.9-20181026
|
||||
allowUnsupported: false
|
||||
upgradePolicy:
|
||||
newCephImage: ceph/ceph:v13.2.1
|
||||
upgrade:
|
||||
cephVersion:
|
||||
image: ceph/ceph:v13.2.2-20181023
|
||||
allowUnsupported: false
|
||||
components:
|
||||
- mon
|
||||
```
|
||||
|
||||
When the admin is completed with the upgrade or he is ready to allow Rook to complete the full upgrade for all daemons, he would set `cephImage: ceph/ceph:v13.2.1`, and the operator would ignore the `upgradePolicy` since the `cephImage` and `newCephImage` match.
|
||||
When the admin is completed with the upgrade or he is ready to allow Rook to complete the full upgrade for all daemons, he would set `cephVersion.image: ceph/ceph:v13.2.2`, and the operator would ignore the `upgradePolicy` since the `cephVersion` and `upgradePolicy.cephVersion` match.
|
||||
|
||||
If the admin wants to pause or otherwise control the upgrade closely, there are a couple of natural back doors:
|
||||
- Deleting the operator pod will effectively pause the upgrade. Starting the operator pod up again would resume the upgrade.
|
||||
@@ -141,18 +145,19 @@ If the admin wants to pause or otherwise control the upgrade closely, there are
|
||||
|
||||
#### Developer controls
|
||||
|
||||
If a developer wants to test the upgrade from mimic to nautilus, he would first create the cluster based on mimic. Then he would update the crd with the "unrecognized version" attributes in the CRD to specify nautilus such as:
|
||||
If a developer wants to test the upgrade from mimic to nautilus, he would first create the cluster based on mimic. Then he would update the crd with the "unrecognized version" attribute in the CRD to specify nautilus such as:
|
||||
```yaml
|
||||
spec:
|
||||
cephImage: ceph/ceph:v14.1.1
|
||||
upgradePolicy:
|
||||
allowUnrecognizedVersion: true
|
||||
unrecognizedMajorVersion: nautilus
|
||||
cephVersion:
|
||||
image: ceph/ceph:v14.1.1
|
||||
allowUnsupported: true
|
||||
```
|
||||
|
||||
Until Nautilus builds are released, the latest Nautilus build can be tested by using the image `ceph/daemon-base:latest-master`.
|
||||
|
||||
### Default Version
|
||||
|
||||
For backward compatibility, if the `cephImage` property is empty, the operator will need to internally set a default version of Ceph.
|
||||
For backward compatibility, if the `cephVersion` property is not set, the operator will need to internally set a default version of Ceph.
|
||||
The operator will assume the desired Ceph version is Luminous 12.2.7, which was shipped with Rook v0.8.
|
||||
This default will allow the Rook upgrade from v0.8 to v0.9 to only impact the Rook version and hold the Ceph version at Luminous.
|
||||
After the Rook upgrade to v0.9, the user can choose to set the `cephImage` property to some newer version of Ceph such as Mimic.
|
||||
After the Rook upgrade to v0.9, the user can choose to set the `cephVersion` property to some newer version of Ceph such as mimic.
|
||||
@@ -12,12 +12,8 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM BASEIMAGE
|
||||
|
||||
RUN yum --assumeyes install \
|
||||
net-tools \
|
||||
nmap-ncat && \
|
||||
yum clean all && rm -rf /tmp/* /var/tmp/*
|
||||
# the latest mimic release is the base image
|
||||
FROM ceph/ceph:v13
|
||||
|
||||
ARG ARCH
|
||||
ARG TINI_VERSION
|
||||
|
||||
@@ -19,8 +19,7 @@ include ../image.mk
|
||||
|
||||
CEPH_IMAGE = $(BUILD_REGISTRY)/ceph-$(GOARCH)
|
||||
|
||||
CEPH_VERSION = $(shell cat ceph_version)
|
||||
BASEIMAGE = $(CEPH_VERSION)-$(PLATFORM_ARCH)
|
||||
BASEIMAGE = ubuntu:18.04
|
||||
|
||||
TEMP := $(shell mktemp -d)
|
||||
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
ceph/daemon-base:v3.0.7-stable-3.0-luminous-centos-7
|
||||
@@ -48,6 +48,9 @@ type ClusterList struct {
|
||||
}
|
||||
|
||||
type ClusterSpec struct {
|
||||
// The version information that instructs Rook to orchestrate a particular version of Ceph.
|
||||
CephVersion CephVersionSpec `json:"cephVersion,omitempty"`
|
||||
|
||||
// A spec for available storage in the cluster and how it should be used
|
||||
Storage rook.StorageScopeSpec `json:"storage,omitempty"`
|
||||
|
||||
@@ -73,6 +76,18 @@ type ClusterSpec struct {
|
||||
Dashboard DashboardSpec `json:"dashboard,omitempty"`
|
||||
}
|
||||
|
||||
// VersionSpec represents the settings for the Ceph version that Rook is orchestrating.
|
||||
type CephVersionSpec struct {
|
||||
// Image is the container image used to launch the ceph daemons, such as ceph/ceph:v12.2.7 or ceph/ceph:v13.2.1
|
||||
Image string `json:"image,omitempty"`
|
||||
|
||||
// The name of the major release of Ceph: luminous, mimic, or nautilus
|
||||
Name string `json:"name,omitempty"`
|
||||
|
||||
// Whether to allow unsupported versions (do not set to true in production)
|
||||
AllowUnsupported bool `json:"allowUnsupported,omitempty"`
|
||||
}
|
||||
|
||||
// DashboardSpec represents the settings for the Ceph dashboard
|
||||
type DashboardSpec struct {
|
||||
// Whether to enable the dashboard
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package v1beta1
|
||||
|
||||
const (
|
||||
Luminous = "luminous"
|
||||
Mimic = "mimic"
|
||||
Nautilus = "nautilus"
|
||||
DefaultLuminousImage = "ceph/ceph:v12.2.9-20181026"
|
||||
)
|
||||
|
||||
func VersionAtLeast(version, minimumVersion string) bool {
|
||||
orderedVersions := []string{Luminous, Mimic, Nautilus}
|
||||
found := false
|
||||
for _, v := range orderedVersions {
|
||||
if v == minimumVersion {
|
||||
found = true
|
||||
}
|
||||
if v == version {
|
||||
// if we found the matching version, the version is at least the minimum version if found==true
|
||||
return found
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package v1beta1
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestVersionAtLeast(t *testing.T) {
|
||||
// Valid combinations
|
||||
assert.True(t, VersionAtLeast(Luminous, Luminous))
|
||||
assert.False(t, VersionAtLeast(Luminous, Mimic))
|
||||
assert.False(t, VersionAtLeast(Luminous, Nautilus))
|
||||
assert.True(t, VersionAtLeast(Mimic, Luminous))
|
||||
assert.True(t, VersionAtLeast(Mimic, Mimic))
|
||||
assert.False(t, VersionAtLeast(Mimic, Nautilus))
|
||||
assert.True(t, VersionAtLeast(Nautilus, Luminous))
|
||||
assert.True(t, VersionAtLeast(Nautilus, Mimic))
|
||||
assert.True(t, VersionAtLeast(Nautilus, Nautilus))
|
||||
|
||||
// Invalid combinations
|
||||
assert.False(t, VersionAtLeast(Mimic, "foo"))
|
||||
assert.False(t, VersionAtLeast("foo", Luminous))
|
||||
}
|
||||
@@ -25,6 +25,22 @@ import (
|
||||
runtime "k8s.io/apimachinery/pkg/runtime"
|
||||
)
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *CephVersionSpec) DeepCopyInto(out *CephVersionSpec) {
|
||||
*out = *in
|
||||
return
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CephVersionSpec.
|
||||
func (in *CephVersionSpec) DeepCopy() *CephVersionSpec {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(CephVersionSpec)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *Cluster) DeepCopyInto(out *Cluster) {
|
||||
*out = *in
|
||||
@@ -89,6 +105,7 @@ func (in *ClusterList) DeepCopyObject() runtime.Object {
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
|
||||
*out = *in
|
||||
out.CephVersion = in.CephVersion
|
||||
in.Storage.DeepCopyInto(&out.Storage)
|
||||
if in.Placement != nil {
|
||||
in, out := &in.Placement, &out.Placement
|
||||
|
||||
@@ -59,6 +59,10 @@ func FinalizeCephCommandArgs(command string, args []string, configDir, clusterNa
|
||||
return command, append(args, configArgs...)
|
||||
}
|
||||
|
||||
func ExecuteCephCommandDebugLog(context *clusterd.Context, clusterName string, args []string) ([]byte, error) {
|
||||
return executeCephCommandWithOutputFile(context, clusterName, true, args)
|
||||
}
|
||||
|
||||
func ExecuteCephCommand(context *clusterd.Context, clusterName string, args []string) ([]byte, error) {
|
||||
return executeCephCommandWithOutputFile(context, clusterName, false, args)
|
||||
}
|
||||
|
||||
@@ -21,6 +21,8 @@ import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
rookv1alpha2 "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
@@ -30,11 +32,23 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
batch "k8s.io/api/batch/v1"
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
detectVersionName = "rook-ceph-detect-version"
|
||||
)
|
||||
|
||||
var (
|
||||
// supportedVersions are production-ready versions that rook supports
|
||||
supportedVersions = []string{cephv1beta1.Luminous, cephv1beta1.Mimic}
|
||||
// allVersions includes all supportedVersions as well as unreleased versions that are being tested with rook
|
||||
allVersions = append(supportedVersions, cephv1beta1.Nautilus)
|
||||
)
|
||||
|
||||
type cluster struct {
|
||||
context *clusterd.Context
|
||||
Namespace string
|
||||
@@ -52,6 +66,79 @@ func newCluster(c *cephv1beta1.Cluster, context *clusterd.Context) *cluster {
|
||||
ownerRef: ClusterOwnerRef(c.Namespace, string(c.UID))}
|
||||
}
|
||||
|
||||
func (c *cluster) setCephMajorVersion(timeout time.Duration) error {
|
||||
version, err := c.detectCephMajorVersion(timeout)
|
||||
if err != nil {
|
||||
// Don't return the err yet here so we can override the failure with a setting in the crd below.
|
||||
logger.Errorf("failed to detect ceph version. %+v", err)
|
||||
} else {
|
||||
logger.Infof("detected ceph version %s for image %s", version, c.Spec.CephVersion.Image)
|
||||
}
|
||||
|
||||
if c.Spec.CephVersion.Name != "" {
|
||||
// if the cephVersion.name is already set, override the detected version
|
||||
logger.Warningf("overriding ceph version to %s specified in the CRD", c.Spec.CephVersion.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
c.Spec.CephVersion.Name = version
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *cluster) detectCephMajorVersion(timeout time.Duration) (string, error) {
|
||||
// get the major ceph version by running "ceph --version" in the ceph image
|
||||
job := &batch.Job{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: detectVersionName,
|
||||
Namespace: c.Namespace,
|
||||
},
|
||||
Spec: batch.JobSpec{
|
||||
Template: v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"job": detectVersionName,
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Command: []string{"ceph"},
|
||||
Args: []string{"--version"},
|
||||
Name: "version",
|
||||
Image: c.Spec.CephVersion.Image,
|
||||
},
|
||||
},
|
||||
RestartPolicy: v1.RestartPolicyOnFailure,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &job.ObjectMeta, &c.ownerRef)
|
||||
|
||||
// run the job to detect the version
|
||||
if err := k8sutil.RunReplaceableJob(c.context.Clientset, job); err != nil {
|
||||
return "", fmt.Errorf("failed to start version job. %+v", err)
|
||||
}
|
||||
|
||||
if err := k8sutil.WaitForJobCompletion(c.context.Clientset, job, timeout); err != nil {
|
||||
return "", fmt.Errorf("failed to complete version job. %+v", err)
|
||||
}
|
||||
|
||||
log, err := k8sutil.GetPodLog(c.context.Clientset, c.Namespace, "job="+detectVersionName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get version job log to detect version. %+v", err)
|
||||
}
|
||||
|
||||
version, err := extractCephVersion(log)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to extract ceph version. %+v", err)
|
||||
}
|
||||
|
||||
// delete the job since we're done with it
|
||||
k8sutil.DeleteBatchJob(c.context.Clientset, c.Namespace, job.Name, false)
|
||||
return version, nil
|
||||
}
|
||||
|
||||
func (c *cluster) createInstance(rookImage string) error {
|
||||
|
||||
// Create a configmap for overriding ceph config settings
|
||||
@@ -73,7 +160,7 @@ func (c *cluster) createInstance(rookImage string) error {
|
||||
}
|
||||
|
||||
// Start the mon pods
|
||||
c.mons = mon.New(c.context, c.Namespace, c.Spec.DataDirHostPath, rookImage, c.Spec.Mon, cephv1beta1.GetMonPlacement(c.Spec.Placement),
|
||||
c.mons = mon.New(c.context, c.Namespace, c.Spec.DataDirHostPath, rookImage, c.Spec.CephVersion, c.Spec.Mon, cephv1beta1.GetMonPlacement(c.Spec.Placement),
|
||||
c.Spec.Network.HostNetwork, cephv1beta1.GetMonResources(c.Spec.Resources), c.ownerRef)
|
||||
err = c.mons.Start()
|
||||
if err != nil {
|
||||
@@ -85,7 +172,7 @@ func (c *cluster) createInstance(rookImage string) error {
|
||||
return fmt.Errorf("failed to create initial crushmap: %+v", err)
|
||||
}
|
||||
|
||||
c.mgrs = mgr.New(c.context, c.Namespace, rookImage, cephv1beta1.GetMgrPlacement(c.Spec.Placement),
|
||||
c.mgrs = mgr.New(c.context, c.Namespace, rookImage, c.Spec.CephVersion, cephv1beta1.GetMgrPlacement(c.Spec.Placement),
|
||||
c.Spec.Network.HostNetwork, c.Spec.Dashboard, cephv1beta1.GetMgrResources(c.Spec.Resources), c.ownerRef)
|
||||
err = c.mgrs.Start()
|
||||
if err != nil {
|
||||
@@ -93,7 +180,7 @@ func (c *cluster) createInstance(rookImage string) error {
|
||||
}
|
||||
|
||||
// Start the OSDs
|
||||
c.osds = osd.New(c.context, c.Namespace, rookImage, c.Spec.ServiceAccount, c.Spec.Storage, c.Spec.DataDirHostPath,
|
||||
c.osds = osd.New(c.context, c.Namespace, rookImage, c.Spec.CephVersion, c.Spec.ServiceAccount, c.Spec.Storage, c.Spec.DataDirHostPath,
|
||||
cephv1beta1.GetOSDPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork, cephv1beta1.GetOSDResources(c.Spec.Resources), c.ownerRef)
|
||||
err = c.osds.Start()
|
||||
if err != nil {
|
||||
@@ -195,5 +282,28 @@ func clusterChanged(oldCluster, newCluster cephv1beta1.ClusterSpec, clusterRef *
|
||||
clusterRef.mons.MonCountMutex.Unlock()
|
||||
}
|
||||
|
||||
if oldCluster.CephVersion.AllowUnsupported != newCluster.CephVersion.AllowUnsupported {
|
||||
logger.Infof("ceph version allowUnsupported has changed from %t to %t", oldCluster.CephVersion.AllowUnsupported, newCluster.CephVersion.AllowUnsupported)
|
||||
changeFound = true
|
||||
}
|
||||
|
||||
return changeFound
|
||||
}
|
||||
|
||||
func extractCephVersion(version string) (string, error) {
|
||||
for _, v := range allVersions {
|
||||
if strings.Contains(version, v) {
|
||||
return v, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("failed to parse version from: %s", version)
|
||||
}
|
||||
|
||||
func versionSupported(version string) bool {
|
||||
for _, v := range supportedVersions {
|
||||
if v == version {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
testop "github.com/rook/rook/pkg/operator/test"
|
||||
exectest "github.com/rook/rook/pkg/util/exec/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestSetCephVersion(t *testing.T) {
|
||||
clientset := testop.New(3)
|
||||
executor := &exectest.MockExecutor{}
|
||||
context := &clusterd.Context{Clientset: clientset, Executor: executor}
|
||||
cluster := &cephv1beta1.Cluster{}
|
||||
c := newCluster(cluster, context)
|
||||
c.Namespace = "myns"
|
||||
|
||||
// expect a failure with no default and no version from the batch job
|
||||
err := c.setCephMajorVersion(time.Millisecond)
|
||||
assert.NotNil(t, err)
|
||||
assert.Equal(t, "", cluster.Spec.CephVersion.Name)
|
||||
|
||||
// expect success when the default is set from the crd
|
||||
cluster.Spec.CephVersion.Name = "mimic"
|
||||
err = c.setCephMajorVersion(time.Millisecond)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "mimic", cluster.Spec.CephVersion.Name)
|
||||
}
|
||||
@@ -182,6 +182,19 @@ func (c *ClusterController) onAdd(obj interface{}) {
|
||||
logger.Warningf("mon count is even (given: %d), should be uneven, continuing", cluster.Spec.Mon.Count)
|
||||
}
|
||||
|
||||
err = cluster.setCephMajorVersion(15 * time.Minute)
|
||||
if err != nil {
|
||||
logger.Errorf("unknown ceph major version. %+v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if !cluster.Spec.CephVersion.AllowUnsupported {
|
||||
if !versionSupported(cluster.Spec.CephVersion.Name) {
|
||||
logger.Errorf("unsupported ceph version detected: %s. allowUnupported must be set to true to run with this version.", cluster.Spec.CephVersion.Name)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Start the Rook cluster components. Retry several times in case of failure.
|
||||
err = wait.Poll(clusterCreateInterval, clusterCreateTimeout, func() (bool, error) {
|
||||
if err := c.updateClusterStatus(clusterObj.Namespace, clusterObj.Name, cephv1beta1.ClusterStateCreating, ""); err != nil {
|
||||
@@ -217,11 +230,11 @@ func (c *ClusterController) onAdd(obj interface{}) {
|
||||
poolController.StartWatch(cluster.Namespace, cluster.stopCh)
|
||||
|
||||
// Start object store CRD watcher
|
||||
objectStoreController := object.NewObjectStoreController(c.context, c.rookImage, cluster.Spec.Network.HostNetwork, cluster.ownerRef)
|
||||
objectStoreController := object.NewObjectStoreController(c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef)
|
||||
objectStoreController.StartWatch(cluster.Namespace, cluster.stopCh)
|
||||
|
||||
// Start file system CRD watcher
|
||||
fileController := file.NewFilesystemController(c.context, c.rookImage, cluster.Spec.Network.HostNetwork, cluster.ownerRef)
|
||||
fileController := file.NewFilesystemController(c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef)
|
||||
fileController.StartWatch(cluster.Namespace, cluster.stopCh)
|
||||
|
||||
// Start mon health checker
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package mgr for the Ceph manager.
|
||||
package mgr
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os/exec"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
dashboardModuleName = "dashboard"
|
||||
dashboardPortHttps = 8443
|
||||
dashboardPortHttp = 7000
|
||||
dashboardUsername = "admin"
|
||||
dashboardPasswordName = "rook-ceph-dashboard-password"
|
||||
passwordLength = 10
|
||||
passwordKeyName = "password"
|
||||
certAlreadyConfiguredErrorCode = 5
|
||||
)
|
||||
|
||||
var (
|
||||
dashboardInitWaitTime = 5 * time.Second
|
||||
)
|
||||
|
||||
func init() {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func (c *Cluster) configureDashboard(port int) error {
|
||||
// enable or disable the dashboard module
|
||||
if err := c.configureDashboardModule(); err != nil {
|
||||
return fmt.Errorf("failed to enable mgr dashboard module. %+v", err)
|
||||
}
|
||||
|
||||
dashboardService := c.makeDashboardService(appName, port)
|
||||
if c.dashboard.Enabled {
|
||||
// expose the dashboard service
|
||||
if _, err := c.context.Clientset.CoreV1().Services(c.Namespace).Create(dashboardService); err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create dashboard mgr service. %+v", err)
|
||||
}
|
||||
logger.Infof("dashboard service already exists")
|
||||
} else {
|
||||
logger.Infof("dashboard service started")
|
||||
}
|
||||
} else {
|
||||
// delete the dashboard service if it exists
|
||||
err := c.context.Clientset.CoreV1().Services(c.Namespace).Delete(dashboardService.Name, &metav1.DeleteOptions{})
|
||||
if err != nil && !errors.IsNotFound(err) {
|
||||
return fmt.Errorf("failed to delete dashboard service. %+v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ceph docs about the dashboard module: http://docs.ceph.com/docs/luminous/mgr/dashboard/
|
||||
func (c *Cluster) configureDashboardModule() error {
|
||||
if c.dashboard.Enabled {
|
||||
if err := client.MgrEnableModule(c.context, c.Namespace, dashboardModuleName, true); err != nil {
|
||||
return fmt.Errorf("failed to enable mgr dashboard module. %+v", err)
|
||||
}
|
||||
|
||||
if err := c.initializeDashboard(); err != nil {
|
||||
return fmt.Errorf("failed to initialize dashboard. %+v", err)
|
||||
}
|
||||
} else {
|
||||
if err := client.MgrDisableModule(c.context, c.Namespace, dashboardModuleName); err != nil {
|
||||
logger.Errorf("failed to disable mgr dashboard module. %+v", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) initializeDashboard() error {
|
||||
if c.cephVersion.Name == cephv1beta1.Luminous || c.cephVersion.Name == "" {
|
||||
logger.Infof("skipping cert and user configuration on luminous")
|
||||
return nil
|
||||
}
|
||||
|
||||
// we need to wait a short period after enabling the module before we can call the `ceph dashboard` commands.
|
||||
time.Sleep(dashboardInitWaitTime)
|
||||
|
||||
password, err := c.getOrGenerateDashboardPassword()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to generate a password. %+v", err)
|
||||
}
|
||||
|
||||
alreadyCreated, err := c.createSelfSignedCert()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create a self signed cert. %+v", err)
|
||||
}
|
||||
if alreadyCreated {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := c.setLoginCredentials(password); err != nil {
|
||||
return fmt.Errorf("failed to set login creds. %+v", err)
|
||||
}
|
||||
|
||||
return c.restartDashboard()
|
||||
}
|
||||
|
||||
func (c *Cluster) createSelfSignedCert() (bool, error) {
|
||||
// create a self-signed cert for the https connections required in mimic
|
||||
args := []string{"dashboard", "create-self-signed-cert"}
|
||||
_, err := client.ExecuteCephCommand(c.context, c.Namespace, args)
|
||||
if err != nil {
|
||||
if exiterr, ok := err.(*exec.ExitError); ok {
|
||||
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
|
||||
exitCode := status.ExitStatus()
|
||||
if exitCode == certAlreadyConfiguredErrorCode {
|
||||
logger.Infof("dashboard is already initialized with a cert")
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, fmt.Errorf("failed to create self signed cert on mgr. %+v", err)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) setLoginCredentials(password string) error {
|
||||
// Set the login credentials. Write the command/args to the debug log so we don't write the password by default to the log.
|
||||
logger.Infof("Running command: ceph dashboard set-login-credentials admin *******")
|
||||
args := []string{"dashboard", "set-login-credentials", dashboardUsername, password}
|
||||
_, err := client.ExecuteCephCommandDebugLog(c.context, c.Namespace, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to set login creds on mgr. %+v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) getOrGenerateDashboardPassword() (string, error) {
|
||||
secret, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(dashboardPasswordName, metav1.GetOptions{})
|
||||
if err == nil {
|
||||
logger.Infof("the dashboard secret was already generated")
|
||||
return decodeSecret(secret)
|
||||
}
|
||||
if !errors.IsNotFound(err) {
|
||||
return "", fmt.Errorf("failed to get dashboard secret. %+v", err)
|
||||
}
|
||||
|
||||
// Generate a password
|
||||
password := generatePassword(passwordLength)
|
||||
|
||||
// Store the keyring in a secret
|
||||
secrets := map[string][]byte{
|
||||
passwordKeyName: []byte(password),
|
||||
}
|
||||
secret = &v1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: dashboardPasswordName,
|
||||
Namespace: c.Namespace,
|
||||
},
|
||||
Data: secrets,
|
||||
Type: k8sutil.RookType,
|
||||
}
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &secret.ObjectMeta, &c.ownerRef)
|
||||
|
||||
_, err = c.context.Clientset.CoreV1().Secrets(c.Namespace).Create(secret)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to save dashboard secret. %+v", err)
|
||||
}
|
||||
return password, nil
|
||||
}
|
||||
|
||||
func generatePassword(length int) string {
|
||||
const passwordChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
passwd := make([]byte, length)
|
||||
for i := range passwd {
|
||||
passwd[i] = passwordChars[rand.Intn(len(passwordChars))]
|
||||
}
|
||||
return string(passwd)
|
||||
}
|
||||
|
||||
func decodeSecret(secret *v1.Secret) (string, error) {
|
||||
password, ok := secret.Data[passwordKeyName]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("password not found in secret")
|
||||
}
|
||||
return string(password), nil
|
||||
}
|
||||
|
||||
func (c *Cluster) restartDashboard() error {
|
||||
logger.Infof("restarting the mgr module")
|
||||
client.MgrDisableModule(c.context, c.Namespace, dashboardModuleName)
|
||||
client.MgrEnableModule(c.context, c.Namespace, dashboardModuleName, true)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package mgr
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/test"
|
||||
exectest "github.com/rook/rook/pkg/util/exec/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func TestGeneratePassword(t *testing.T) {
|
||||
password := generatePassword(0)
|
||||
assert.Equal(t, "", password)
|
||||
|
||||
password = generatePassword(1)
|
||||
assert.Equal(t, 1, len(password))
|
||||
logger.Infof("password: %s", password)
|
||||
|
||||
password = generatePassword(10)
|
||||
assert.Equal(t, 10, len(password))
|
||||
logger.Infof("password: %s", password)
|
||||
}
|
||||
|
||||
func TestGetOrGeneratePassword(t *testing.T) {
|
||||
c := &Cluster{context: &clusterd.Context{Clientset: test.New(3)}, Namespace: "myns"}
|
||||
_, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(dashboardPasswordName, metav1.GetOptions{})
|
||||
assert.True(t, errors.IsNotFound(err))
|
||||
|
||||
// Generate a password
|
||||
password, err := c.getOrGenerateDashboardPassword()
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, passwordLength, len(password))
|
||||
|
||||
secret, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(dashboardPasswordName, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(secret.Data))
|
||||
passwordFromSecret, err := decodeSecret(secret)
|
||||
assert.Equal(t, password, passwordFromSecret)
|
||||
|
||||
// We should retrieve the same password on the second call
|
||||
retrievedPassword, err := c.getOrGenerateDashboardPassword()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, password, retrievedPassword)
|
||||
}
|
||||
|
||||
func TestStartSecureDashboard(t *testing.T) {
|
||||
enables := 0
|
||||
disables := 0
|
||||
executor := &exectest.MockExecutor{
|
||||
MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) {
|
||||
logger.Infof("command: %s %v", command, args)
|
||||
if args[1] == "module" {
|
||||
if args[2] == "enable" {
|
||||
enables++
|
||||
} else if args[2] == "disable" {
|
||||
disables++
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
},
|
||||
}
|
||||
c := &Cluster{context: &clusterd.Context{Clientset: test.New(3), Executor: executor}, Namespace: "myns",
|
||||
dashboard: cephv1beta1.DashboardSpec{Enabled: true}, cephVersion: cephv1beta1.CephVersionSpec{Name: cephv1beta1.Mimic, Image: "ceph/ceph:v13.2.2"}}
|
||||
dashboardInitWaitTime = 0
|
||||
err := c.configureDashboard(dashboardPortHttp)
|
||||
assert.Nil(t, err)
|
||||
// the dashboard is enabled, then disabled and enabled again to restart it with the cert
|
||||
assert.Equal(t, 2, enables)
|
||||
assert.Equal(t, 1, disables)
|
||||
|
||||
svc, err := c.context.Clientset.CoreV1().Services(c.Namespace).Get("rook-ceph-mgr-dashboard", metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.NotNil(t, svc)
|
||||
|
||||
// disable the dashboard
|
||||
c.dashboard.Enabled = false
|
||||
err = c.configureDashboard(dashboardPortHttp)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, enables)
|
||||
assert.Equal(t, 2, disables)
|
||||
|
||||
svc, err = c.context.Clientset.CoreV1().Services(c.Namespace).Get("rook-ceph-mgr-dashboard", metav1.GetOptions{})
|
||||
assert.NotNil(t, err)
|
||||
assert.True(t, errors.IsNotFound(err))
|
||||
assert.Nil(t, svc)
|
||||
}
|
||||
@@ -25,7 +25,6 @@ import (
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
opspec "github.com/rook/rook/pkg/operator/ceph/spec"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
@@ -38,9 +37,7 @@ const (
|
||||
appName = "rook-ceph-mgr"
|
||||
keyringSecretKeyName = "keyring"
|
||||
prometheusModuleName = "prometheus"
|
||||
dashboardModuleName = "dashboard"
|
||||
metricsPort = 9283
|
||||
dashboardPort = 7000
|
||||
)
|
||||
|
||||
var mgrNames = []string{"a", "b"}
|
||||
@@ -48,7 +45,6 @@ var mgrNames = []string{"a", "b"}
|
||||
// Cluster represents the Rook and environment configuration settings needed to set up Ceph mgrs.
|
||||
type Cluster struct {
|
||||
Namespace string
|
||||
Version string
|
||||
Replicas int
|
||||
placement rookalpha.Placement
|
||||
context *clusterd.Context
|
||||
@@ -57,6 +53,8 @@ type Cluster struct {
|
||||
resources v1.ResourceRequirements
|
||||
ownerRef metav1.OwnerReference
|
||||
dashboard cephv1beta1.DashboardSpec
|
||||
cephVersion cephv1beta1.CephVersionSpec
|
||||
rookVersion string
|
||||
}
|
||||
|
||||
// mgrConfig for a single mgr
|
||||
@@ -66,13 +64,14 @@ type mgrConfig struct {
|
||||
}
|
||||
|
||||
// New creates an instance of the mgr
|
||||
func New(context *clusterd.Context, namespace, version string, placement rookalpha.Placement, hostNetwork bool, dashboard cephv1beta1.DashboardSpec,
|
||||
func New(context *clusterd.Context, namespace, rookVersion string, cephVersion cephv1beta1.CephVersionSpec, placement rookalpha.Placement, hostNetwork bool, dashboard cephv1beta1.DashboardSpec,
|
||||
resources v1.ResourceRequirements, ownerRef metav1.OwnerReference) *Cluster {
|
||||
return &Cluster{
|
||||
context: context,
|
||||
Namespace: namespace,
|
||||
placement: placement,
|
||||
Version: version,
|
||||
rookVersion: rookVersion,
|
||||
cephVersion: cephVersion,
|
||||
Replicas: 1,
|
||||
dataDir: k8sutil.DataDir,
|
||||
dashboard: dashboard,
|
||||
@@ -86,6 +85,11 @@ func New(context *clusterd.Context, namespace, version string, placement rookalp
|
||||
func (c *Cluster) Start() error {
|
||||
logger.Infof("start running mgr")
|
||||
|
||||
dashboardPort := dashboardPortHttps
|
||||
if c.cephVersion.Name == cephv1beta1.Luminous {
|
||||
dashboardPort = dashboardPortHttp
|
||||
}
|
||||
|
||||
for i := 0; i < c.Replicas; i++ {
|
||||
if i >= len(mgrNames) {
|
||||
logger.Errorf("cannot have more than %d mgrs", len(mgrNames))
|
||||
@@ -104,7 +108,7 @@ func (c *Cluster) Start() error {
|
||||
}
|
||||
|
||||
// start the deployment
|
||||
deployment := c.makeDeployment(mgrConfig)
|
||||
deployment := c.makeDeployment(mgrConfig, dashboardPort)
|
||||
if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Create(deployment); err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create %s deployment. %+v", resourceName, err)
|
||||
@@ -115,8 +119,16 @@ func (c *Cluster) Start() error {
|
||||
}
|
||||
}
|
||||
|
||||
if err := c.configureOrchestratorModules(); err != nil {
|
||||
logger.Errorf("failed to enable orchestrator modules. %+v", err)
|
||||
}
|
||||
|
||||
if err := c.enablePrometheusModule(c.Namespace); err != nil {
|
||||
return fmt.Errorf("failed to enable mgr prometheus module. %+v", err)
|
||||
logger.Errorf("failed to enable mgr prometheus module. %+v", err)
|
||||
}
|
||||
|
||||
if err := c.configureDashboard(dashboardPort); err != nil {
|
||||
logger.Errorf("failed to enable mgr dashboard. %+v", err)
|
||||
}
|
||||
|
||||
// create the metrics service
|
||||
@@ -130,86 +142,9 @@ func (c *Cluster) Start() error {
|
||||
logger.Infof("mgr metrics service started")
|
||||
}
|
||||
|
||||
return c.configureDashboard()
|
||||
}
|
||||
|
||||
func (c *Cluster) configureDashboard() error {
|
||||
// enable or disable the dashboard module
|
||||
if err := c.configureDashboardModule(c.Namespace, c.dashboard.Enabled); err != nil {
|
||||
return fmt.Errorf("failed to enable mgr dashboard module. %+v", err)
|
||||
}
|
||||
|
||||
dashboardService := c.makeDashboardService(appName)
|
||||
if c.dashboard.Enabled {
|
||||
// expose the dashboard service
|
||||
if _, err := c.context.Clientset.CoreV1().Services(c.Namespace).Create(dashboardService); err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create dashboard mgr service. %+v", err)
|
||||
}
|
||||
logger.Infof("dashboard service already exists")
|
||||
} else {
|
||||
logger.Infof("dashboard service started")
|
||||
}
|
||||
} else {
|
||||
// delete the dashboard service if it exists
|
||||
err := c.context.Clientset.CoreV1().Services(c.Namespace).Delete(dashboardService.Name, &metav1.DeleteOptions{})
|
||||
if err != nil && !errors.IsNotFound(err) {
|
||||
return fmt.Errorf("failed to delete dashboard service. %+v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) makeMetricsService(name string) *v1.Service {
|
||||
labels := opspec.AppLabels(appName, c.Namespace)
|
||||
svc := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: c.Namespace,
|
||||
Labels: labels,
|
||||
},
|
||||
Spec: v1.ServiceSpec{
|
||||
Selector: labels,
|
||||
Type: v1.ServiceTypeClusterIP,
|
||||
Ports: []v1.ServicePort{
|
||||
{
|
||||
Name: "http-metrics",
|
||||
Port: int32(metricsPort),
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &svc.ObjectMeta, &c.ownerRef)
|
||||
return svc
|
||||
}
|
||||
|
||||
func (c *Cluster) makeDashboardService(name string) *v1.Service {
|
||||
labels := opspec.AppLabels(appName, c.Namespace)
|
||||
svc := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: fmt.Sprintf("%s-dashboard", name),
|
||||
Namespace: c.Namespace,
|
||||
Labels: labels,
|
||||
},
|
||||
Spec: v1.ServiceSpec{
|
||||
Selector: labels,
|
||||
Type: v1.ServiceTypeClusterIP,
|
||||
Ports: []v1.ServicePort{
|
||||
{
|
||||
Name: "http-dashboard",
|
||||
Port: int32(dashboardPort),
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &svc.ObjectMeta, &c.ownerRef)
|
||||
return svc
|
||||
}
|
||||
|
||||
func (c *Cluster) createKeyring(clusterName, name, daemonName string) error {
|
||||
_, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(name, metav1.GetOptions{})
|
||||
if err == nil {
|
||||
@@ -256,23 +191,9 @@ func (c *Cluster) enablePrometheusModule(clusterName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ceph docs about the dashboard module: http://docs.ceph.com/docs/luminous/mgr/dashboard/
|
||||
func (c *Cluster) configureDashboardModule(clusterName string, enable bool) error {
|
||||
if enable {
|
||||
if err := client.MgrEnableModule(c.context, clusterName, dashboardModuleName, true); err != nil {
|
||||
return fmt.Errorf("failed to enable mgr dashboard module. %+v", err)
|
||||
}
|
||||
} else {
|
||||
if err := client.MgrDisableModule(c.context, clusterName, dashboardModuleName); err != nil {
|
||||
return fmt.Errorf("failed to disable mgr dashboard module. %+v", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getKeyringProperties(name string) (string, []string) {
|
||||
username := fmt.Sprintf("mgr.%s", name)
|
||||
access := []string{"mon", "allow *"}
|
||||
access := []string{"mon", "allow profile mgr", "mds", "allow *", "osd", "allow *"}
|
||||
return username, access
|
||||
}
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ func TestStartMGR(t *testing.T) {
|
||||
Executor: executor,
|
||||
ConfigDir: configDir,
|
||||
Clientset: testop.New(3)}
|
||||
c := New(context, "ns", "myversion", rookalpha.Placement{}, false, cephv1beta1.DashboardSpec{Enabled: true}, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c := New(context, "ns", "myversion", cephv1beta1.CephVersionSpec{}, rookalpha.Placement{}, false, cephv1beta1.DashboardSpec{Enabled: true}, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
defer os.RemoveAll(c.dataDir)
|
||||
|
||||
// start a basic service
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package mgr for the Ceph manager.
|
||||
package mgr
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
)
|
||||
|
||||
const (
|
||||
orchestratorModuleName = "orchestrator_cli"
|
||||
rookModuleName = "rook"
|
||||
)
|
||||
|
||||
// Ceph docs about the orchestrator modules: http://docs.ceph.com/docs/master/mgr/orchestrator_cli/
|
||||
func (c *Cluster) configureOrchestratorModules() error {
|
||||
if !cephv1beta1.VersionAtLeast(c.cephVersion.Name, cephv1beta1.Nautilus) {
|
||||
logger.Infof("skipping enabling orchestrator modules on releases older than nautilus")
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := client.MgrEnableModule(c.context, c.Namespace, orchestratorModuleName, true); err != nil {
|
||||
return fmt.Errorf("failed to enable mgr orchestrator module. %+v", err)
|
||||
}
|
||||
if err := client.MgrEnableModule(c.context, c.Namespace, rookModuleName, true); err != nil {
|
||||
return fmt.Errorf("failed to enable mgr rook module. %+v", err)
|
||||
}
|
||||
if _, err := client.ExecuteCephCommand(c.context, c.Namespace, []string{"orchestrator", "set", "backend", "rook"}); err != nil {
|
||||
return fmt.Errorf("failed to set rook as the orchestrator backend. %+v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package mgr
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
exectest "github.com/rook/rook/pkg/util/exec/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestOrchestratorModules(t *testing.T) {
|
||||
executor := &exectest.MockExecutor{}
|
||||
context := &clusterd.Context{Executor: executor}
|
||||
orchestratorModuleEnabled := false
|
||||
rookModuleEnabled := false
|
||||
rookBackendSet := false
|
||||
executor.MockExecuteCommandWithOutputFile = func(debug bool, actionName, command, outputFile string, args ...string) (string, error) {
|
||||
logger.Infof("Command: %s %v", command, args)
|
||||
if args[0] == "mgr" && args[1] == "module" && args[2] == "enable" {
|
||||
if args[3] == "orchestrator_cli" {
|
||||
orchestratorModuleEnabled = true
|
||||
return "", nil
|
||||
}
|
||||
if args[3] == "rook" {
|
||||
rookModuleEnabled = true
|
||||
return "", nil
|
||||
}
|
||||
}
|
||||
if args[0] == "orchestrator" && args[1] == "set" && args[2] == "backend" && args[3] == "rook" {
|
||||
rookBackendSet = true
|
||||
return "", nil
|
||||
}
|
||||
return "", fmt.Errorf("unexpected ceph command '%v'", args)
|
||||
}
|
||||
|
||||
c := &Cluster{context: context}
|
||||
|
||||
// the modules are skipped on luminous
|
||||
c.cephVersion.Name = cephv1beta1.Luminous
|
||||
err := c.configureOrchestratorModules()
|
||||
assert.Nil(t, err)
|
||||
assert.False(t, orchestratorModuleEnabled)
|
||||
assert.False(t, rookModuleEnabled)
|
||||
assert.False(t, rookBackendSet)
|
||||
|
||||
// the modules are skipped on mimic
|
||||
c.cephVersion.Name = cephv1beta1.Mimic
|
||||
err = c.configureOrchestratorModules()
|
||||
assert.Nil(t, err)
|
||||
assert.False(t, orchestratorModuleEnabled)
|
||||
assert.False(t, rookModuleEnabled)
|
||||
assert.False(t, rookBackendSet)
|
||||
|
||||
// the modules are configured on nautilus
|
||||
c.cephVersion.Name = cephv1beta1.Nautilus
|
||||
err = c.configureOrchestratorModules()
|
||||
assert.Nil(t, err)
|
||||
assert.True(t, orchestratorModuleEnabled)
|
||||
assert.True(t, rookModuleEnabled)
|
||||
assert.True(t, rookBackendSet)
|
||||
}
|
||||
@@ -33,7 +33,7 @@ const (
|
||||
mgrDaemonCommand = "ceph-mgr"
|
||||
)
|
||||
|
||||
func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) *extensions.Deployment {
|
||||
func (c *Cluster) makeDeployment(mgrConfig *mgrConfig, port int) *extensions.Deployment {
|
||||
podSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: mgrConfig.ResourceName,
|
||||
@@ -47,7 +47,7 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) *extensions.Deployment {
|
||||
c.makeConfigInitContainer(mgrConfig),
|
||||
},
|
||||
Containers: []v1.Container{
|
||||
c.makeMgrDaemonContainer(mgrConfig),
|
||||
c.makeMgrDaemonContainer(mgrConfig, port),
|
||||
},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: opspec.PodVolumes(""),
|
||||
@@ -80,7 +80,7 @@ func (c *Cluster) makeConfigInitContainer(mgrConfig *mgrConfig) v1.Container {
|
||||
fmt.Sprintf("--config-dir=%s", k8sutil.DataDir),
|
||||
fmt.Sprintf("--mgr-name=%s", mgrConfig.DaemonName),
|
||||
},
|
||||
Image: k8sutil.MakeRookImage(c.Version),
|
||||
Image: k8sutil.MakeRookImage(c.rookVersion),
|
||||
Env: []v1.EnvVar{
|
||||
// Set '--mgr-keyring' flag with an env var sourced from the secret
|
||||
{Name: "ROOK_MGR_KEYRING",
|
||||
@@ -102,7 +102,7 @@ func (c *Cluster) makeConfigInitContainer(mgrConfig *mgrConfig) v1.Container {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container {
|
||||
func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig, port int) v1.Container {
|
||||
container := v1.Container{
|
||||
Name: "mgr",
|
||||
Command: []string{
|
||||
@@ -113,7 +113,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container {
|
||||
"--id", mgrConfig.DaemonName,
|
||||
// do not add the '--cluster/--conf/--keyring' flags; rook wants their default values
|
||||
},
|
||||
Image: k8sutil.MakeRookImage(c.Version),
|
||||
Image: c.cephVersion.Image,
|
||||
VolumeMounts: opspec.CephVolumeMounts(),
|
||||
Ports: []v1.ContainerPort{
|
||||
{
|
||||
@@ -128,7 +128,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container {
|
||||
},
|
||||
{
|
||||
Name: "dashboard",
|
||||
ContainerPort: int32(dashboardPort),
|
||||
ContainerPort: int32(port),
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
},
|
||||
@@ -139,6 +139,55 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container {
|
||||
return container
|
||||
}
|
||||
|
||||
func (c *Cluster) makeMetricsService(name string) *v1.Service {
|
||||
labels := opspec.AppLabels(appName, c.Namespace)
|
||||
svc := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: c.Namespace,
|
||||
Labels: labels,
|
||||
},
|
||||
Spec: v1.ServiceSpec{
|
||||
Selector: labels,
|
||||
Type: v1.ServiceTypeClusterIP,
|
||||
Ports: []v1.ServicePort{
|
||||
{
|
||||
Name: "http-metrics",
|
||||
Port: int32(metricsPort),
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &svc.ObjectMeta, &c.ownerRef)
|
||||
return svc
|
||||
}
|
||||
|
||||
func (c *Cluster) makeDashboardService(name string, port int) *v1.Service {
|
||||
labels := opspec.AppLabels(appName, c.Namespace)
|
||||
svc := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: fmt.Sprintf("%s-dashboard", name),
|
||||
Namespace: c.Namespace,
|
||||
Labels: labels,
|
||||
},
|
||||
Spec: v1.ServiceSpec{
|
||||
Selector: labels,
|
||||
Type: v1.ServiceTypeClusterIP,
|
||||
Ports: []v1.ServicePort{
|
||||
{
|
||||
Name: "https-dashboard",
|
||||
Port: int32(port),
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &svc.ObjectMeta, &c.ownerRef)
|
||||
return svc
|
||||
}
|
||||
|
||||
func (c *Cluster) getPodLabels(daemonName string) map[string]string {
|
||||
labels := opspec.PodLabels(appName, c.Namespace, "mgr", daemonName)
|
||||
// leave "instance" key for legacy usage
|
||||
|
||||
@@ -41,6 +41,7 @@ func TestPodSpec(t *testing.T) {
|
||||
&clusterd.Context{Clientset: testop.New(1)},
|
||||
"ns",
|
||||
"rook/rook:myversion",
|
||||
cephv1beta1.CephVersionSpec{Image: "ceph/ceph:myceph"},
|
||||
rookalpha.Placement{},
|
||||
false,
|
||||
cephv1beta1.DashboardSpec{},
|
||||
@@ -60,7 +61,7 @@ func TestPodSpec(t *testing.T) {
|
||||
ResourceName: "mgr-a",
|
||||
}
|
||||
|
||||
d := c.makeDeployment(&mgrTestConfig)
|
||||
d := c.makeDeployment(&mgrTestConfig, dashboardPortHttp)
|
||||
|
||||
assert.NotNil(t, d)
|
||||
assert.Equal(t, "mgr-a", d.Name)
|
||||
@@ -106,8 +107,7 @@ func TestPodSpec(t *testing.T) {
|
||||
configContainerDefinition.TestContainer(t, "config init", cont, logger)
|
||||
assert.Equal(t, "100", cont.Resources.Limits.Cpu().String())
|
||||
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
|
||||
|
||||
daemonImage := "rook/rook:myversion"
|
||||
daemonImage := "ceph/ceph:myceph"
|
||||
// +1 for $ROOK_CLUSTER_NAME
|
||||
daemonEnvs := len(k8sutil.ClusterDaemonEnvVars()) + 1
|
||||
daemonContainerDefinition := cephtest.ContainerTestDefinition{
|
||||
@@ -126,7 +126,7 @@ func TestPodSpec(t *testing.T) {
|
||||
Protocol: v1.ProtocolTCP},
|
||||
{ContainerPort: int32(metricsPort),
|
||||
Protocol: v1.ProtocolTCP},
|
||||
{ContainerPort: int32(dashboardPort),
|
||||
{ContainerPort: int32(dashboardPortHttp),
|
||||
Protocol: v1.ProtocolTCP}},
|
||||
IsPrivileged: nil, // not set in spec
|
||||
}
|
||||
@@ -146,7 +146,7 @@ func TestPodSpec(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestServiceSpec(t *testing.T) {
|
||||
c := New(&clusterd.Context{}, "ns", "myversion", rookalpha.Placement{}, false, cephv1beta1.DashboardSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c := New(&clusterd.Context{}, "ns", "myversion", cephv1beta1.CephVersionSpec{}, rookalpha.Placement{}, false, cephv1beta1.DashboardSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
s := c.makeMetricsService("rook-mgr")
|
||||
assert.NotNil(t, s)
|
||||
@@ -159,6 +159,7 @@ func TestHostNetwork(t *testing.T) {
|
||||
&clusterd.Context{Clientset: testop.New(1)},
|
||||
"ns",
|
||||
"myversion",
|
||||
cephv1beta1.CephVersionSpec{},
|
||||
rookalpha.Placement{},
|
||||
true,
|
||||
cephv1beta1.DashboardSpec{},
|
||||
@@ -171,7 +172,7 @@ func TestHostNetwork(t *testing.T) {
|
||||
ResourceName: "mgr-a",
|
||||
}
|
||||
|
||||
d := c.makeDeployment(&mgrTestConfig)
|
||||
d := c.makeDeployment(&mgrTestConfig, dashboardPortHttp)
|
||||
assert.NotNil(t, d)
|
||||
|
||||
assert.Equal(t, true, d.Spec.Template.Spec.HostNetwork)
|
||||
|
||||
@@ -48,7 +48,9 @@ func getClusterObject(obj interface{}) (cluster *cephv1beta1.Cluster, migrationN
|
||||
cluster, ok = obj.(*cephv1beta1.Cluster)
|
||||
if ok {
|
||||
// the cluster object is of the latest type, simply return it
|
||||
return cluster.DeepCopy(), false, nil
|
||||
cluster = cluster.DeepCopy()
|
||||
setClusterDefaults(cluster)
|
||||
return cluster, false, nil
|
||||
}
|
||||
|
||||
// type assertion to current cluster type failed, try instead asserting to the legacy cluster types
|
||||
@@ -61,6 +63,16 @@ func getClusterObject(obj interface{}) (cluster *cephv1beta1.Cluster, migrationN
|
||||
return nil, false, fmt.Errorf("not a known cluster object: %+v", obj)
|
||||
}
|
||||
|
||||
func setClusterDefaults(cluster *cephv1beta1.Cluster) {
|
||||
// The ceph version image should be set in the CRD.
|
||||
// If/when the v1beta1 CRD is converted to v1, we could set this permanently during the conversion instead of
|
||||
// setting this default in memory every time we run the operator.
|
||||
if cluster.Spec.CephVersion.Image == "" {
|
||||
logger.Infof("setting default luminous image: %s", cephv1beta1.DefaultLuminousImage)
|
||||
cluster.Spec.CephVersion.Image = cephv1beta1.DefaultLuminousImage
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ClusterController) migrateClusterObject(clusterToMigrate *cephv1beta1.Cluster, legacyObj interface{}) error {
|
||||
logger.Infof("migrating legacy cluster %s in namespace %s", clusterToMigrate.Name, clusterToMigrate.Namespace)
|
||||
|
||||
|
||||
@@ -54,6 +54,14 @@ func TestGetClusterObject(t *testing.T) {
|
||||
assert.NotNil(t, err)
|
||||
}
|
||||
|
||||
func TestDefaultClusterValues(t *testing.T) {
|
||||
// the default ceph version should be set
|
||||
cluster, _, err := getClusterObject(&cephv1beta1.Cluster{})
|
||||
assert.NotNil(t, cluster)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, cephv1beta1.DefaultLuminousImage, cluster.Spec.CephVersion.Image)
|
||||
}
|
||||
|
||||
func TestMigrateClusterObject(t *testing.T) {
|
||||
// create a legacy cluster that will get migrated
|
||||
legacyCluster := &rookv1alpha1.Cluster{
|
||||
|
||||
@@ -52,7 +52,7 @@ func TestCheckHealth(t *testing.T) {
|
||||
ConfigDir: configDir,
|
||||
Executor: executor,
|
||||
}
|
||||
c := New(context, "ns", "", "myversion", cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
c := New(context, "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(1)
|
||||
logger.Infof("initial mons: %v", c.clusterInfo.Monitors)
|
||||
@@ -96,7 +96,7 @@ func TestCheckHealthNotFound(t *testing.T) {
|
||||
ConfigDir: configDir,
|
||||
Executor: executor,
|
||||
}
|
||||
c := New(context, "ns", "", "myversion", cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
c := New(context, "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(2)
|
||||
c.waitForStart = false
|
||||
@@ -165,7 +165,7 @@ func TestCheckHealthTwoMonsOneNode(t *testing.T) {
|
||||
ConfigDir: configDir,
|
||||
Executor: executor,
|
||||
}
|
||||
c := New(context, "ns", "", "myversion", cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
c := New(context, "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(2)
|
||||
c.waitForStart = false
|
||||
@@ -276,7 +276,7 @@ func TestCheckMonsValid(t *testing.T) {
|
||||
ConfigDir: configDir,
|
||||
Executor: executor,
|
||||
}
|
||||
c := New(context, "ns", "", "myversion", cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
c := New(context, "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(1)
|
||||
c.waitForStart = false
|
||||
@@ -361,7 +361,7 @@ func TestAddRemoveMons(t *testing.T) {
|
||||
ConfigDir: configDir,
|
||||
Executor: executor,
|
||||
}
|
||||
c := New(context, "ns", "", "myversion", cephv1beta1.MonSpec{Count: 5, AllowMultiplePerNode: true},
|
||||
c := New(context, "ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 5, AllowMultiplePerNode: true},
|
||||
rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.maxMonID = 0
|
||||
c.clusterInfo = test.CreateConfigDir(0)
|
||||
|
||||
@@ -75,7 +75,8 @@ type Cluster struct {
|
||||
context *clusterd.Context
|
||||
Namespace string
|
||||
Keyring string
|
||||
Version string
|
||||
rookVersion string
|
||||
cephVersion cephv1beta1.CephVersionSpec
|
||||
Count int
|
||||
AllowMultiplePerNode bool
|
||||
MonCountMutex sync.Mutex
|
||||
@@ -120,14 +121,15 @@ type NodeInfo struct {
|
||||
}
|
||||
|
||||
// New creates an instance of a mon cluster
|
||||
func New(context *clusterd.Context, namespace, dataDirHostPath, version string, mon cephv1beta1.MonSpec, placement rookalpha.Placement, hostNetwork bool,
|
||||
resources v1.ResourceRequirements, ownerRef metav1.OwnerReference) *Cluster {
|
||||
func New(context *clusterd.Context, namespace, dataDirHostPath, rookVersion string, cephVersion cephv1beta1.CephVersionSpec, mon cephv1beta1.MonSpec,
|
||||
placement rookalpha.Placement, hostNetwork bool, resources v1.ResourceRequirements, ownerRef metav1.OwnerReference) *Cluster {
|
||||
return &Cluster{
|
||||
context: context,
|
||||
placement: placement,
|
||||
dataDirHostPath: dataDirHostPath,
|
||||
Namespace: namespace,
|
||||
Version: version,
|
||||
rookVersion: rookVersion,
|
||||
cephVersion: cephVersion,
|
||||
Count: mon.Count,
|
||||
AllowMultiplePerNode: mon.AllowMultiplePerNode,
|
||||
maxMonID: -1,
|
||||
|
||||
@@ -68,7 +68,7 @@ func newCluster(context *clusterd.Context, namespace string, hostNetwork bool, r
|
||||
HostNetwork: hostNetwork,
|
||||
context: context,
|
||||
Namespace: namespace,
|
||||
Version: "myversion",
|
||||
rookVersion: "myversion",
|
||||
Count: 3,
|
||||
AllowMultiplePerNode: true,
|
||||
maxMonID: -1,
|
||||
@@ -166,7 +166,7 @@ func TestSaveMonEndpoints(t *testing.T) {
|
||||
clientset := test.New(1)
|
||||
configDir, _ := ioutil.TempDir("", "")
|
||||
defer os.RemoveAll(configDir)
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", "", "myversion",
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", "", "myversion", cephv1beta1.CephVersionSpec{},
|
||||
cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false,
|
||||
v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(1)
|
||||
@@ -240,7 +240,7 @@ func TestNameToIndex(t *testing.T) {
|
||||
|
||||
func TestAvailableMonNodes(t *testing.T) {
|
||||
clientset := test.New(1)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion",
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1beta1.CephVersionSpec{},
|
||||
cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{},
|
||||
false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(0)
|
||||
@@ -267,7 +267,7 @@ func TestAvailableMonNodes(t *testing.T) {
|
||||
|
||||
func TestAvailableNodesInUse(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion",
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1beta1.CephVersionSpec{},
|
||||
cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{},
|
||||
false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(0)
|
||||
@@ -308,7 +308,7 @@ func TestAvailableNodesInUse(t *testing.T) {
|
||||
|
||||
func TestTaintedNodes(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion",
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1beta1.CephVersionSpec{},
|
||||
cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{},
|
||||
false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(0)
|
||||
@@ -343,7 +343,7 @@ func TestTaintedNodes(t *testing.T) {
|
||||
|
||||
func TestNodeAffinity(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion",
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1beta1.CephVersionSpec{},
|
||||
cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{},
|
||||
false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(0)
|
||||
@@ -385,7 +385,7 @@ func TestNodeAffinity(t *testing.T) {
|
||||
|
||||
func TestHostNetwork(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion",
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1beta1.CephVersionSpec{},
|
||||
cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{},
|
||||
false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(0)
|
||||
@@ -417,7 +417,7 @@ func TestGetNodeInfoFromNode(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion",
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1beta1.CephVersionSpec{},
|
||||
cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{},
|
||||
true, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(0)
|
||||
@@ -445,10 +445,8 @@ func TestHostNetworkPortIncrease(t *testing.T) {
|
||||
|
||||
configDir, _ := ioutil.TempDir("", "")
|
||||
defer os.RemoveAll(configDir)
|
||||
c := New(&clusterd.Context{
|
||||
Clientset: clientset,
|
||||
ConfigDir: configDir,
|
||||
}, "ns", "", "myversion", cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir},
|
||||
"ns", "", "myversion", cephv1beta1.CephVersionSpec{}, cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
rookalpha.Placement{}, true, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(0)
|
||||
|
||||
|
||||
@@ -142,7 +142,7 @@ func (c *Cluster) makeConfigInitContainer(monConfig *monConfig) v1.Container {
|
||||
fmt.Sprintf("--port=%d", monConfig.Port),
|
||||
fmt.Sprintf("--fsid=%s", c.clusterInfo.FSID),
|
||||
},
|
||||
Image: k8sutil.MakeRookImage(c.Version),
|
||||
Image: k8sutil.MakeRookImage(c.rookVersion),
|
||||
Env: []v1.EnvVar{
|
||||
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
|
||||
{Name: k8sutil.PublicIPEnvVar, Value: monConfig.PublicIP},
|
||||
@@ -186,7 +186,7 @@ func (c *Cluster) makeMonmapInitContainer(monConfig *monConfig) v1.Container {
|
||||
},
|
||||
monmapAddMonArgs...,
|
||||
),
|
||||
Image: k8sutil.MakeRookImage(c.Version), // TODO: ceph:<vers> image
|
||||
Image: c.cephVersion.Image,
|
||||
VolumeMounts: opspec.CephVolumeMounts(),
|
||||
SecurityContext: podSecurityContext(),
|
||||
// monmap creation does not require ports to be exposed
|
||||
@@ -215,7 +215,7 @@ func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) v1.Container {
|
||||
},
|
||||
c.cephMonCommonArgs(monConfig)...,
|
||||
),
|
||||
Image: k8sutil.MakeRookImage(c.Version), // TODO: ceph:<vers> image
|
||||
Image: c.cephVersion.Image,
|
||||
VolumeMounts: opspec.CephVolumeMounts(),
|
||||
SecurityContext: podSecurityContext(),
|
||||
// filesystem creation does not require ports to be exposed
|
||||
@@ -246,7 +246,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container {
|
||||
},
|
||||
c.cephMonCommonArgs(monConfig)...,
|
||||
),
|
||||
Image: k8sutil.MakeRookImage(c.Version), // TODO: ceph:<vers> image
|
||||
Image: c.cephVersion.Image,
|
||||
VolumeMounts: opspec.CephVolumeMounts(),
|
||||
SecurityContext: podSecurityContext(),
|
||||
Ports: []v1.ContainerPort{
|
||||
|
||||
@@ -53,6 +53,7 @@ func testPodSpec(t *testing.T, dataDir string) {
|
||||
"ns",
|
||||
dataDir,
|
||||
"rook/rook:myversion",
|
||||
cephv1beta1.CephVersionSpec{Image: "ceph/ceph:myceph"},
|
||||
cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
rookalpha.Placement{},
|
||||
false,
|
||||
@@ -123,7 +124,7 @@ func testPodSpec(t *testing.T, dataDir string) {
|
||||
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
|
||||
|
||||
// All ceph images have the same image, no envs, and the same volume mounts
|
||||
cephImage := "rook/rook:myversion"
|
||||
cephImage := "ceph/ceph:myceph"
|
||||
cephEnvs := 0
|
||||
cephVolumeMountNames := []string{
|
||||
"rook-data",
|
||||
|
||||
@@ -22,9 +22,9 @@ import (
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
@@ -60,7 +60,8 @@ type Cluster struct {
|
||||
Namespace string
|
||||
placement rookalpha.Placement
|
||||
Keyring string
|
||||
Version string
|
||||
rookVersion string
|
||||
cephVersion cephv1beta1.CephVersionSpec
|
||||
Storage rookalpha.StorageScopeSpec
|
||||
dataDirHostPath string
|
||||
HostNetwork bool
|
||||
@@ -73,8 +74,9 @@ type Cluster struct {
|
||||
// New creates an instance of the OSD manager
|
||||
func New(
|
||||
context *clusterd.Context,
|
||||
namespace,
|
||||
version,
|
||||
namespace string,
|
||||
rookVersion string,
|
||||
cephVersion cephv1beta1.CephVersionSpec,
|
||||
serviceAccount string,
|
||||
storageSpec rookalpha.StorageScopeSpec,
|
||||
dataDirHostPath string,
|
||||
@@ -95,7 +97,8 @@ func New(
|
||||
Namespace: namespace,
|
||||
serviceAccount: serviceAccount,
|
||||
placement: placement,
|
||||
Version: version,
|
||||
rookVersion: rookVersion,
|
||||
cephVersion: cephVersion,
|
||||
Storage: storageSpec,
|
||||
dataDirHostPath: dataDirHostPath,
|
||||
HostNetwork: hostNetwork,
|
||||
@@ -255,7 +258,7 @@ func (c *Cluster) startProvisioning(config *provisionConfig) {
|
||||
}
|
||||
}
|
||||
|
||||
if !c.updateJob(job, n.Name, config, "provision") {
|
||||
if !c.runJob(job, n.Name, config, "provision") {
|
||||
if err = discover.FreeDevices(c.context, n.Name, c.Namespace); err != nil {
|
||||
logger.Warningf("failed to free devices: %s", err)
|
||||
}
|
||||
@@ -263,27 +266,8 @@ func (c *Cluster) startProvisioning(config *provisionConfig) {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) updateJob(job *batch.Job, nodeName string, config *provisionConfig, action string) bool {
|
||||
// check if the job was already created and what its status is
|
||||
existingJob, err := c.context.Clientset.Batch().Jobs(c.Namespace).Get(job.Name, metav1.GetOptions{})
|
||||
if err != nil && !errors.IsNotFound(err) {
|
||||
config.addError("failed to detect %s job for node %s. %+v", action, nodeName, err)
|
||||
} else if err == nil {
|
||||
// delete the job that already exists from a previous run
|
||||
if existingJob.Status.Active > 0 {
|
||||
logger.Infof("Found previous %s job for node %s. Status=%+v", action, nodeName, existingJob.Status)
|
||||
return true
|
||||
}
|
||||
|
||||
logger.Infof("Removing previous %s job for node %s to start a new one", action, nodeName)
|
||||
err := c.deleteBatchJob(existingJob.Name)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to remove job %s. %+v", nodeName, err)
|
||||
}
|
||||
}
|
||||
|
||||
_, err = c.context.Clientset.Batch().Jobs(c.Namespace).Create(job)
|
||||
if err != nil {
|
||||
func (c *Cluster) runJob(job *batch.Job, nodeName string, config *provisionConfig, action string) bool {
|
||||
if err := k8sutil.RunReplaceableJob(c.context.Clientset, job); err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
// we failed to create job, update the orchestration status for this node
|
||||
message := fmt.Sprintf("failed to create %s job for node %s. %+v", action, nodeName, err)
|
||||
@@ -298,32 +282,6 @@ func (c *Cluster) updateJob(job *batch.Job, nodeName string, config *provisionCo
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *Cluster) deleteBatchJob(name string) error {
|
||||
propagation := metav1.DeletePropagationForeground
|
||||
gracePeriod := int64(0)
|
||||
options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
|
||||
|
||||
if err := c.context.Clientset.Batch().Jobs(c.Namespace).Delete(name, options); err != nil {
|
||||
return fmt.Errorf("failed to remove previous provisioning job for node %s. %+v", name, err)
|
||||
}
|
||||
|
||||
retries := 20
|
||||
sleepInterval := 2 * time.Second
|
||||
for i := 0; i < retries; i++ {
|
||||
_, err := c.context.Clientset.Batch().Jobs(c.Namespace).Get(name, metav1.GetOptions{})
|
||||
if err != nil && errors.IsNotFound(err) {
|
||||
logger.Infof("batch job %s deleted", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Infof("batch job %s still exists", name)
|
||||
time.Sleep(sleepInterval)
|
||||
}
|
||||
|
||||
logger.Warningf("gave up waiting for batch job %s to be deleted", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) startOSDDaemonsOnNode(nodeName string, config *provisionConfig, configMap *v1.ConfigMap, status *OrchestrationStatus) {
|
||||
|
||||
osds := status.OSDs
|
||||
@@ -461,7 +419,7 @@ func (c *Cluster) cleanupRemovedNode(config *provisionConfig, nodeName, crushNam
|
||||
return
|
||||
}
|
||||
|
||||
if !c.updateJob(job, nodeName, config, "remove") {
|
||||
if !c.runJob(job, nodeName, config, "remove") {
|
||||
status := OrchestrationStatus{Status: OrchestrationStatusCompleted, Message: fmt.Sprintf("failed to cleanup osd config on node %s", nodeName)}
|
||||
if err := c.updateNodeStatus(nodeName, status); err != nil {
|
||||
config.addError("failed to update node %s status. %+v", nodeName, err)
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
discoverDaemon "github.com/rook/rook/pkg/daemon/discover"
|
||||
@@ -40,7 +41,7 @@ import (
|
||||
|
||||
func TestStart(t *testing.T) {
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "",
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", cephv1beta1.CephVersionSpec{}, "",
|
||||
rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
// Start the first time
|
||||
@@ -89,7 +90,7 @@ func createNode(nodeName string, condition v1.NodeConditionType, clientset *fake
|
||||
|
||||
func TestLegacyDeployment(t *testing.T) {
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "myversion", "",
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "myversion", cephv1beta1.CephVersionSpec{}, "",
|
||||
rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
osdID := 23
|
||||
@@ -144,7 +145,7 @@ func TestAddRemoveNode(t *testing.T) {
|
||||
statusMapWatcher := watch.NewFake()
|
||||
clientset.PrependWatchReactor("configmaps", k8stesting.DefaultWatchReactor(statusMapWatcher, nil))
|
||||
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", "",
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", cephv1beta1.CephVersionSpec{}, "",
|
||||
storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
// kick off the start of the orchestration in a goroutine
|
||||
@@ -229,7 +230,7 @@ func TestAddRemoveNode(t *testing.T) {
|
||||
|
||||
// modify the storage spec to remove the node from the cluster
|
||||
storageSpec.Nodes = []rookalpha.Node{}
|
||||
c = New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: mockExec}, "ns-add-remove", "myversion", "",
|
||||
c = New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: mockExec}, "ns-add-remove", "myversion", cephv1beta1.CephVersionSpec{}, "",
|
||||
storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
// reset the orchestration status watcher
|
||||
@@ -268,7 +269,7 @@ func TestGetIDFromDeployment(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDiscoverOSDs(t *testing.T) {
|
||||
c := New(&clusterd.Context{}, "ns", "myversion", "",
|
||||
c := New(&clusterd.Context{}, "ns", "myversion", cephv1beta1.CephVersionSpec{}, "",
|
||||
rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
node1 := "n1"
|
||||
node2 := "n2"
|
||||
@@ -336,7 +337,7 @@ func TestAddNodeFailure(t *testing.T) {
|
||||
cmErr := createDiscoverConfigmap(nodeName, "rook-system", clientset)
|
||||
assert.Nil(t, cmErr)
|
||||
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", "",
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", cephv1beta1.CephVersionSpec{}, "",
|
||||
storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
// kick off the start of the orchestration in a goroutine
|
||||
|
||||
@@ -44,6 +44,8 @@ const (
|
||||
osdWalSizeEnvVarName = "ROOK_OSD_WAL_SIZE"
|
||||
osdJournalSizeEnvVarName = "ROOK_OSD_JOURNAL_SIZE"
|
||||
osdMetadataDeviceEnvVarName = "ROOK_METADATA_DEVICE"
|
||||
rookBinariesMountPath = "/rook"
|
||||
rookBinariesVolumeName = "rook-binaries"
|
||||
)
|
||||
|
||||
func (c *Cluster) makeJob(nodeName string, devices []rookalpha.Device,
|
||||
@@ -138,6 +140,7 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
|
||||
|
||||
var command []string
|
||||
var args []string
|
||||
var copyBinariesContainer *v1.Container
|
||||
if !osd.IsDirectory && osd.IsFileStore {
|
||||
// All scenarios except one can call the ceph-osd daemon directly. The one different scenario is when
|
||||
// filestore is running on a device. Rook needs to mount the device, run the ceph-osd daemon, and then
|
||||
@@ -145,27 +148,21 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
|
||||
// for this scenario, we will copy the binaries necessary to a mount, which will then be mounted
|
||||
// to the daemon container.
|
||||
sourcePath := path.Join("/dev/disk/by-partuuid", osd.DevicePartUUID)
|
||||
mountPath := "/rook"
|
||||
command = []string{path.Join(mountPath, "tini")}
|
||||
command = []string{path.Join(rookBinariesMountPath, "tini")}
|
||||
args = append([]string{
|
||||
"--", path.Join(mountPath, "rook"),
|
||||
"--", path.Join(rookBinariesMountPath, "rook"),
|
||||
"ceph", "osd", "filestore-device",
|
||||
"--source-path", sourcePath,
|
||||
"--mount-path", osd.DataPath,
|
||||
"--"},
|
||||
commonArgs...)
|
||||
|
||||
// To get rook inside the container, the config init container needs to copy "tini" and "rook" binaries into a volume.
|
||||
// Set the config flag so rook will copy the binaries.
|
||||
configEnvVars = append(configEnvVars, v1.EnvVar{Name: "ROOK_COPY_BINARIES_PATH", Value: mountPath})
|
||||
var copyBinariesVolume v1.Volume
|
||||
copyBinariesVolume, copyBinariesContainer = c.getCopyBinariesContainer()
|
||||
// Add the volume to the spec and the mount to the daemon container
|
||||
volumes = append(volumes, copyBinariesVolume)
|
||||
volumeMounts = append(volumeMounts, copyBinariesContainer.VolumeMounts[0])
|
||||
|
||||
// Create the volume and mount that will be shared between the init container and the daemon container
|
||||
volumeName := "rookbinaries"
|
||||
binariesVolume := v1.Volume{Name: volumeName, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}
|
||||
volumes = append(volumes, binariesVolume)
|
||||
binariesMount := v1.VolumeMount{Name: volumeName, MountPath: mountPath}
|
||||
volumeMounts = append(volumeMounts, binariesMount)
|
||||
configVolumeMounts = append(configVolumeMounts, binariesMount)
|
||||
} else {
|
||||
// other osds can launch the osd daemon directly
|
||||
command = []string{"ceph-osd"}
|
||||
@@ -220,7 +217,7 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
|
||||
{
|
||||
Args: []string{"ceph", "osd", "init"},
|
||||
Name: opspec.ConfigInitContainerName,
|
||||
Image: k8sutil.MakeRookImage(c.Version),
|
||||
Image: k8sutil.MakeRookImage(c.rookVersion),
|
||||
VolumeMounts: configVolumeMounts,
|
||||
Env: configEnvVars,
|
||||
SecurityContext: securityContext,
|
||||
@@ -231,7 +228,7 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
|
||||
Command: command,
|
||||
Args: args,
|
||||
Name: "osd",
|
||||
Image: k8sutil.MakeRookImage(c.Version),
|
||||
Image: c.cephVersion.Image,
|
||||
VolumeMounts: volumeMounts,
|
||||
Env: envVars,
|
||||
Resources: resources,
|
||||
@@ -244,15 +241,36 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
|
||||
Replicas: &replicaCount,
|
||||
},
|
||||
}
|
||||
if copyBinariesContainer != nil {
|
||||
deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, *copyBinariesContainer)
|
||||
}
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &deployment.ObjectMeta, &c.ownerRef)
|
||||
c.placement.ApplyToPodSpec(&deployment.Spec.Template.Spec)
|
||||
return deployment, nil
|
||||
}
|
||||
|
||||
// To get rook inside the container, the config init container needs to copy "tini" and "rook" binaries into a volume.
|
||||
// Get the config flag so rook will copy the binaries and create the volume and mount that will be shared between
|
||||
// the init container and the daemon container
|
||||
func (c *Cluster) getCopyBinariesContainer() (v1.Volume, *v1.Container) {
|
||||
volume := v1.Volume{Name: rookBinariesVolumeName, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}
|
||||
mount := v1.VolumeMount{Name: rookBinariesVolumeName, MountPath: rookBinariesMountPath}
|
||||
|
||||
return volume, &v1.Container{
|
||||
Args: []string{"ceph", "osd", "copybins"},
|
||||
Name: "copy-bins",
|
||||
Image: k8sutil.MakeRookImage(c.rookVersion),
|
||||
VolumeMounts: []v1.VolumeMount{mount},
|
||||
Env: []v1.EnvVar{{Name: "ROOK_PATH", Value: rookBinariesMountPath}},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) provisionPodTemplateSpec(devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements,
|
||||
storeConfig config.StoreConfig, metadataDevice, nodeName, location string, restart v1.RestartPolicy) (*v1.PodTemplateSpec, error) {
|
||||
|
||||
volumes := opspec.PodVolumes(c.dataDirHostPath)
|
||||
copyBinariesVolume, copyBinariesContainer := c.getCopyBinariesContainer()
|
||||
|
||||
volumes := append(opspec.PodVolumes(c.dataDirHostPath), copyBinariesVolume)
|
||||
|
||||
// by default, don't define any volume config unless it is required
|
||||
if len(devices) > 0 || selection.DeviceFilter != "" || selection.GetUseAllDevices() || metadataDevice != "" {
|
||||
@@ -278,10 +296,13 @@ func (c *Cluster) provisionPodTemplateSpec(devices []rookalpha.Device, selection
|
||||
|
||||
podSpec := v1.PodSpec{
|
||||
ServiceAccountName: c.serviceAccount,
|
||||
Containers: []v1.Container{c.provisionOSDContainer(devices, selection, resources, storeConfig, metadataDevice, nodeName, location)},
|
||||
RestartPolicy: restart,
|
||||
Volumes: volumes,
|
||||
HostNetwork: c.HostNetwork,
|
||||
Containers: []v1.Container{
|
||||
*copyBinariesContainer,
|
||||
c.provisionOSDContainer(devices, selection, resources, storeConfig, metadataDevice, nodeName, location, copyBinariesContainer.VolumeMounts[0]),
|
||||
},
|
||||
RestartPolicy: restart,
|
||||
Volumes: volumes,
|
||||
HostNetwork: c.HostNetwork,
|
||||
}
|
||||
if c.HostNetwork {
|
||||
podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
@@ -339,7 +360,7 @@ func (c *Cluster) getConfigEnvVars(storeConfig config.StoreConfig, dataDir, node
|
||||
}
|
||||
|
||||
func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements,
|
||||
storeConfig config.StoreConfig, metadataDevice, nodeName, location string) v1.Container {
|
||||
storeConfig config.StoreConfig, metadataDevice, nodeName, location string, copyBinariesMount v1.VolumeMount) v1.Container {
|
||||
|
||||
envVars := c.getConfigEnvVars(storeConfig, k8sutil.DataDir, nodeName, location)
|
||||
devMountNeeded := false
|
||||
@@ -366,7 +387,7 @@ func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection ro
|
||||
devMountNeeded = true
|
||||
}
|
||||
|
||||
volumeMounts := opspec.CephVolumeMounts()
|
||||
volumeMounts := append(opspec.CephVolumeMounts(), copyBinariesMount)
|
||||
if devMountNeeded {
|
||||
devMount := v1.VolumeMount{Name: "devices", MountPath: "/dev"}
|
||||
volumeMounts = append(volumeMounts, devMount)
|
||||
@@ -395,11 +416,12 @@ func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection ro
|
||||
runAsUser := int64(0)
|
||||
runAsNonRoot := false
|
||||
readOnlyRootFilesystem := false
|
||||
|
||||
return v1.Container{
|
||||
// Set the hostname so we have the pod's host in the crush map rather than the pod container name
|
||||
Args: []string{"ceph", "osd", "provision"},
|
||||
Name: appName,
|
||||
Image: k8sutil.MakeRookImage(c.Version),
|
||||
Command: []string{path.Join(rookBinariesMountPath, "tini")},
|
||||
Args: []string{"--", path.Join(rookBinariesMountPath, "rook"), "ceph", "osd", "provision"},
|
||||
Name: "provision",
|
||||
Image: c.cephVersion.Image,
|
||||
VolumeMounts: volumeMounts,
|
||||
Env: envVars,
|
||||
SecurityContext: &v1.SecurityContext{
|
||||
|
||||
@@ -20,6 +20,7 @@ package osd
|
||||
import (
|
||||
"testing"
|
||||
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
|
||||
@@ -35,14 +36,23 @@ import (
|
||||
)
|
||||
|
||||
func TestPodContainer(t *testing.T) {
|
||||
cluster := &Cluster{Namespace: "myosd", Version: "23"}
|
||||
cluster := &Cluster{Namespace: "myosd", rookVersion: "23"}
|
||||
c, err := cluster.provisionPodTemplateSpec([]rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "node", "", v1.RestartPolicyAlways)
|
||||
assert.NotNil(t, c)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(c.Spec.Containers))
|
||||
assert.Equal(t, 2, len(c.Spec.Containers))
|
||||
container := c.Spec.Containers[0]
|
||||
logger.Infof("container: %+v", container)
|
||||
assert.Equal(t, "ceph", container.Args[0])
|
||||
assert.Equal(t, "osd", container.Args[1])
|
||||
assert.Equal(t, "copybins", container.Args[2])
|
||||
container = c.Spec.Containers[1]
|
||||
assert.Equal(t, "/rook/tini", container.Command[0])
|
||||
assert.Equal(t, "--", container.Args[0])
|
||||
assert.Equal(t, "/rook/rook", container.Args[1])
|
||||
assert.Equal(t, "ceph", container.Args[2])
|
||||
assert.Equal(t, "osd", container.Args[3])
|
||||
assert.Equal(t, "provision", container.Args[4])
|
||||
}
|
||||
|
||||
func TestDaemonset(t *testing.T) {
|
||||
@@ -62,7 +72,8 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
|
||||
}
|
||||
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "mysa",
|
||||
cephVersion := cephv1beta1.CephVersionSpec{Image: "ceph/ceph:v12.2.8"}
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", cephVersion, "mysa",
|
||||
storageSpec, dataDir, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
devMountNeeded := deviceName != "" || allDevices
|
||||
@@ -110,7 +121,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
|
||||
|
||||
assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers))
|
||||
cont := deployment.Spec.Template.Spec.Containers[0]
|
||||
assert.Equal(t, "rook/rook:myversion", cont.Image)
|
||||
assert.Equal(t, cephVersion.Image, cont.Image)
|
||||
if devMountNeeded {
|
||||
assert.Equal(t, 3, len(cont.VolumeMounts))
|
||||
} else {
|
||||
@@ -149,7 +160,7 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) {
|
||||
}
|
||||
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "",
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", cephv1beta1.CephVersionSpec{}, "",
|
||||
storageSpec, "/var/lib/rook", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
n := c.Storage.ResolveNode(storageSpec.Nodes[0].Name)
|
||||
@@ -236,7 +247,7 @@ func TestStorageSpecConfig(t *testing.T) {
|
||||
}
|
||||
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "",
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", cephv1beta1.CephVersionSpec{}, "",
|
||||
storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
n := c.Storage.ResolveNode(storageSpec.Nodes[0].Name)
|
||||
@@ -249,6 +260,9 @@ func TestStorageSpecConfig(t *testing.T) {
|
||||
assert.Equal(t, "rook-ceph-osd-prepare-node1", job.ObjectMeta.Name)
|
||||
container := job.Spec.Template.Spec.Containers[0]
|
||||
assert.NotNil(t, container)
|
||||
verifyEnvVar(t, container.Env, "ROOK_PATH", "/rook", true)
|
||||
container = job.Spec.Template.Spec.Containers[1]
|
||||
assert.NotNil(t, container)
|
||||
verifyEnvVar(t, container.Env, "ROOK_OSD_STORE", "bluestore", true)
|
||||
verifyEnvVar(t, container.Env, "ROOK_OSD_DATABASE_SIZE", "10", true)
|
||||
verifyEnvVar(t, container.Env, "ROOK_OSD_WAL_SIZE", "20", true)
|
||||
@@ -283,7 +297,7 @@ func TestHostNetwork(t *testing.T) {
|
||||
}
|
||||
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "",
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", cephv1beta1.CephVersionSpec{}, "",
|
||||
storageSpec, "", rookalpha.Placement{}, true, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
n := c.Storage.ResolveNode(storageSpec.Nodes[0].Name)
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
@@ -35,7 +36,7 @@ import (
|
||||
|
||||
func TestOrchestrationStatus(t *testing.T) {
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "",
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", cephv1beta1.CephVersionSpec{}, "",
|
||||
rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
kv := k8sutil.NewConfigMapKVStore(c.Namespace, clientset, metav1.OwnerReference{})
|
||||
nodeName := "mynode"
|
||||
|
||||
@@ -64,7 +64,8 @@ var filesystemResourceRookLegacy = opkit.CustomResource{
|
||||
// FilesystemController represents a controller for file system custom resources
|
||||
type FilesystemController struct {
|
||||
context *clusterd.Context
|
||||
rookImage string
|
||||
rookVersion string
|
||||
cephVersion cephv1beta1.CephVersionSpec
|
||||
hostNetwork bool
|
||||
ownerRef metav1.OwnerReference
|
||||
}
|
||||
@@ -72,13 +73,15 @@ type FilesystemController struct {
|
||||
// NewFilesystemController create controller for watching file system custom resources created
|
||||
func NewFilesystemController(
|
||||
context *clusterd.Context,
|
||||
rookImage string,
|
||||
rookVersion string,
|
||||
cephVersion cephv1beta1.CephVersionSpec,
|
||||
hostNetwork bool,
|
||||
ownerRef metav1.OwnerReference,
|
||||
) *FilesystemController {
|
||||
return &FilesystemController{
|
||||
context: context,
|
||||
rookImage: rookImage,
|
||||
rookVersion: rookVersion,
|
||||
cephVersion: cephVersion,
|
||||
hostNetwork: hostNetwork,
|
||||
ownerRef: ownerRef,
|
||||
}
|
||||
@@ -117,7 +120,7 @@ func (c *FilesystemController) onAdd(obj interface{}) {
|
||||
return
|
||||
}
|
||||
|
||||
err = createFilesystem(c.context, *filesystem, c.rookImage, c.hostNetwork, c.filesystemOwners(filesystem))
|
||||
err = createFilesystem(c.context, *filesystem, c.rookVersion, c.cephVersion, c.hostNetwork, c.filesystemOwners(filesystem))
|
||||
if err != nil {
|
||||
logger.Errorf("failed to create file system %s: %+v", filesystem.Name, err)
|
||||
}
|
||||
@@ -149,7 +152,7 @@ func (c *FilesystemController) onUpdate(oldObj, newObj interface{}) {
|
||||
|
||||
// if the file system is modified, allow the file system to be created if it wasn't already
|
||||
logger.Infof("updating filesystem %s", newFS.Name)
|
||||
err = createFilesystem(c.context, *newFS, c.rookImage, c.hostNetwork, c.filesystemOwners(newFS))
|
||||
err = createFilesystem(c.context, *newFS, c.rookVersion, c.cephVersion, c.hostNetwork, c.filesystemOwners(newFS))
|
||||
if err != nil {
|
||||
logger.Errorf("failed to create (modify) file system %s: %+v", newFS.Name, err)
|
||||
}
|
||||
|
||||
@@ -83,7 +83,7 @@ func TestMigrateFilesystemObject(t *testing.T) {
|
||||
Clientset: clientset,
|
||||
RookClientset: rookfake.NewSimpleClientset(legacyFilesystem),
|
||||
}
|
||||
controller := NewFilesystemController(context, "", false, metav1.OwnerReference{})
|
||||
controller := NewFilesystemController(context, "", cephv1beta1.CephVersionSpec{}, false, metav1.OwnerReference{})
|
||||
|
||||
// convert the legacy filesystem object in memory and assert that a migration is needed
|
||||
convertedFilesystem, migrationNeeded, err := getFilesystemObject(legacyFilesystem)
|
||||
|
||||
@@ -32,7 +32,8 @@ import (
|
||||
func createFilesystem(
|
||||
context *clusterd.Context,
|
||||
fs cephv1beta1.Filesystem,
|
||||
version string,
|
||||
rookVersion string,
|
||||
cephVersion cephv1beta1.CephVersionSpec,
|
||||
hostNetwork bool,
|
||||
ownerRefs []metav1.OwnerReference,
|
||||
) error {
|
||||
@@ -55,7 +56,7 @@ func createFilesystem(
|
||||
}
|
||||
|
||||
logger.Infof("start running mdses for file system %s", fs.Name)
|
||||
c := newCluster(context, version, hostNetwork, fs, filesystem, ownerRefs)
|
||||
c := newCluster(context, rookVersion, cephVersion, hostNetwork, fs, filesystem, ownerRefs)
|
||||
if err := c.start(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -106,12 +106,12 @@ func TestCreateFilesystem(t *testing.T) {
|
||||
}
|
||||
|
||||
// start a basic cluster
|
||||
err := createFilesystem(context, fs, "v0.1", false, []metav1.OwnerReference{})
|
||||
err := createFilesystem(context, fs, "v0.1", cephv1beta1.CephVersionSpec{}, false, []metav1.OwnerReference{})
|
||||
assert.Nil(t, err)
|
||||
validateStart(t, context, fs)
|
||||
|
||||
// starting again should be a no-op
|
||||
err = createFilesystem(context, fs, "v0.1", false, []metav1.OwnerReference{})
|
||||
err = createFilesystem(context, fs, "v0.1", cephv1beta1.CephVersionSpec{}, false, []metav1.OwnerReference{})
|
||||
assert.Nil(t, err)
|
||||
validateStart(t, context, fs)
|
||||
|
||||
@@ -130,7 +130,7 @@ func TestCreateFilesystem(t *testing.T) {
|
||||
Clientset: testop.New(3)}
|
||||
|
||||
//Create another filesystem which should fail
|
||||
err = createFilesystem(context, fs, "v0.1", false, []metav1.OwnerReference{})
|
||||
err = createFilesystem(context, fs, "v0.1", cephv1beta1.CephVersionSpec{}, false, []metav1.OwnerReference{})
|
||||
assert.Equal(t, "failed to create file system myfs: Cannot create multiple filesystems. Enable ROOK_ALLOW_MULTIPLE_FILESYSTEMS env variable to create more than one", err.Error())
|
||||
}
|
||||
|
||||
|
||||
@@ -44,7 +44,8 @@ const (
|
||||
|
||||
type cluster struct {
|
||||
context *clusterd.Context
|
||||
Version string
|
||||
rookVersion string
|
||||
cephVersion cephv1beta1.CephVersionSpec
|
||||
HostNetwork bool
|
||||
fs cephv1beta1.Filesystem
|
||||
fsID string
|
||||
@@ -53,7 +54,8 @@ type cluster struct {
|
||||
|
||||
func newCluster(
|
||||
context *clusterd.Context,
|
||||
version string,
|
||||
rookVersion string,
|
||||
cephVersion cephv1beta1.CephVersionSpec,
|
||||
hostNetwork bool,
|
||||
fs cephv1beta1.Filesystem,
|
||||
fsdetails *client.CephFilesystemDetails,
|
||||
@@ -61,7 +63,8 @@ func newCluster(
|
||||
) *cluster {
|
||||
return &cluster{
|
||||
context: context,
|
||||
Version: version,
|
||||
rookVersion: rookVersion,
|
||||
cephVersion: cephVersion,
|
||||
HostNetwork: hostNetwork,
|
||||
fs: fs,
|
||||
fsID: strconv.Itoa(fsdetails.ID),
|
||||
|
||||
@@ -79,7 +79,7 @@ func (c *cluster) makeConfigInitContainer(mdsConfig *mdsConfig) v1.Container {
|
||||
"--filesystem-id", c.fsID,
|
||||
"--active-standby", strconv.FormatBool(c.fs.Spec.MetadataServer.ActiveStandby),
|
||||
},
|
||||
Image: k8sutil.MakeRookImage(c.Version),
|
||||
Image: k8sutil.MakeRookImage(c.rookVersion),
|
||||
Env: []v1.EnvVar{
|
||||
// Set '--mds-keyring' flag with an env var sourced from the secret
|
||||
{Name: "ROOK_MDS_KEYRING",
|
||||
@@ -112,7 +112,7 @@ func (c *cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container {
|
||||
"--id", mdsConfig.DaemonName,
|
||||
// do not add the '--cluster/--conf/--keyring' flags; rook wants their default values
|
||||
},
|
||||
Image: k8sutil.MakeRookImage(c.Version),
|
||||
Image: c.cephVersion.Image,
|
||||
Env: k8sutil.ClusterDaemonEnvVars(),
|
||||
VolumeMounts: opspec.CephVolumeMounts(),
|
||||
// TODO: mds doesn't need ports?
|
||||
|
||||
@@ -52,6 +52,7 @@ func testDeploymentObject(hostNetwork bool) *extensions.Deployment {
|
||||
c := newCluster(
|
||||
&clusterd.Context{Clientset: testop.New(1)},
|
||||
"rook/rook:myversion",
|
||||
cephv1beta1.CephVersionSpec{Image: "ceph/ceph:testversion"},
|
||||
hostNetwork,
|
||||
fs,
|
||||
&client.CephFilesystemDetails{ID: 15},
|
||||
@@ -113,7 +114,7 @@ func TestPodSpecs(t *testing.T) {
|
||||
assert.Equal(t, "100", cont.Resources.Limits.Cpu().String())
|
||||
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
|
||||
|
||||
daemonImage := "rook/rook:myversion"
|
||||
daemonImage := "ceph/ceph:testversion"
|
||||
daemonEnvs := len(k8sutil.ClusterDaemonEnvVars())
|
||||
daemonContainerDefinition := cephtest.ContainerTestDefinition{
|
||||
Image: &daemonImage,
|
||||
|
||||
@@ -64,15 +64,17 @@ var ObjectStoreResourceRookLegacy = opkit.CustomResource{
|
||||
type ObjectStoreController struct {
|
||||
context *clusterd.Context
|
||||
rookImage string
|
||||
cephVersion cephv1beta1.CephVersionSpec
|
||||
hostNetwork bool
|
||||
ownerRef metav1.OwnerReference
|
||||
}
|
||||
|
||||
// NewObjectStoreController create controller for watching object store custom resources created
|
||||
func NewObjectStoreController(context *clusterd.Context, rookImage string, hostNetwork bool, ownerRef metav1.OwnerReference) *ObjectStoreController {
|
||||
func NewObjectStoreController(context *clusterd.Context, rookImage string, cephVersion cephv1beta1.CephVersionSpec, hostNetwork bool, ownerRef metav1.OwnerReference) *ObjectStoreController {
|
||||
return &ObjectStoreController{
|
||||
context: context,
|
||||
rookImage: rookImage,
|
||||
cephVersion: cephVersion,
|
||||
hostNetwork: hostNetwork,
|
||||
ownerRef: ownerRef,
|
||||
}
|
||||
@@ -111,7 +113,8 @@ func (c *ObjectStoreController) onAdd(obj interface{}) {
|
||||
return
|
||||
}
|
||||
|
||||
if err = CreateStore(c.context, *objectstore, c.rookImage, c.hostNetwork, c.storeOwners(objectstore)); err != nil {
|
||||
cfg := config{c.context, *objectstore, c.rookImage, c.cephVersion, c.hostNetwork, c.storeOwners(objectstore)}
|
||||
if err = cfg.createStore(); err != nil {
|
||||
logger.Errorf("failed to create object store %s. %+v", objectstore.Name, err)
|
||||
}
|
||||
}
|
||||
@@ -142,7 +145,8 @@ func (c *ObjectStoreController) onUpdate(oldObj, newObj interface{}) {
|
||||
}
|
||||
|
||||
logger.Infof("applying object store %s changes", newStore.Name)
|
||||
if err = UpdateStore(c.context, *newStore, c.rookImage, c.hostNetwork, c.storeOwners(newStore)); err != nil {
|
||||
cfg := config{c.context, *newStore, c.rookImage, c.cephVersion, c.hostNetwork, c.storeOwners(newStore)}
|
||||
if err = cfg.updateStore(); err != nil {
|
||||
logger.Errorf("failed to create (modify) object store %s. %+v", newStore.Name, err)
|
||||
}
|
||||
}
|
||||
@@ -159,7 +163,8 @@ func (c *ObjectStoreController) onDelete(obj interface{}) {
|
||||
return
|
||||
}
|
||||
|
||||
if err = DeleteStore(c.context, *objectstore); err != nil {
|
||||
cfg := config{context: c.context, store: *objectstore}
|
||||
if err = cfg.deleteStore(); err != nil {
|
||||
logger.Errorf("failed to delete object store %s. %+v", objectstore.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,7 +91,7 @@ func TestMigrateObjectStoreObject(t *testing.T) {
|
||||
Clientset: clientset,
|
||||
RookClientset: rookfake.NewSimpleClientset(legacyObjectStore),
|
||||
}
|
||||
controller := NewObjectStoreController(context, "", false, metav1.OwnerReference{})
|
||||
controller := NewObjectStoreController(context, "", cephv1beta1.CephVersionSpec{}, false, metav1.OwnerReference{})
|
||||
|
||||
// convert the legacy objectstore object in memory and assert that a migration is needed
|
||||
convertedObjectStore, migrationNeeded, err := getObjectStoreObject(legacyObjectStore)
|
||||
|
||||
@@ -40,67 +40,76 @@ const (
|
||||
certFilename = "rgw-cert.pem"
|
||||
)
|
||||
|
||||
type config struct {
|
||||
context *clusterd.Context
|
||||
store cephv1beta1.ObjectStore
|
||||
rookVersion string
|
||||
cephVersion cephv1beta1.CephVersionSpec
|
||||
hostNetwork bool
|
||||
ownerRefs []metav1.OwnerReference
|
||||
}
|
||||
|
||||
// Start the rgw manager
|
||||
func CreateStore(context *clusterd.Context, store cephv1beta1.ObjectStore, version string, hostNetwork bool, ownerRefs []metav1.OwnerReference) error {
|
||||
return createOrUpdate(context, store, version, hostNetwork, false, ownerRefs)
|
||||
func (c *config) createStore() error {
|
||||
return c.createOrUpdate(false)
|
||||
}
|
||||
|
||||
func UpdateStore(context *clusterd.Context, store cephv1beta1.ObjectStore, version string, hostNetwork bool, ownerRefs []metav1.OwnerReference) error {
|
||||
return createOrUpdate(context, store, version, hostNetwork, true, ownerRefs)
|
||||
func (c *config) updateStore() error {
|
||||
return c.createOrUpdate(true)
|
||||
}
|
||||
|
||||
func createOrUpdate(context *clusterd.Context, store cephv1beta1.ObjectStore, version string, hostNetwork, update bool, ownerRefs []metav1.OwnerReference) error {
|
||||
func (c *config) createOrUpdate(update bool) error {
|
||||
// validate the object store settings
|
||||
if err := validateStore(context, store); err != nil {
|
||||
return fmt.Errorf("invalid object store %s arguments. %+v", store.Name, err)
|
||||
if err := validateStore(c.context, c.store); err != nil {
|
||||
return fmt.Errorf("invalid object store %s arguments. %+v", c.store.Name, err)
|
||||
}
|
||||
|
||||
// check if the object store already exists
|
||||
exists, err := storeExists(context, store)
|
||||
exists, err := c.storeExists()
|
||||
if err == nil && exists {
|
||||
if !update {
|
||||
logger.Infof("object store %s exists in namespace %s", store.Name, store.Namespace)
|
||||
logger.Infof("object store %s exists in namespace %s", c.store.Name, c.store.Namespace)
|
||||
return nil
|
||||
}
|
||||
logger.Infof("object store %s exists in namespace %s. checking for updates", store.Name, store.Namespace)
|
||||
logger.Infof("object store %s exists in namespace %s. checking for updates", c.store.Name, c.store.Namespace)
|
||||
}
|
||||
|
||||
logger.Infof("creating object store %s in namespace %s", store.Name, store.Namespace)
|
||||
err = createKeyring(context, store, ownerRefs)
|
||||
logger.Infof("creating object store %s in namespace %s", c.store.Name, c.store.Namespace)
|
||||
err = c.createKeyring()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create rgw keyring. %+v", err)
|
||||
}
|
||||
|
||||
// start the service
|
||||
serviceIP, err := startService(context, store, hostNetwork, ownerRefs)
|
||||
serviceIP, err := c.startService()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start rgw service. %+v", err)
|
||||
}
|
||||
|
||||
// create the ceph artifacts for the object store
|
||||
objContext := rgwdaemon.NewContext(context, store.Name, store.Namespace)
|
||||
err = rgwdaemon.CreateObjectStore(objContext, *store.Spec.MetadataPool.ToModel(""), *store.Spec.DataPool.ToModel(""), serviceIP, store.Spec.Gateway.Port)
|
||||
objContext := rgwdaemon.NewContext(c.context, c.store.Name, c.store.Namespace)
|
||||
err = rgwdaemon.CreateObjectStore(objContext, *c.store.Spec.MetadataPool.ToModel(""), *c.store.Spec.DataPool.ToModel(""), serviceIP, c.store.Spec.Gateway.Port)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create pools. %+v", err)
|
||||
}
|
||||
|
||||
if err := startRGWPods(context, store, version, hostNetwork, update, ownerRefs); err != nil {
|
||||
if err := c.startRGWPods(update); err != nil {
|
||||
return fmt.Errorf("failed to start pods. %+v", err)
|
||||
}
|
||||
|
||||
logger.Infof("created object store %s", store.Name)
|
||||
logger.Infof("created object store %s", c.store.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func startRGWPods(context *clusterd.Context, store cephv1beta1.ObjectStore, version string, hostNetwork, update bool, ownerRefs []metav1.OwnerReference) error {
|
||||
func (c *config) startRGWPods(update bool) error {
|
||||
|
||||
// if intended to update, remove the old pods so they can be created with the new spec settings
|
||||
if update {
|
||||
err := k8sutil.DeleteDeployment(context.Clientset, store.Namespace, instanceName(store))
|
||||
err := k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, c.instanceName())
|
||||
if err != nil {
|
||||
logger.Warning(err.Error())
|
||||
}
|
||||
err = k8sutil.DeleteDaemonset(context.Clientset, store.Namespace, instanceName(store))
|
||||
err = k8sutil.DeleteDaemonset(c.context.Clientset, c.store.Namespace, c.instanceName())
|
||||
if err != nil {
|
||||
logger.Warning(err.Error())
|
||||
}
|
||||
@@ -109,12 +118,12 @@ func startRGWPods(context *clusterd.Context, store cephv1beta1.ObjectStore, vers
|
||||
// start the deployment or daemonset
|
||||
var rgwType string
|
||||
var err error
|
||||
if store.Spec.Gateway.AllNodes {
|
||||
if c.store.Spec.Gateway.AllNodes {
|
||||
rgwType = "daemonset"
|
||||
err = startDaemonset(context, store, version, hostNetwork, ownerRefs)
|
||||
err = c.startDaemonset()
|
||||
} else {
|
||||
rgwType = "deployment"
|
||||
err = startDeployment(context, store, version, store.Spec.Gateway.Instances, hostNetwork, ownerRefs)
|
||||
err = c.startDeployment()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -131,59 +140,59 @@ func startRGWPods(context *clusterd.Context, store cephv1beta1.ObjectStore, vers
|
||||
|
||||
// Delete the object store.
|
||||
// WARNING: This is a very destructive action that deletes all metadata and data pools.
|
||||
func DeleteStore(context *clusterd.Context, store cephv1beta1.ObjectStore) error {
|
||||
func (c *config) deleteStore() error {
|
||||
// check if the object store exists
|
||||
exists, err := storeExists(context, store)
|
||||
exists, err := c.storeExists()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to detect if there is an object store to delete. %+v", err)
|
||||
}
|
||||
if !exists {
|
||||
logger.Infof("Object store %s does not exist in namespace %s", store.Name, store.Namespace)
|
||||
logger.Infof("Object store %s does not exist in namespace %s", c.store.Name, c.store.Namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Infof("Deleting object store %s from namespace %s", store.Name, store.Namespace)
|
||||
logger.Infof("Deleting object store %s from namespace %s", c.store.Name, c.store.Namespace)
|
||||
|
||||
var gracePeriod int64
|
||||
propagation := metav1.DeletePropagationForeground
|
||||
options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
|
||||
|
||||
// Delete the rgw service
|
||||
err = context.Clientset.CoreV1().Services(store.Namespace).Delete(instanceName(store), options)
|
||||
err = c.context.Clientset.CoreV1().Services(c.store.Namespace).Delete(c.instanceName(), options)
|
||||
if err != nil && !errors.IsNotFound(err) {
|
||||
logger.Warningf("failed to delete rgw service. %+v", err)
|
||||
}
|
||||
|
||||
// Make a best effort to delete the rgw pods
|
||||
err = k8sutil.DeleteDeployment(context.Clientset, store.Namespace, instanceName(store))
|
||||
err = k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, c.instanceName())
|
||||
if err != nil {
|
||||
logger.Warning(err.Error())
|
||||
}
|
||||
err = k8sutil.DeleteDaemonset(context.Clientset, store.Namespace, instanceName(store))
|
||||
err = k8sutil.DeleteDaemonset(c.context.Clientset, c.store.Namespace, c.instanceName())
|
||||
if err != nil {
|
||||
logger.Warning(err.Error())
|
||||
}
|
||||
|
||||
// Delete the rgw keyring
|
||||
err = context.Clientset.CoreV1().Secrets(store.Namespace).Delete(instanceName(store), options)
|
||||
err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Delete(c.instanceName(), options)
|
||||
if err != nil && !errors.IsNotFound(err) {
|
||||
logger.Warningf("failed to delete rgw secret. %+v", err)
|
||||
}
|
||||
|
||||
// Delete the realm and pools
|
||||
objContext := rgwdaemon.NewContext(context, store.Name, store.Namespace)
|
||||
objContext := rgwdaemon.NewContext(c.context, c.store.Name, c.store.Namespace)
|
||||
err = rgwdaemon.DeleteObjectStore(objContext)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete the realm and pools. %+v", err)
|
||||
}
|
||||
|
||||
logger.Infof("Completed deleting object store %s", store.Name)
|
||||
logger.Infof("Completed deleting object store %s", c.store.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check if the object store exists depending on either the deployment or the daemonset
|
||||
func storeExists(context *clusterd.Context, store cephv1beta1.ObjectStore) (bool, error) {
|
||||
_, err := context.Clientset.ExtensionsV1beta1().Deployments(store.Namespace).Get(instanceName(store), metav1.GetOptions{})
|
||||
func (c *config) storeExists() (bool, error) {
|
||||
_, err := c.context.Clientset.ExtensionsV1beta1().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
|
||||
if err == nil {
|
||||
// the deployment was found
|
||||
return true, nil
|
||||
@@ -192,7 +201,7 @@ func storeExists(context *clusterd.Context, store cephv1beta1.ObjectStore) (bool
|
||||
return false, err
|
||||
}
|
||||
|
||||
_, err = context.Clientset.ExtensionsV1beta1().DaemonSets(store.Namespace).Get(instanceName(store), metav1.GetOptions{})
|
||||
_, err = c.context.Clientset.ExtensionsV1beta1().DaemonSets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
|
||||
if err == nil {
|
||||
// the daemonset was found
|
||||
return true, nil
|
||||
@@ -205,8 +214,8 @@ func storeExists(context *clusterd.Context, store cephv1beta1.ObjectStore) (bool
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func createKeyring(context *clusterd.Context, store cephv1beta1.ObjectStore, ownerRefs []metav1.OwnerReference) error {
|
||||
_, err := context.Clientset.CoreV1().Secrets(store.Namespace).Get(instanceName(store), metav1.GetOptions{})
|
||||
func (c *config) createKeyring() error {
|
||||
_, err := c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
|
||||
if err == nil {
|
||||
logger.Infof("the rgw keyring was already generated")
|
||||
return nil
|
||||
@@ -217,7 +226,7 @@ func createKeyring(context *clusterd.Context, store cephv1beta1.ObjectStore, own
|
||||
|
||||
// create the keyring
|
||||
logger.Infof("generating rgw keyring")
|
||||
keyring, err := createRGWKeyring(context, store.Namespace)
|
||||
keyring, err := c.createRGWKeyring()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create keyring. %+v", err)
|
||||
}
|
||||
@@ -228,14 +237,14 @@ func createKeyring(context *clusterd.Context, store cephv1beta1.ObjectStore, own
|
||||
}
|
||||
secret := &v1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: instanceName(store),
|
||||
Namespace: store.Namespace,
|
||||
Name: c.instanceName(),
|
||||
Namespace: c.store.Namespace,
|
||||
},
|
||||
StringData: secrets,
|
||||
Type: k8sutil.RookType,
|
||||
}
|
||||
k8sutil.SetOwnerRefs(context.Clientset, store.Namespace, &secret.ObjectMeta, ownerRefs)
|
||||
_, err = context.Clientset.CoreV1().Secrets(store.Namespace).Create(secret)
|
||||
k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &secret.ObjectMeta, c.ownerRefs)
|
||||
_, err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Create(secret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save rgw secrets. %+v", err)
|
||||
}
|
||||
@@ -243,17 +252,17 @@ func createKeyring(context *clusterd.Context, store cephv1beta1.ObjectStore, own
|
||||
return nil
|
||||
}
|
||||
|
||||
func instanceName(store cephv1beta1.ObjectStore) string {
|
||||
return fmt.Sprintf("%s-%s", appName, store.Name)
|
||||
func (c *config) instanceName() string {
|
||||
return fmt.Sprintf("%s-%s", appName, c.store.Name)
|
||||
}
|
||||
|
||||
// create a keyring for the rgw client with a limited set of privileges
|
||||
func createRGWKeyring(context *clusterd.Context, clusterName string) (string, error) {
|
||||
func (c *config) createRGWKeyring() (string, error) {
|
||||
username := "client.radosgw.gateway"
|
||||
access := []string{"osd", "allow rwx", "mon", "allow rw"}
|
||||
|
||||
// get-or-create-key for the user account
|
||||
key, err := client.AuthGetOrCreateKey(context, clusterName, username, access)
|
||||
key, err := client.AuthGetOrCreateKey(c.context, c.store.Namespace, username, access)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get or create auth key for %store. %+v", username, err)
|
||||
}
|
||||
|
||||
@@ -49,43 +49,44 @@ func TestStartRGW(t *testing.T) {
|
||||
version := "v1.1.0"
|
||||
|
||||
// start a basic cluster
|
||||
err := CreateStore(context, store, version, false, []metav1.OwnerReference{})
|
||||
c := &config{context, store, version, cephv1beta1.CephVersionSpec{}, false, []metav1.OwnerReference{}}
|
||||
err := c.createStore()
|
||||
assert.Nil(t, err)
|
||||
|
||||
validateStart(t, store, clientset, false)
|
||||
validateStart(t, c, clientset, false)
|
||||
|
||||
// starting again should update the pods with the new settings
|
||||
store.Spec.Gateway.AllNodes = true
|
||||
err = UpdateStore(context, store, version, false, []metav1.OwnerReference{})
|
||||
c.store.Spec.Gateway.AllNodes = true
|
||||
err = c.updateStore()
|
||||
assert.Nil(t, err)
|
||||
|
||||
validateStart(t, store, clientset, true)
|
||||
validateStart(t, c, clientset, true)
|
||||
}
|
||||
|
||||
func validateStart(t *testing.T, store cephv1beta1.ObjectStore, clientset *fake.Clientset, allNodes bool) {
|
||||
func validateStart(t *testing.T, c *config, clientset *fake.Clientset, allNodes bool) {
|
||||
if !allNodes {
|
||||
r, err := clientset.ExtensionsV1beta1().Deployments(store.Namespace).Get(instanceName(store), metav1.GetOptions{})
|
||||
r, err := clientset.ExtensionsV1beta1().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, instanceName(store), r.Name)
|
||||
assert.Equal(t, c.instanceName(), r.Name)
|
||||
|
||||
_, err = clientset.ExtensionsV1beta1().DaemonSets(store.Namespace).Get(instanceName(store), metav1.GetOptions{})
|
||||
_, err = clientset.ExtensionsV1beta1().DaemonSets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
|
||||
assert.True(t, errors.IsNotFound(err))
|
||||
} else {
|
||||
r, err := clientset.ExtensionsV1beta1().DaemonSets(store.Namespace).Get(instanceName(store), metav1.GetOptions{})
|
||||
r, err := clientset.ExtensionsV1beta1().DaemonSets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, instanceName(store), r.Name)
|
||||
assert.Equal(t, c.instanceName(), r.Name)
|
||||
|
||||
_, err = clientset.ExtensionsV1beta1().Deployments(store.Namespace).Get(instanceName(store), metav1.GetOptions{})
|
||||
_, err = clientset.ExtensionsV1beta1().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
|
||||
assert.True(t, errors.IsNotFound(err))
|
||||
}
|
||||
|
||||
s, err := clientset.CoreV1().Services(store.Namespace).Get(instanceName(store), metav1.GetOptions{})
|
||||
s, err := clientset.CoreV1().Services(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, instanceName(store), s.Name)
|
||||
assert.Equal(t, c.instanceName(), s.Name)
|
||||
|
||||
secret, err := clientset.CoreV1().Secrets(store.Namespace).Get(instanceName(store), metav1.GetOptions{})
|
||||
secret, err := clientset.CoreV1().Secrets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, instanceName(store), secret.Name)
|
||||
assert.Equal(t, c.instanceName(), secret.Name)
|
||||
assert.Equal(t, 1, len(secret.StringData))
|
||||
}
|
||||
|
||||
@@ -115,7 +116,8 @@ func TestCreateObjectStore(t *testing.T) {
|
||||
context := &clusterd.Context{Executor: executor, Clientset: clientset}
|
||||
|
||||
// create the pools
|
||||
err := CreateStore(context, store, "1.2.3.4", false, []metav1.OwnerReference{})
|
||||
c := &config{context, store, "1.2.3.4", cephv1beta1.CephVersionSpec{}, false, []metav1.OwnerReference{}}
|
||||
err := c.createStore()
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
|
||||
@@ -21,8 +21,6 @@ import (
|
||||
"fmt"
|
||||
"path"
|
||||
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
rgwdaemon "github.com/rook/rook/pkg/daemon/ceph/rgw"
|
||||
opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
opspec "github.com/rook/rook/pkg/operator/ceph/spec"
|
||||
@@ -34,103 +32,103 @@ import (
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
)
|
||||
|
||||
func startDeployment(context *clusterd.Context, store cephv1beta1.ObjectStore, version string, replicas int32, hostNetwork bool, ownerRefs []metav1.OwnerReference) error {
|
||||
func (c *config) startDeployment() error {
|
||||
|
||||
deployment := &extensions.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: instanceName(store),
|
||||
Namespace: store.Namespace,
|
||||
Name: c.instanceName(),
|
||||
Namespace: c.store.Namespace,
|
||||
},
|
||||
Spec: extensions.DeploymentSpec{Template: makeRGWPodSpec(store, version, hostNetwork), Replicas: &replicas},
|
||||
Spec: extensions.DeploymentSpec{Template: c.makeRGWPodSpec(), Replicas: &c.store.Spec.Gateway.Instances},
|
||||
}
|
||||
k8sutil.SetOwnerRefs(context.Clientset, store.Namespace, &deployment.ObjectMeta, ownerRefs)
|
||||
_, err := context.Clientset.ExtensionsV1beta1().Deployments(store.Namespace).Create(deployment)
|
||||
k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &deployment.ObjectMeta, c.ownerRefs)
|
||||
_, err := c.context.Clientset.ExtensionsV1beta1().Deployments(c.store.Namespace).Create(deployment)
|
||||
return err
|
||||
}
|
||||
|
||||
func startDaemonset(context *clusterd.Context, store cephv1beta1.ObjectStore, version string, hostNetwork bool, ownerRefs []metav1.OwnerReference) error {
|
||||
func (c *config) startDaemonset() error {
|
||||
|
||||
daemonset := &extensions.DaemonSet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: instanceName(store),
|
||||
Namespace: store.Namespace,
|
||||
Name: c.instanceName(),
|
||||
Namespace: c.store.Namespace,
|
||||
},
|
||||
Spec: extensions.DaemonSetSpec{
|
||||
UpdateStrategy: extensions.DaemonSetUpdateStrategy{
|
||||
Type: extensions.RollingUpdateDaemonSetStrategyType,
|
||||
},
|
||||
Template: makeRGWPodSpec(store, version, hostNetwork),
|
||||
Template: c.makeRGWPodSpec(),
|
||||
},
|
||||
}
|
||||
k8sutil.SetOwnerRefs(context.Clientset, store.Namespace, &daemonset.ObjectMeta, ownerRefs)
|
||||
k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &daemonset.ObjectMeta, c.ownerRefs)
|
||||
|
||||
_, err := context.Clientset.ExtensionsV1beta1().DaemonSets(store.Namespace).Create(daemonset)
|
||||
_, err := c.context.Clientset.ExtensionsV1beta1().DaemonSets(c.store.Namespace).Create(daemonset)
|
||||
return err
|
||||
}
|
||||
|
||||
func makeRGWPodSpec(store cephv1beta1.ObjectStore, version string, hostNetwork bool) v1.PodTemplateSpec {
|
||||
func (c *config) makeRGWPodSpec() v1.PodTemplateSpec {
|
||||
podSpec := v1.PodSpec{
|
||||
InitContainers: []v1.Container{
|
||||
makeConfigInitContainer(store, version),
|
||||
c.makeConfigInitContainer(),
|
||||
},
|
||||
Containers: []v1.Container{
|
||||
makeDaemonContainer(store, version),
|
||||
c.makeDaemonContainer(),
|
||||
},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: opspec.PodVolumes(""),
|
||||
HostNetwork: hostNetwork,
|
||||
HostNetwork: c.hostNetwork,
|
||||
}
|
||||
if hostNetwork {
|
||||
if c.hostNetwork {
|
||||
podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
}
|
||||
|
||||
// Set the ssl cert if specified
|
||||
if store.Spec.Gateway.SSLCertificateRef != "" {
|
||||
if c.store.Spec.Gateway.SSLCertificateRef != "" {
|
||||
certVol := v1.Volume{Name: certVolumeName, VolumeSource: v1.VolumeSource{Secret: &v1.SecretVolumeSource{
|
||||
SecretName: store.Spec.Gateway.SSLCertificateRef,
|
||||
SecretName: c.store.Spec.Gateway.SSLCertificateRef,
|
||||
Items: []v1.KeyToPath{{Key: certKeyName, Path: certFilename}},
|
||||
}}}
|
||||
podSpec.Volumes = append(podSpec.Volumes, certVol)
|
||||
}
|
||||
|
||||
store.Spec.Gateway.Placement.ApplyToPodSpec(&podSpec)
|
||||
c.store.Spec.Gateway.Placement.ApplyToPodSpec(&podSpec)
|
||||
|
||||
return v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: instanceName(store),
|
||||
Labels: getLabels(store),
|
||||
Name: c.instanceName(),
|
||||
Labels: c.getLabels(),
|
||||
Annotations: map[string]string{},
|
||||
},
|
||||
Spec: podSpec,
|
||||
}
|
||||
}
|
||||
|
||||
func makeConfigInitContainer(store cephv1beta1.ObjectStore, version string) v1.Container {
|
||||
func (c *config) makeConfigInitContainer() v1.Container {
|
||||
container := v1.Container{
|
||||
Name: opspec.ConfigInitContainerName,
|
||||
Image: k8sutil.MakeRookImage(version),
|
||||
Image: k8sutil.MakeRookImage(c.rookVersion),
|
||||
Args: []string{
|
||||
"ceph",
|
||||
"rgw",
|
||||
fmt.Sprintf("--config-dir=%s", k8sutil.DataDir),
|
||||
fmt.Sprintf("--rgw-name=%s", store.Name),
|
||||
fmt.Sprintf("--rgw-port=%d", store.Spec.Gateway.Port),
|
||||
fmt.Sprintf("--rgw-secure-port=%d", store.Spec.Gateway.SecurePort),
|
||||
fmt.Sprintf("--rgw-name=%s", c.store.Name),
|
||||
fmt.Sprintf("--rgw-port=%d", c.store.Spec.Gateway.Port),
|
||||
fmt.Sprintf("--rgw-secure-port=%d", c.store.Spec.Gateway.SecurePort),
|
||||
},
|
||||
VolumeMounts: opspec.RookVolumeMounts(),
|
||||
Env: []v1.EnvVar{
|
||||
{Name: "ROOK_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: instanceName(store)}, Key: keyringName}}},
|
||||
{Name: "ROOK_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: c.instanceName()}, Key: keyringName}}},
|
||||
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
|
||||
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
|
||||
opmon.ClusterNameEnvVar(store.Namespace),
|
||||
opmon.ClusterNameEnvVar(c.store.Namespace),
|
||||
opmon.EndpointEnvVar(),
|
||||
opmon.SecretEnvVar(),
|
||||
k8sutil.ConfigOverrideEnvVar(),
|
||||
},
|
||||
Resources: store.Spec.Gateway.Resources,
|
||||
Resources: c.store.Spec.Gateway.Resources,
|
||||
}
|
||||
|
||||
if store.Spec.Gateway.SSLCertificateRef != "" {
|
||||
if c.store.Spec.Gateway.SSLCertificateRef != "" {
|
||||
// Add a volume mount for the ssl certificate
|
||||
mount := v1.VolumeMount{Name: certVolumeName, MountPath: certMountPath, ReadOnly: true}
|
||||
container.VolumeMounts = append(container.VolumeMounts, mount)
|
||||
@@ -143,12 +141,12 @@ func makeConfigInitContainer(store cephv1beta1.ObjectStore, version string) v1.C
|
||||
return container
|
||||
}
|
||||
|
||||
func makeDaemonContainer(store cephv1beta1.ObjectStore, version string) v1.Container {
|
||||
func (c *config) makeDaemonContainer() v1.Container {
|
||||
|
||||
// start the rgw daemon in the foreground
|
||||
container := v1.Container{
|
||||
Name: "rgw",
|
||||
Image: k8sutil.MakeRookImage(version),
|
||||
Image: c.cephVersion.Image,
|
||||
Command: []string{
|
||||
"radosgw",
|
||||
},
|
||||
@@ -159,45 +157,45 @@ func makeDaemonContainer(store cephv1beta1.ObjectStore, version string) v1.Conta
|
||||
},
|
||||
VolumeMounts: opspec.CephVolumeMounts(),
|
||||
Env: k8sutil.ClusterDaemonEnvVars(),
|
||||
Resources: store.Spec.Gateway.Resources,
|
||||
Resources: c.store.Spec.Gateway.Resources,
|
||||
}
|
||||
|
||||
return container
|
||||
}
|
||||
|
||||
func startService(context *clusterd.Context, store cephv1beta1.ObjectStore, hostNetwork bool, ownerRefs []metav1.OwnerReference) (string, error) {
|
||||
labels := getLabels(store)
|
||||
func (c *config) startService() (string, error) {
|
||||
labels := c.getLabels()
|
||||
svc := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: instanceName(store),
|
||||
Namespace: store.Namespace,
|
||||
Name: c.instanceName(),
|
||||
Namespace: c.store.Namespace,
|
||||
Labels: labels,
|
||||
},
|
||||
Spec: v1.ServiceSpec{
|
||||
Selector: labels,
|
||||
},
|
||||
}
|
||||
k8sutil.SetOwnerRefs(context.Clientset, store.Namespace, &svc.ObjectMeta, ownerRefs)
|
||||
if hostNetwork {
|
||||
k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &svc.ObjectMeta, c.ownerRefs)
|
||||
if c.hostNetwork {
|
||||
svc.Spec.ClusterIP = v1.ClusterIPNone
|
||||
}
|
||||
|
||||
addPort(svc, "http", store.Spec.Gateway.Port)
|
||||
addPort(svc, "https", store.Spec.Gateway.SecurePort)
|
||||
addPort(svc, "http", c.store.Spec.Gateway.Port)
|
||||
addPort(svc, "https", c.store.Spec.Gateway.SecurePort)
|
||||
|
||||
svc, err := context.Clientset.CoreV1().Services(store.Namespace).Create(svc)
|
||||
svc, err := c.context.Clientset.CoreV1().Services(c.store.Namespace).Create(svc)
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return "", fmt.Errorf("failed to create rgw service. %+v", err)
|
||||
}
|
||||
svc, err = context.Clientset.CoreV1().Services(store.Namespace).Get(instanceName(store), metav1.GetOptions{})
|
||||
svc, err = c.context.Clientset.CoreV1().Services(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get existing service IP. %+v", err)
|
||||
}
|
||||
return svc.Spec.ClusterIP, nil
|
||||
}
|
||||
|
||||
logger.Infof("Gateway service running at %s:%d", svc.Spec.ClusterIP, store.Spec.Gateway.Port)
|
||||
logger.Infof("Gateway service running at %s:%d", svc.Spec.ClusterIP, c.store.Spec.Gateway.Port)
|
||||
return svc.Spec.ClusterIP, nil
|
||||
}
|
||||
|
||||
@@ -213,10 +211,10 @@ func addPort(service *v1.Service, name string, port int32) {
|
||||
})
|
||||
}
|
||||
|
||||
func getLabels(store cephv1beta1.ObjectStore) map[string]string {
|
||||
func (c *config) getLabels() map[string]string {
|
||||
return map[string]string{
|
||||
k8sutil.AppAttr: appName,
|
||||
k8sutil.ClusterAttr: store.Namespace,
|
||||
"rook_object_store": store.Name,
|
||||
k8sutil.ClusterAttr: c.store.Namespace,
|
||||
"rook_object_store": c.store.Name,
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
@@ -40,7 +41,8 @@ func TestPodSpecs(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
s := makeRGWPodSpec(store, "rook/rook:myversion", true)
|
||||
c := &config{store: store, rookVersion: "rook/rook:myversion", cephVersion: cephv1beta1.CephVersionSpec{Image: "ceph/ceph:v13.2.1"}, hostNetwork: true}
|
||||
s := c.makeRGWPodSpec()
|
||||
assert.NotNil(t, s)
|
||||
//assert.Equal(t, instanceName(store), s.Name)
|
||||
assert.Equal(t, v1.RestartPolicyAlways, s.Spec.RestartPolicy)
|
||||
@@ -48,7 +50,7 @@ func TestPodSpecs(t *testing.T) {
|
||||
assert.Equal(t, "rook-data", s.Spec.Volumes[0].Name)
|
||||
assert.Equal(t, cephconfig.DefaultConfigMountName, s.Spec.Volumes[1].Name)
|
||||
|
||||
assert.Equal(t, instanceName(store), s.ObjectMeta.Name)
|
||||
assert.Equal(t, c.instanceName(), s.ObjectMeta.Name)
|
||||
assert.Equal(t, appName, s.ObjectMeta.Labels["app"])
|
||||
assert.Equal(t, store.Namespace, s.ObjectMeta.Labels["rook_cluster"])
|
||||
assert.Equal(t, store.Name, s.ObjectMeta.Labels["rook_object_store"])
|
||||
@@ -68,7 +70,7 @@ func TestPodSpecs(t *testing.T) {
|
||||
assert.Equal(t, "--rgw-secure-port=0", cont.Args[5])
|
||||
|
||||
cont = s.Spec.Containers[0]
|
||||
assert.Equal(t, "rook/rook:myversion", cont.Image)
|
||||
assert.Equal(t, "ceph/ceph:v13.2.1", cont.Image)
|
||||
assert.Equal(t, 2, len(cont.VolumeMounts))
|
||||
|
||||
assert.Equal(t, 1, len(cont.Command))
|
||||
@@ -89,9 +91,10 @@ func TestSSLPodSpec(t *testing.T) {
|
||||
store.Spec.Gateway.SSLCertificateRef = "mycert"
|
||||
store.Spec.Gateway.SecurePort = 443
|
||||
|
||||
s := makeRGWPodSpec(store, "v1.0", true)
|
||||
c := &config{store: store, rookVersion: "v1.0", hostNetwork: true}
|
||||
s := c.makeRGWPodSpec()
|
||||
assert.NotNil(t, s)
|
||||
assert.Equal(t, instanceName(store), s.Name)
|
||||
assert.Equal(t, c.instanceName(), s.Name)
|
||||
assert.Equal(t, 4, len(s.Spec.Volumes))
|
||||
assert.Equal(t, certVolumeName, s.Spec.Volumes[3].Name)
|
||||
assert.True(t, s.Spec.HostNetwork)
|
||||
|
||||
@@ -17,7 +17,6 @@ package k8sutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
@@ -26,30 +25,25 @@ import (
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
// GetDeploymentVersion returns the version of the image running in the pod spec for the desired container
|
||||
func GetDeploymentVersion(clientset kubernetes.Interface, namespace, name, container string) (string, error) {
|
||||
// GetDeploymentImage returns the version of the image running in the pod spec for the desired container
|
||||
func GetDeploymentImage(clientset kubernetes.Interface, namespace, name, container string) (string, error) {
|
||||
d, err := clientset.Extensions().Deployments(namespace).Get(name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to find deployment %s. %v", name, err)
|
||||
}
|
||||
return GetDeploymentSpecVersion(clientset, *d, container)
|
||||
return GetDeploymentSpecImage(clientset, *d, container)
|
||||
}
|
||||
|
||||
func GetDeploymentSpecVersion(clientset kubernetes.Interface, d extensions.Deployment, container string) (string, error) {
|
||||
func GetDeploymentSpecImage(clientset kubernetes.Interface, d extensions.Deployment, container string) (string, error) {
|
||||
image, err := GetSpecContainerImage(d.Spec.Template.Spec, container)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
parts := strings.Split(image, ":")
|
||||
if len(parts) != 2 {
|
||||
return "", fmt.Errorf("unexpected version in image: %s", image)
|
||||
}
|
||||
|
||||
return parts[1], nil
|
||||
return image, nil
|
||||
}
|
||||
|
||||
func WaitForDeploymentVersion(clientset kubernetes.Interface, namespace, label, container, desiredVersion string) error {
|
||||
func WaitForDeploymentImage(clientset kubernetes.Interface, namespace, label, container, desiredImage string) error {
|
||||
|
||||
sleepTime := 3
|
||||
attempts := 30
|
||||
@@ -61,25 +55,25 @@ func WaitForDeploymentVersion(clientset kubernetes.Interface, namespace, label,
|
||||
|
||||
matches := 0
|
||||
for _, d := range deployments.Items {
|
||||
version, err := GetDeploymentSpecVersion(clientset, d, container)
|
||||
image, err := GetDeploymentSpecImage(clientset, d, container)
|
||||
if err != nil {
|
||||
logger.Infof("failed to get version for deployment %s. %+v", d.Name, err)
|
||||
logger.Infof("failed to get image for deployment %s. %+v", d.Name, err)
|
||||
continue
|
||||
}
|
||||
if version == desiredVersion {
|
||||
if image == desiredImage {
|
||||
matches++
|
||||
}
|
||||
}
|
||||
|
||||
if matches == len(deployments.Items) && matches > 0 {
|
||||
logger.Infof("all %d %s deployments are on version %s", matches, container, desiredVersion)
|
||||
logger.Infof("all %d %s deployments are on image %s", matches, container, desiredImage)
|
||||
break
|
||||
}
|
||||
|
||||
if len(deployments.Items) == 0 {
|
||||
logger.Infof("waiting for at least one deployment to start to see the version")
|
||||
} else {
|
||||
logger.Infof("%d/%d %s deployments match version %s", matches, len(deployments.Items), container, desiredVersion)
|
||||
logger.Infof("%d/%d %s deployments match image %s", matches, len(deployments.Items), container, desiredImage)
|
||||
}
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package k8sutil for Kubernetes helpers.
|
||||
package k8sutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
batch "k8s.io/api/batch/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
func RunReplaceableJob(clientset kubernetes.Interface, job *batch.Job) error {
|
||||
// check if the job was already created and what its status is
|
||||
existingJob, err := clientset.Batch().Jobs(job.Namespace).Get(job.Name, metav1.GetOptions{})
|
||||
if err != nil && !errors.IsNotFound(err) {
|
||||
logger.Warningf("failed to detect job %s. %+v", job.Name, err)
|
||||
} else if err == nil {
|
||||
// if the job is still running, allow it to continue to completion
|
||||
if existingJob.Status.Active > 0 {
|
||||
logger.Infof("Found previous job %s. Status=%+v", job.Name, existingJob.Status)
|
||||
return nil
|
||||
}
|
||||
|
||||
// delete the job that already exists from a previous run
|
||||
logger.Infof("Removing previous job %s to start a new one", job.Name)
|
||||
err := DeleteBatchJob(clientset, job.Namespace, existingJob.Name, true)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to remove job %s. %+v", job.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
_, err = clientset.Batch().Jobs(job.Namespace).Create(job)
|
||||
return err
|
||||
}
|
||||
|
||||
// WaitForJobCompletion waits for a job to reach the completed state.
|
||||
// Assumes that only one pod needs to complete.
|
||||
func WaitForJobCompletion(clientset kubernetes.Interface, job *batch.Job, timeout time.Duration) error {
|
||||
logger.Infof("waiting for job %s to complete...", job.Name)
|
||||
return wait.Poll(5*time.Second, timeout, func() (bool, error) {
|
||||
job, err := clientset.Batch().Jobs(job.Namespace).Get(job.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to detect job %s. %+v", job.Name, err)
|
||||
}
|
||||
|
||||
// if the job is still running, allow it to continue to completion
|
||||
if job.Status.Active > 0 {
|
||||
logger.Debugf("job is still running. Status=%+v", job.Status)
|
||||
return false, nil
|
||||
}
|
||||
if job.Status.Failed > 0 {
|
||||
return false, fmt.Errorf("job %s failed.", job.Name)
|
||||
}
|
||||
if job.Status.Succeeded > 0 {
|
||||
return true, nil
|
||||
}
|
||||
logger.Debugf("job is still initializing")
|
||||
return false, nil
|
||||
})
|
||||
}
|
||||
|
||||
func DeleteBatchJob(clientset kubernetes.Interface, namespace, name string, wait bool) error {
|
||||
propagation := metav1.DeletePropagationForeground
|
||||
gracePeriod := int64(0)
|
||||
options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
|
||||
|
||||
if err := clientset.Batch().Jobs(namespace).Delete(name, options); err != nil {
|
||||
return fmt.Errorf("failed to remove previous provisioning job for node %s. %+v", name, err)
|
||||
}
|
||||
|
||||
if !wait {
|
||||
return nil
|
||||
}
|
||||
|
||||
retries := 20
|
||||
sleepInterval := 2 * time.Second
|
||||
for i := 0; i < retries; i++ {
|
||||
_, err := clientset.Batch().Jobs(namespace).Get(name, metav1.GetOptions{})
|
||||
if err != nil && errors.IsNotFound(err) {
|
||||
logger.Infof("batch job %s deleted", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Infof("batch job %s still exists", name)
|
||||
time.Sleep(sleepInterval)
|
||||
}
|
||||
|
||||
logger.Warningf("gave up waiting for batch job %s to be deleted", name)
|
||||
return nil
|
||||
}
|
||||
@@ -19,8 +19,10 @@ package k8sutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -186,6 +188,32 @@ func GetPodPhaseMap(pods *v1.PodList) map[v1.PodPhase][]string {
|
||||
return podPhaseMap
|
||||
}
|
||||
|
||||
// GetJobLog gets the logs for the pod. If there is more than one pod with the label selector, the logs from
|
||||
// the first pod will be returned.
|
||||
func GetPodLog(clientset kubernetes.Interface, namespace string, labelSelector string) (string, error) {
|
||||
opts := metav1.ListOptions{
|
||||
LabelSelector: labelSelector,
|
||||
}
|
||||
pods, err := clientset.CoreV1().Pods(namespace).List(opts)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get version pod. %+v", err)
|
||||
}
|
||||
for _, pod := range pods.Items {
|
||||
req := clientset.CoreV1().Pods(namespace).GetLogs(pod.Name, &v1.PodLogOptions{})
|
||||
readCloser, err := req.Stream()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read from stream. %+v", err)
|
||||
}
|
||||
|
||||
builder := &strings.Builder{}
|
||||
defer readCloser.Close()
|
||||
_, err = io.Copy(builder, readCloser)
|
||||
return builder.String(), err
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("did not find any pods with label %s", labelSelector)
|
||||
}
|
||||
|
||||
// DeleteDeployment makes a best effort at deleting a deployment and its pods, then waits for them to be deleted
|
||||
func DeleteDeployment(clientset kubernetes.Interface, namespace, name string) error {
|
||||
logger.Debugf("removing %s deployment if it exists", name)
|
||||
|
||||
@@ -47,7 +47,8 @@ func (o *ObjectOperation) Create(namespace, storeName string, replicaCount int32
|
||||
return err
|
||||
}
|
||||
|
||||
err := o.k8sh.WaitForLabeledPodsToRun(fmt.Sprintf("rook_object_store=%s", storeName), namespace)
|
||||
// Starting an object store takes longer than the average operation, so add more retries
|
||||
err := o.k8sh.WaitForLabeledPodsToRunWithRetries(fmt.Sprintf("rook_object_store=%s", storeName), namespace, 40)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rgw did not start via crd. %+v", err)
|
||||
}
|
||||
|
||||
@@ -32,9 +32,7 @@ import (
|
||||
"flag"
|
||||
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
@@ -43,11 +41,20 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
// test with the latest luminous build
|
||||
luminousTestImage = "ceph/ceph:v12"
|
||||
// test with the latest mimic build
|
||||
mimicTestImage = "ceph/ceph:v13"
|
||||
rookOperatorCreatedCrd = "clusters.ceph.rook.io"
|
||||
helmChartName = "local/rook-ceph"
|
||||
helmDeployName = "rook-ceph"
|
||||
)
|
||||
|
||||
var (
|
||||
LuminousVersion = cephv1beta1.CephVersionSpec{Image: luminousTestImage, Name: cephv1beta1.Luminous}
|
||||
MimicVersion = cephv1beta1.CephVersionSpec{Image: mimicTestImage, Name: cephv1beta1.Mimic}
|
||||
)
|
||||
|
||||
//CephInstaller wraps installing and uninstalling rook on a platform
|
||||
type CephInstaller struct {
|
||||
Manifests CephManifests
|
||||
@@ -56,6 +63,7 @@ type CephInstaller struct {
|
||||
helmHelper *utils.HelmHelper
|
||||
k8sVersion string
|
||||
changeHostnames bool
|
||||
cephVersion cephv1beta1.CephVersionSpec
|
||||
T func() *testing.T
|
||||
}
|
||||
|
||||
@@ -180,12 +188,13 @@ func (h *CephInstaller) CreateK8sRookToolbox(namespace string) (err error) {
|
||||
|
||||
func (h *CephInstaller) CreateK8sRookCluster(namespace, systemNamespace string, storeType string) (err error) {
|
||||
return h.CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType, false,
|
||||
cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, true /* startWithAllNodes */)
|
||||
cephv1beta1.MonSpec{Count: 3, AllowMultiplePerNode: true}, true, /* startWithAllNodes */
|
||||
LuminousVersion)
|
||||
}
|
||||
|
||||
//CreateK8sRookCluster creates rook cluster via kubectl
|
||||
func (h *CephInstaller) CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType string,
|
||||
useAllDevices bool, mon cephv1beta1.MonSpec, startWithAllNodes bool) error {
|
||||
useAllDevices bool, mon cephv1beta1.MonSpec, startWithAllNodes bool, cephVersion cephv1beta1.CephVersionSpec) error {
|
||||
|
||||
dataDirHostPath, err := h.initTestDir(namespace)
|
||||
if err != nil {
|
||||
@@ -207,71 +216,11 @@ func (h *CephInstaller) CreateK8sRookClusterWithHostPathAndDevices(namespace, sy
|
||||
return fmt.Errorf("Failed to create cluster roles. %+v", err)
|
||||
}
|
||||
|
||||
// for increasing the test matrix, sometimes use the strongly typed client instead of the yaml
|
||||
if h.k8shelper.VersionAtLeast("v1.10.0") {
|
||||
logger.Infof("Starting Rook cluster with strongly typed clientset")
|
||||
|
||||
clust := &cephv1beta1.Cluster{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: namespace,
|
||||
Namespace: namespace,
|
||||
},
|
||||
Spec: cephv1beta1.ClusterSpec{
|
||||
ServiceAccount: "rook-ceph-cluster",
|
||||
DataDirHostPath: dataDirHostPath,
|
||||
Mon: cephv1beta1.MonSpec{
|
||||
Count: mon.Count,
|
||||
AllowMultiplePerNode: mon.AllowMultiplePerNode,
|
||||
},
|
||||
Storage: rookalpha.StorageScopeSpec{
|
||||
UseAllNodes: startWithAllNodes,
|
||||
Selection: rookalpha.Selection{
|
||||
UseAllDevices: &useAllDevices,
|
||||
},
|
||||
Config: map[string]string{
|
||||
config.StoreTypeKey: storeType,
|
||||
config.DatabaseSizeMBKey: "1024",
|
||||
config.JournalSizeMBKey: "1024",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err := h.k8shelper.RookClientset.CephV1beta1().Clusters(namespace).Create(clust)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create cluster %s. %+v", clust.Name, err)
|
||||
}
|
||||
|
||||
if !startWithAllNodes {
|
||||
// now that the cluster is created, let's get all the k8s nodes so we can update the cluster CRD with them
|
||||
logger.Info("cluster was started without all nodes, will update cluster to add nodes now.")
|
||||
nodeNames, err := h.GetNodeHostnames()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get k8s nodes to add to cluster CRD: %+v", err)
|
||||
}
|
||||
|
||||
// add all discovered k8s nodes to the cluster CRD
|
||||
rookNodes := make([]rookalpha.Node, len(nodeNames))
|
||||
for i, hostname := range nodeNames {
|
||||
rookNodes[i] = rookalpha.Node{Name: hostname}
|
||||
}
|
||||
clust, err = h.k8shelper.RookClientset.CephV1beta1().Clusters(namespace).Get(namespace, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get rook cluster to add nodes to it: %+v", err)
|
||||
}
|
||||
clust.Spec.Storage.Nodes = rookNodes
|
||||
|
||||
// update the cluster CRD now
|
||||
_, err = h.k8shelper.RookClientset.CephV1beta1().Clusters(namespace).Update(clust)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update cluster %s with nodes. %+v", clust.Name, err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logger.Infof("Starting Rook Cluster with yaml")
|
||||
rookCluster := h.Manifests.GetRookCluster(&ClusterSettings{namespace, storeType, dataDirHostPath, useAllDevices, mon.Count})
|
||||
if _, err := h.k8shelper.KubectlWithStdin(rookCluster, createFromStdinArgs...); err != nil {
|
||||
return fmt.Errorf("Failed to create rook cluster : %v ", err)
|
||||
}
|
||||
logger.Infof("Starting Rook Cluster with yaml")
|
||||
settings := &ClusterSettings{namespace, storeType, dataDirHostPath, useAllDevices, mon.Count, cephVersion}
|
||||
rookCluster := h.Manifests.GetRookCluster(settings)
|
||||
if _, err := h.k8shelper.KubectlWithStdin(rookCluster, createFromStdinArgs...); err != nil {
|
||||
return fmt.Errorf("Failed to create rook cluster : %v ", err)
|
||||
}
|
||||
|
||||
if err := h.k8shelper.WaitForPodCount("app=rook-ceph-mon", namespace, mon.Count); err != nil {
|
||||
@@ -372,7 +321,8 @@ func (h *CephInstaller) InstallRookOnK8sWithHostPathAndDevices(namespace, storeT
|
||||
|
||||
//Create rook cluster
|
||||
err = h.CreateK8sRookClusterWithHostPathAndDevices(namespace, onamespace, storeType,
|
||||
useDevices, cephv1beta1.MonSpec{Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode}, startWithAllNodes)
|
||||
useDevices, cephv1beta1.MonSpec{Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode}, startWithAllNodes,
|
||||
h.cephVersion)
|
||||
if err != nil {
|
||||
logger.Errorf("Rook cluster %s not installed, error -> %v", namespace, err)
|
||||
return false, err
|
||||
@@ -488,7 +438,7 @@ func (h *CephInstaller) GatherAllRookLogs(namespace, systemNamespace string, tes
|
||||
}
|
||||
|
||||
// NewCephInstaller creates new instance of CephInstaller
|
||||
func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, rookVersion string) *CephInstaller {
|
||||
func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, rookVersion string, cephVersion cephv1beta1.CephVersionSpec) *CephInstaller {
|
||||
|
||||
// All e2e tests should run ceph commands in the toolbox since we are not inside a container
|
||||
client.RunAllCephCommandsInToolbox = true
|
||||
@@ -502,11 +452,14 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, rook
|
||||
if err != nil {
|
||||
panic("failed to get kubectl client :" + err.Error())
|
||||
}
|
||||
logger.Infof("Rook Version: %s", rookVersion)
|
||||
logger.Infof("Ceph Version: %s (%s)", cephVersion.Image, cephVersion.Name)
|
||||
h := &CephInstaller{
|
||||
Manifests: NewCephManifests(rookVersion),
|
||||
k8shelper: k8shelp,
|
||||
helmHelper: utils.NewHelmHelper(Env.Helm),
|
||||
k8sVersion: version.String(),
|
||||
cephVersion: cephVersion,
|
||||
changeHostnames: rookVersion != Version0_8 && k8shelp.VersionAtLeast("v1.11.0"),
|
||||
T: t,
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"strconv"
|
||||
|
||||
"github.com/google/uuid"
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
)
|
||||
|
||||
type CephManifests interface {
|
||||
@@ -45,6 +46,7 @@ type ClusterSettings struct {
|
||||
DataDirHostPath string
|
||||
UseAllDevices bool
|
||||
Mons int
|
||||
CephVersion cephv1beta1.CephVersionSpec
|
||||
}
|
||||
|
||||
//CephManifestsMaster wraps rook yaml definitions
|
||||
@@ -190,6 +192,7 @@ rules:
|
||||
resources:
|
||||
- secrets
|
||||
- pods
|
||||
- pods/log
|
||||
- services
|
||||
- configmaps
|
||||
verbs:
|
||||
@@ -419,6 +422,9 @@ metadata:
|
||||
name: ` + settings.Namespace + `
|
||||
namespace: ` + settings.Namespace + `
|
||||
spec:
|
||||
cephVersion:
|
||||
image: ` + settings.CephVersion.Image + `
|
||||
allowUnsupported: ` + strconv.FormatBool(settings.CephVersion.AllowUnsupported) + `
|
||||
serviceAccount: rook-ceph-cluster
|
||||
dataDirHostPath: ` + settings.DataDirHostPath + `
|
||||
network:
|
||||
|
||||
@@ -32,7 +32,7 @@ const (
|
||||
// Version tag for the latest manifests
|
||||
VersionMaster = "master"
|
||||
// Version tag for Rook v0.8
|
||||
Version0_8 = "v0.8.1"
|
||||
Version0_8 = "v0.8.3"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@@ -457,11 +457,16 @@ func (k8sh *K8sHelper) IsPodWithLabelPresent(label string, namespace string) boo
|
||||
return count > 0
|
||||
}
|
||||
|
||||
//WaitForLabelWaitForLabeledPodsToRun calls WaitForLabeledPodsToRunWithRetries with the default number of retries
|
||||
func (k8sh *K8sHelper) WaitForLabeledPodsToRun(label, namespace string) error {
|
||||
return k8sh.WaitForLabeledPodsToRunWithRetries(label, namespace, RetryLoop)
|
||||
}
|
||||
|
||||
//WaitForLabeledPodsToRun returns true if a Pod is running status or goes to Running status within 90s else returns false
|
||||
func (k8sh *K8sHelper) WaitForLabeledPodsToRun(label string, namespace string) error {
|
||||
func (k8sh *K8sHelper) WaitForLabeledPodsToRunWithRetries(label string, namespace string, retries int) error {
|
||||
options := metav1.ListOptions{LabelSelector: label}
|
||||
var lastPod v1.Pod
|
||||
for i := 0; i < RetryLoop; i++ {
|
||||
for i := 0; i < retries; i++ {
|
||||
pods, err := k8sh.Clientset.CoreV1().Pods(namespace).List(options)
|
||||
lastStatus := ""
|
||||
running := 0
|
||||
|
||||
@@ -100,11 +100,11 @@ type TestCluster struct {
|
||||
}
|
||||
|
||||
// StartTestCluster creates new instance of TestCluster struct
|
||||
func StartTestCluster(t func() *testing.T, namespace, storeType string, helmInstalled, useDevices bool, mons int, rookVersion string) (*TestCluster, *utils.K8sHelper) {
|
||||
func StartTestCluster(t func() *testing.T, namespace, storeType string, helmInstalled, useDevices bool, mons int, rookVersion string, cephVersion cephv1beta1.CephVersionSpec) (*TestCluster, *utils.K8sHelper) {
|
||||
kh, err := utils.CreateK8sHelper(t)
|
||||
require.NoError(t(), err)
|
||||
|
||||
i := installer.NewCephInstaller(t, kh.Clientset, rookVersion)
|
||||
i := installer.NewCephInstaller(t, kh.Clientset, rookVersion, cephVersion)
|
||||
|
||||
op := &TestCluster{i, kh, nil, t, namespace, storeType, helmInstalled, useDevices, mons}
|
||||
|
||||
@@ -113,6 +113,7 @@ func StartTestCluster(t func() *testing.T, namespace, storeType string, helmInst
|
||||
assert.NoError(t(), kh.GetDockerImage("rook/ceph:"+rookVersion))
|
||||
assert.NoError(t(), kh.GetDockerImage("rook/ceph-toolbox:"+rookVersion))
|
||||
}
|
||||
assert.NoError(t(), kh.GetDockerImage(cephVersion.Image))
|
||||
|
||||
op.Setup()
|
||||
return op, kh
|
||||
|
||||
@@ -54,7 +54,7 @@ func (s *BlockCreateSuite) SetupSuite() {
|
||||
|
||||
var err error
|
||||
s.namespace = "block-k8s-ns"
|
||||
s.op, s.kh = StartTestCluster(s.T, s.namespace, "bluestore", false, false, 1, installer.VersionMaster)
|
||||
s.op, s.kh = StartTestCluster(s.T, s.namespace, "bluestore", false, false, 1, installer.VersionMaster, installer.MimicVersion)
|
||||
s.testClient = clients.CreateTestClient(s.kh, s.op.installer.Manifests)
|
||||
initialBlocks, err := s.testClient.BlockClient.List(s.namespace)
|
||||
assert.Nil(s.T(), err)
|
||||
|
||||
@@ -63,7 +63,7 @@ type HelmSuite struct {
|
||||
|
||||
func (hs *HelmSuite) SetupSuite() {
|
||||
hs.namespace = "helm-ns"
|
||||
hs.op, hs.kh = StartTestCluster(hs.T, hs.namespace, "bluestore", true, false, 1, installer.VersionMaster)
|
||||
hs.op, hs.kh = StartTestCluster(hs.T, hs.namespace, "bluestore", true, false, 1, installer.VersionMaster, installer.MimicVersion)
|
||||
hs.helper = clients.CreateTestClient(hs.kh, hs.op.installer.Manifests)
|
||||
}
|
||||
|
||||
|
||||
@@ -135,7 +135,7 @@ func NewMCTestOperations(t func() *testing.T, namespace1 string, namespace2 stri
|
||||
|
||||
kh, err := utils.CreateK8sHelper(t)
|
||||
require.NoError(t(), err)
|
||||
i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster)
|
||||
i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster, installer.LuminousVersion)
|
||||
|
||||
op := &MCTestOperations{i, kh, t, namespace1, namespace2, installer.SystemNamespace(namespace1)}
|
||||
op.Setup()
|
||||
|
||||
@@ -76,7 +76,7 @@ type SmokeSuite struct {
|
||||
func (suite *SmokeSuite) SetupSuite() {
|
||||
suite.namespace = "smoke-ns"
|
||||
useDevices := true
|
||||
suite.op, suite.k8sh = StartTestCluster(suite.T, suite.namespace, "bluestore", false, useDevices, 3, installer.VersionMaster)
|
||||
suite.op, suite.k8sh = StartTestCluster(suite.T, suite.namespace, "bluestore", false, useDevices, 3, installer.VersionMaster, installer.MimicVersion)
|
||||
suite.helper = clients.CreateTestClient(suite.k8sh, suite.op.installer.Manifests)
|
||||
}
|
||||
|
||||
@@ -135,7 +135,10 @@ func checkOrderedSubstrings(t *testing.T, input string, substrings ...string) {
|
||||
}
|
||||
original := input
|
||||
for i, substring := range substrings {
|
||||
assert.Contains(t, input, substring, fmt.Sprintf("missing substring %d. original=%s", i, original))
|
||||
if !strings.Contains(input, substring) {
|
||||
assert.Fail(t, fmt.Sprintf("missing substring %d. original=%s", i, original))
|
||||
return
|
||||
}
|
||||
index := strings.Index(input, substring)
|
||||
input = input[index+len(substring):]
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
@@ -61,7 +62,7 @@ func (s *UpgradeSuite) SetupSuite() {
|
||||
s.namespace = "upgrade-ns"
|
||||
useDevices := true
|
||||
|
||||
s.op, s.k8sh = StartTestCluster(s.T, s.namespace, "bluestore", false, useDevices, 3, installer.Version0_8)
|
||||
s.op, s.k8sh = StartTestCluster(s.T, s.namespace, "bluestore", false, useDevices, 3, installer.Version0_8, installer.LuminousVersion)
|
||||
s.helper = clients.CreateTestClient(s.k8sh, s.op.installer.Manifests)
|
||||
}
|
||||
|
||||
@@ -94,9 +95,9 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
|
||||
|
||||
// verify that we're actually running 0.8 before the upgrade
|
||||
operatorContainer := "rook-ceph-operator"
|
||||
version, err := k8sutil.GetDeploymentVersion(s.k8sh.Clientset, systemNamespace, operatorContainer, operatorContainer)
|
||||
version, err := k8sutil.GetDeploymentImage(s.k8sh.Clientset, systemNamespace, operatorContainer, operatorContainer)
|
||||
assert.Nil(s.T(), err)
|
||||
assert.Equal(s.T(), "v0.8.1", version)
|
||||
assert.Equal(s.T(), "rook/ceph:v0.8.3", version)
|
||||
|
||||
message := "my simple message"
|
||||
preFilename := "pre-upgrade-file"
|
||||
@@ -109,22 +110,22 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
|
||||
require.Nil(s.T(), s.k8sh.SetDeploymentVersion(systemNamespace, operatorContainer, operatorContainer, installer.VersionMaster))
|
||||
|
||||
// verify that the operator spec is updated
|
||||
version, err = k8sutil.GetDeploymentVersion(s.k8sh.Clientset, systemNamespace, operatorContainer, operatorContainer)
|
||||
version, err = k8sutil.GetDeploymentImage(s.k8sh.Clientset, systemNamespace, operatorContainer, operatorContainer)
|
||||
assert.Nil(s.T(), err)
|
||||
assert.Equal(s.T(), installer.VersionMaster, version)
|
||||
assert.Equal(s.T(), "rook/ceph:"+installer.VersionMaster, version)
|
||||
|
||||
// wait for the legacy mon replicasets to be deleted
|
||||
err = s.waitForLegacyMonReplicaSetDeletion()
|
||||
require.Nil(s.T(), err)
|
||||
|
||||
// wait for the mon pods to be running
|
||||
err = k8sutil.WaitForDeploymentVersion(s.k8sh.Clientset, s.namespace, "app=rook-ceph-mon", "rook-ceph-mon", installer.VersionMaster)
|
||||
err = k8sutil.WaitForDeploymentImage(s.k8sh.Clientset, s.namespace, "app=rook-ceph-mon", "mon", cephv1beta1.DefaultLuminousImage)
|
||||
require.Nil(s.T(), err)
|
||||
|
||||
s.k8sh.WaitForLabeledPodsToRun("app=rook-ceph-mon", s.namespace)
|
||||
|
||||
// wait for the osd pods to be updated
|
||||
err = k8sutil.WaitForDeploymentVersion(s.k8sh.Clientset, s.namespace, "app=rook-ceph-osd", "rook-ceph-osd", installer.VersionMaster)
|
||||
err = k8sutil.WaitForDeploymentImage(s.k8sh.Clientset, s.namespace, "app=rook-ceph-osd", "osd", cephv1beta1.DefaultLuminousImage)
|
||||
require.Nil(s.T(), err)
|
||||
|
||||
s.k8sh.WaitForLabeledPodsToRun("app=rook-ceph-osd", s.namespace)
|
||||
@@ -135,15 +136,15 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
|
||||
|
||||
// Test writing and reading from the pod with cephfs mounted that was created before the upgrade.
|
||||
postFilename := "post-upgrade-file"
|
||||
assert.Nil(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, preFilename, message, 3))
|
||||
assert.Nil(s.T(), s.k8sh.WriteToPodRetry(s.namespace, filePodName, postFilename, message, 3))
|
||||
assert.Nil(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, postFilename, message, 3))
|
||||
assert.Nil(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, preFilename, message, 5))
|
||||
assert.Nil(s.T(), s.k8sh.WriteToPodRetry(s.namespace, filePodName, postFilename, message, 5))
|
||||
assert.Nil(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, postFilename, message, 5))
|
||||
|
||||
// Test writing and reading from the pod with rbd mounted that was created before the upgrade.
|
||||
// There is some unreliability right after the upgrade when there is only one osd, so we will retry if needed
|
||||
assert.Nil(s.T(), s.k8sh.ReadFromPodRetry("", podName, preFilename, message, 3))
|
||||
assert.Nil(s.T(), s.k8sh.WriteToPodRetry("", podName, postFilename, message, 3))
|
||||
assert.Nil(s.T(), s.k8sh.ReadFromPodRetry("", podName, postFilename, message, 3))
|
||||
assert.Nil(s.T(), s.k8sh.ReadFromPodRetry("", podName, preFilename, message, 5))
|
||||
assert.Nil(s.T(), s.k8sh.WriteToPodRetry("", podName, postFilename, message, 5))
|
||||
assert.Nil(s.T(), s.k8sh.ReadFromPodRetry("", podName, postFilename, message, 5))
|
||||
}
|
||||
|
||||
func (s *UpgradeSuite) waitForLegacyMonReplicaSetDeletion() error {
|
||||
|
||||
@@ -86,7 +86,7 @@ func (s *BlockMountUnMountSuite) SetupSuite() {
|
||||
s.pvcNameRWX = "block-persistent-rwx"
|
||||
useHelm := false
|
||||
useDevices := true
|
||||
s.op, s.kh = StartTestCluster(s.T, s.namespace, "filestore", useHelm, useDevices, 1, installer.VersionMaster)
|
||||
s.op, s.kh = StartTestCluster(s.T, s.namespace, "filestore", useHelm, useDevices, 1, installer.VersionMaster, installer.LuminousVersion)
|
||||
s.testClient = clients.CreateTestClient(s.kh, s.op.installer.Manifests)
|
||||
s.bc = s.testClient.BlockClient
|
||||
}
|
||||
|
||||
@@ -147,7 +147,7 @@ func StartLoadTestCluster(t func() *testing.T, namespace string) (LoadTestCluste
|
||||
kh, err := utils.CreateK8sHelper(t)
|
||||
require.NoError(t(), err)
|
||||
|
||||
i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster)
|
||||
i := installer.NewCephInstaller(t, kh.Clientset, installer.VersionMaster, cephv1beta1.CephVersionSpec{Image: "ceph/ceph:v12.2.7", Name: "luminous"})
|
||||
|
||||
op := LoadTestCluster{i, kh, nil, t, namespace}
|
||||
op.Setup()
|
||||
|
||||
Reference in new issue
Block a user