From 2a37a403cecc3f86f19a92e4f89761dca02127cf Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Wed, 11 Nov 2020 13:55:40 -0700 Subject: [PATCH] docs: revise Ceph upgrade docs for v1.5 release Update docs and manifests as needed for v1.5 release. Signed-off-by: Blaine Gardner --- Documentation/ceph-upgrade.md | 199 ++--- .../ceph/upgrade-from-v1.3-apply.yaml | 352 --------- .../ceph/upgrade-from-v1.3-crds.yaml | 706 ------------------ .../ceph/upgrade-from-v1.3-delete.yaml | 79 -- 4 files changed, 59 insertions(+), 1277 deletions(-) delete mode 100644 cluster/examples/kubernetes/ceph/upgrade-from-v1.3-apply.yaml delete mode 100644 cluster/examples/kubernetes/ceph/upgrade-from-v1.3-crds.yaml delete mode 100644 cluster/examples/kubernetes/ceph/upgrade-from-v1.3-delete.yaml diff --git a/Documentation/ceph-upgrade.md b/Documentation/ceph-upgrade.md index 86c13a393..a12ddb054 100644 --- a/Documentation/ceph-upgrade.md +++ b/Documentation/ceph-upgrade.md @@ -10,15 +10,15 @@ This guide will walk you through the steps to upgrade the software in a Rook-Cep version to the next. This includes both the Rook-Ceph operator software itself as well as the Ceph cluster software. -Upgrades for both the operator and for Ceph are nearly entirely -automated save for where Rook's permissions need to be explicitly updated by an admin or when -incompatibilities need to be addressed manually due to customizations. +Upgrades for both the operator and for Ceph are nearly entirely automated save for where Rook's +permissions need to be explicitly updated by an admin or when incompatibilities need to be addressed +manually due to customizations. We welcome feedback and opening issues! ## Supported Versions -This guide is for upgrading from **Rook v1.3.x to Rook v1.4.x**. +This guide is for upgrading from **Rook v1.4.x to Rook v1.5.x**. Please refer to the upgrade guides from previous releases for supported upgrade paths. Rook upgrades are only supported between official releases. Upgrades to and from `master` are not @@ -27,6 +27,7 @@ supported. For a guide to upgrade previous versions of Rook, please refer to the version of documentation for those releases. +* [Upgrade 1.3 to 1.4](https://rook.io/docs/rook/v1.4/ceph-upgrade.html) * [Upgrade 1.2 to 1.3](https://rook.io/docs/rook/v1.3/ceph-upgrade.html) * [Upgrade 1.1 to 1.2](https://rook.io/docs/rook/v1.2/ceph-upgrade.html) * [Upgrade 1.0 to 1.1](https://rook.io/docs/rook/v1.1/ceph-upgrade.html) @@ -48,116 +49,34 @@ With this upgrade guide, there are a few notes to consider: ## Before you Upgrade -Rook v1.4 has a breaking change that should be considered before upgrading. - -1. Make sure the `lvm2` package is installed on the host where OSDs are running. If not, the prepare job will fail - and the upgrade of OSDs will be blocked. - * This only applies to non-PVC OSDs. Beginning in v1.3, OSDs created on PVCs no longer rely on LVM. -2. CSI Snapshots: See the next section if you are currently using Ceph-CSI snapshots. - - -### CSI Snapshots - -CSI snapshots have moved from Alpha to Beta and are **not** backward compatible. The snapshots -created with the Alpha version must be deleted before the upgrade. - -If you desire to continue using Ceph-CSI 2.x with Alpha snapshots, then: - -* Skip this section -* Set the CSI version to 2.x as described below in the [CSI Version](#csi-version) section. - -To continue with the Ceph-CSI v3.0 driver that is recommended with v1.4: - -1. List all the volumesnapshots created - -```bash -kubectl get volumesnapshot -NAME AGE -rbd-pvc-snapshot 22s -``` - -2. Delete all volumesnapshots - -```bash -kubectl delete volumesnapshot rbd-pvc-snapshot -volumesnapshot.snapshot.storage.k8s.io "rbd-pvc-snapshot" deleted -``` - -3. List all volumesnapshotclasses created - -```bash -kubectl get volumesnapshotclass -NAME AGE -csi-rbdplugin-snapclass 86s -``` - -4. Delete all volumesnapshotclasses - -```bash -kubectl delete volumesnapshotclass csi-rbdplugin-snapclass -volumesnapshotclass.snapshot.storage.k8s.io "csi-rbdplugin-snapclass" deleted -``` - -*Note:* The underlying snapshots on the storage system will be deleted by ceph-csi - -#### Delete the Alpha CRDs - -As we are updating the snapshot resources from `Alpha` to `Beta` we need to -delete the old `alphav1` snapshot CRD created by external-snapshotter sidecar container - -Check if we have any `v1alpha1` CRD created in our kubernetes cluster - -```console -kubectl get crd volumesnapshotclasses.snapshot.storage.k8s.io -o yaml |grep v1alpha1 - - name: v1alpha1 - - v1alpha1 -kubectl get crd volumesnapshotcontents.snapshot.storage.k8s.io -o yaml |grep v1alpha1 - - name: v1alpha1 - - v1alpha1 -kubectl get crd volumesnapshots.snapshot.storage.k8s.io -o yaml |grep v1alpha1 - - name: v1alpha1 - - v1alpha1 -``` - -As we have `v1alpha1` CRD created in our kubernetes cluster, we need to delete -the Alpha CRD - -```console -kubectl delete crd volumesnapshotclasses.snapshot.storage.k8s.io volumesnapshotcontents.snapshot.storage.k8s.io volumesnapshots.snapshot.storage.k8s.io - -customresourcedefinition.apiextensions.k8s.io "volumesnapshotclasses.snapshot.storage.k8s.io" deleted -customresourcedefinition.apiextensions.k8s.io "volumesnapshotcontents.snapshot.storage.k8s.io" deleted -customresourcedefinition.apiextensions.k8s.io "volumesnapshots.snapshot.storage.k8s.io" deleted -``` - -Finally, if you desire to use the beta snapshots, check that the [prerequisites](ceph-csi-snapshot.md#prerequisites) are met. - - ## Upgrading the Rook-Ceph Operator ## Patch Release Upgrades Unless otherwise noted due to extenuating requirements, upgrades from one patch release of Rook to -another are as simple as updating the image of the Rook operator. For example, when Rook v1.4.1 is -released, the process of updating from v1.4.0 is as simple as running the following: +another are as simple as updating the image of the Rook operator. For example, when Rook v1.5.1 is +released, the process of updating from v1.5.0 is as simple as running the following: ```console -kubectl -n rook-ceph set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v1.4.1 +kubectl -n rook-ceph set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v1.5.1 ``` ## Helm Upgrades If you have installed Rook via the Helm chart, Helm will handle some details of the upgrade for you. -In particular, Helm will handle updating the RBAC and trigger the operator update and restart. +The upgrade steps will clarify if Helm manages the step for you. + +Helm will **not** update the Ceph version. See [Ceph Version Upgrades](#ceph-version-upgrades) for +instructions on updating the Ceph version. -## Upgrading from v1.3 to v1.4 +## Upgrading from v1.4 to v1.5 **Rook releases from master are expressly unsupported.** It is strongly recommended that you use [official releases](https://github.com/rook/rook/releases) of Rook. Unreleased versions from the master branch are subject to changes and incompatibilities that will not be supported in the -official releases. Builds from the master branch can have functionality changed and even removed at -any time without compatibility support and without prior notice. +official releases. Builds from the master branch can have functionality changed or removed at any +time without compatibility support and without prior notice. ## Prerequisites @@ -167,8 +86,8 @@ We will do all our work in the Ceph example manifests directory. cd $YOUR_ROOK_REPO/cluster/examples/kubernetes/ceph/ ``` -Unless your Rook cluster was created with customized namespaces, namespaces for Rook clusters -created before v0.8 are likely to be: +Unless your Rook cluster was created with customized namespaces, namespaces for Rook clusters are +likely to be: * Clusters created by v0.7 or earlier: `rook-system` and `rook` * Clusters created in v0.8 or v0.9: `rook-ceph-system` and `rook-ceph` @@ -290,7 +209,7 @@ output. For example for the monitor pod `mon-b`, we can verify the container ver with the below commands: ```sh -POD_NAME=$(kubectl -n $ROOK_NAMESPACE get pod -o custom-columns=name:.metadata.name --no-headers | grep rook-ceph-mon-a) +POD_NAME=$(kubectl -n $ROOK_NAMESPACE get pod -o custom-columns=name:.metadata.name --no-headers | grep rook-ceph-mon-b) kubectl -n $ROOK_NAMESPACE get pod ${POD_NAME} -o jsonpath='{.spec.containers[0].image}' ``` @@ -324,9 +243,9 @@ Any pod that is using a Rook volume should also remain healthy: ## Rook Operator Upgrade Process -In the examples given in this guide, we will be upgrading a live Rook cluster running `v1.3.8` to -the version `v1.4.0`. This upgrade should work from any official patch release of Rook v1.3 to any -official patch release of v1.4. +In the examples given in this guide, we will be upgrading a live Rook cluster running `v1.4.7` to +the version `v1.5.0`. This upgrade should work from any official patch release of Rook v1.4 to any +official patch release of v1.5. **Rook release from `master` are expressly unsupported.** It is strongly recommended that you use [official releases](https://github.com/rook/rook/releases) of Rook. Unreleased versions from the @@ -336,13 +255,7 @@ time without compatibility support and without prior notice. Let's get started! -## 1. Remove Alpha Snapshots - -As described above in the [CSI Snapshots](#csi-snapshots) section, if you are using alpha snapshots -it is required to remove them before the upgrade since they are not compatible with beta snaphots -available with Ceph-CSI v3.0. - -## 2. Update the RBAC and CRDs +## 1. Update the RBAC and CRDs > Automatically updated if you are upgrading via the helm chart @@ -352,32 +265,30 @@ and updates to the Custom Resource Definitions (CRDs). > **IMPORTANT:** If you are using Kubernetes version v1.15 or lower, you will need to manually > modify the `upgrade-from-v1.4-apply.yaml` file to use > `rbac.authorization.k8s.io/v1beta1` instead of `rbac.authorization.k8s.io/v1` -> You will also need to apply `pre-k8s-1.16/crd.yaml` instead of `upgrade-from-v1.4-crds.yaml`. +> You will also need to apply `pre-k8s-1.16/crds.yaml` instead of `crds.yaml`. ```sh -kubectl apply -f upgrade-from-v1.4-apply.yaml -f upgrade-from-v1.4-crds.yaml +kubectl apply -f upgrade-from-v1.4-apply.yaml -f crds.yaml ``` -## 3. Update Ceph CSI version to v3.0 +## 2. Update Ceph CSI versions -> Rook v1.4 will install Ceph-CSI v3.0 use the latest drivers by default. -> If you have not specified custom CSI images in the Operator deployment this step is unnecessary. +If you have specified custom CSI images in the Rook-Ceph Operator deployment, we recommended you +update to use the latest Ceph-CSI drivers. See the [CSI Version](#csi-version) section for more +details. -If you have specified custom CSI images in the Rook-Ceph Operator deployment, it is recommended to update to use the latest Ceph-CSI v3.0 driver. -See the section [CSI Version](#csi-version) for more details. - -## 4. Update the Rook Operator +## 3. Update the Rook Operator > Automatically updated if you are upgrading via the helm chart -The largest portion of the upgrade is triggered when the operator's image is updated to `v1.4.x`. +The largest portion of the upgrade is triggered when the operator's image is updated to `v1.5.x`. When the operator is updated, it will proceed to update all of the Ceph daemons. ```sh -kubectl -n $ROOK_SYSTEM_NAMESPACE set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v1.4.0 +kubectl -n $ROOK_SYSTEM_NAMESPACE set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v1.5.0 ``` -## 5. Wait for the upgrade to complete +## 4. Wait for the upgrade to complete Watch now in amazement as the Ceph mons, mgrs, OSDs, rbd-mirrors, MDSes and RGWs are terminated and replaced with updated versions in sequence. The cluster may be offline very briefly as mons update, @@ -389,20 +300,20 @@ The versions of the components can be viewed as they are updated: watch --exec kubectl -n $ROOK_NAMESPACE get deployments -l rook_cluster=$ROOK_NAMESPACE -o jsonpath='{range .items[*]}{.metadata.name}{" \treq/upd/avl: "}{.spec.replicas}{"/"}{.status.updatedReplicas}{"/"}{.status.readyReplicas}{" \trook-version="}{.metadata.labels.rook-version}{"\n"}{end}' ``` -As an example, this cluster is midway through updating the OSDs from v1.3 to v1.4. When all -deployments report `1/1/1` availability and `rook-version=v1.4.0`, the Ceph cluster's core +As an example, this cluster is midway through updating the OSDs from v1.4 to v1.5. When all +deployments report `1/1/1` availability and `rook-version=v1.5.0`, the Ceph cluster's core components are fully updated. ```console Every 2.0s: kubectl -n rook-ceph get deployment -o j... -rook-ceph-mgr-a req/upd/avl: 1/1/1 rook-version=v1.4.0 -rook-ceph-mon-a req/upd/avl: 1/1/1 rook-version=v1.4.0 -rook-ceph-mon-b req/upd/avl: 1/1/1 rook-version=v1.4.0 -rook-ceph-mon-c req/upd/avl: 1/1/1 rook-version=v1.4.0 -rook-ceph-osd-0 req/upd/avl: 1// rook-version=v1.4.0 -rook-ceph-osd-1 req/upd/avl: 1/1/1 rook-version=v1.3.8 -rook-ceph-osd-2 req/upd/avl: 1/1/1 rook-version=v1.3.8 +rook-ceph-mgr-a req/upd/avl: 1/1/1 rook-version=v1.5.0 +rook-ceph-mon-a req/upd/avl: 1/1/1 rook-version=v1.5.0 +rook-ceph-mon-b req/upd/avl: 1/1/1 rook-version=v1.5.0 +rook-ceph-mon-c req/upd/avl: 1/1/1 rook-version=v1.5.0 +rook-ceph-osd-0 req/upd/avl: 1// rook-version=v1.5.0 +rook-ceph-osd-1 req/upd/avl: 1/1/1 rook-version=v1.4.7 +rook-ceph-osd-2 req/upd/avl: 1/1/1 rook-version=v1.4.7 ``` An easy check to see if the upgrade is totally @@ -411,13 +322,13 @@ finished is to check that there is only one `rook-version` reported across the c ```console # kubectl -n $ROOK_NAMESPACE get deployment -l rook_cluster=$ROOK_NAMESPACE -o jsonpath='{range .items[*]}{"rook-version="}{.metadata.labels.rook-version}{"\n"}{end}' | sort | uniq This cluster is not yet finished: - rook-version=v1.3.8 - rook-version=v1.4.0 + rook-version=v1.4.7 + rook-version=v1.5.0 This cluster is finished: - rook-version=v1.4.0 + rook-version=v1.5.0 ``` -## 6. Verify the updated cluster +## 5. Verify the updated cluster At this point, your Rook operator should be running version `rook/ceph:v1.4.0`. @@ -436,7 +347,7 @@ updated in the CR), Rook will go through all the daemons one by one and will ind checks on them. It will make sure a particular daemon can be stopped before performing the upgrade. Once the deployment has been updated, it checks if this is ok to continue. After each daemon is updated we wait for things to settle (monitors to be in a quorum, PGs to be clean for OSDs, up for -MDSs, etc.), then only when the condition is met we move to the next daemon. We repeat this process +MDSes, etc.), then only when the condition is met we move to the next daemon. We repeat this process until all the daemons have been updated. ### Ceph images @@ -454,7 +365,15 @@ These images are tagged in a few ways: ### Example upgrade to Ceph Octopus -#### 1. Update the main Ceph daemons +#### 1. Update the CRDs + +It is a good practice to update Rook-Ceph CRDs from the example manifests before any update. + +```sh +kubectl apply -f crds.yaml +``` + +#### 2. Update the main Ceph daemons The majority of the upgrade will be handled by the Rook operator. Begin the upgrade by changing the Ceph image field in the cluster CRD (`spec.cephVersion.image`). @@ -465,7 +384,7 @@ CLUSTER_NAME="$ROOK_NAMESPACE" # change if your cluster name is not the Rook na kubectl -n $ROOK_NAMESPACE patch CephCluster $CLUSTER_NAME --type=merge -p "{\"spec\": {\"cephVersion\": {\"image\": \"$NEW_CEPH_IMAGE\"}}}" ``` -#### 2. Wait for the daemon pod updates to complete +#### 3. Wait for the daemon pod updates to complete As with upgrading Rook, you must now wait for the upgrade to complete. Status can be determined in a similar way to the Rook upgrade as well. @@ -485,7 +404,7 @@ This cluster is finished: ceph-version=15.2.4-0 ``` -#### 3. Verify the updated cluster +#### 4. Verify the updated cluster Verify the Ceph cluster's health using the [health verification section](#health-verification). @@ -499,8 +418,8 @@ The operator configuration variables have recently moved from the operator deplo `rook-ceph-operator-config` ConfigMap. The values in the operator deployment can still be set, but if the ConfigMap settings are applied, they will override the operator deployment settings. -> If the cluster was originally installed prior to v1.3, the configmap may not exist. -> See the latest operator.yaml for example configmap settings. +> If the cluster was originally installed prior to v1.3, the ConfigMap may not exist. +> See the latest `operator.yaml` for example ConfigMap settings. ```console kubectl -n $ROOK_NAMESPACE edit configmap rook-ceph-operator-config diff --git a/cluster/examples/kubernetes/ceph/upgrade-from-v1.3-apply.yaml b/cluster/examples/kubernetes/ceph/upgrade-from-v1.3-apply.yaml deleted file mode 100644 index db2b1a723..000000000 --- a/cluster/examples/kubernetes/ceph/upgrade-from-v1.3-apply.yaml +++ /dev/null @@ -1,352 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-global - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - # Pod access is needed for fencing - - pods - # Node access is needed for determining nodes where mons should run - - nodes - - nodes/proxy - - services - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - # PVs and PVCs are managed by the Rook provisioner - - persistentvolumes - - persistentvolumeclaims - - endpoints - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - policy - - apps - - extensions - resources: - # This is for the clusterdisruption controller - - poddisruptionbudgets - # This is for both clusterdisruption and nodedrain controllers - - deployments - - replicasets - verbs: - - "*" -- apiGroups: - - healthchecking.openshift.io - resources: - - machinedisruptionbudgets - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - machine.openshift.io - resources: - - machines - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - csidrivers - verbs: - - create - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-cluster-mgmt - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - - apps - - extensions - resources: - - secrets - - pods - - pods/log - - services - - configmaps - - deployments - - daemonsets - verbs: - - get - - list - - watch - - patch - - create - - update - - delete ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-cluster - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - - configmaps - - nodes - - nodes/proxy - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - list - - get - - watch ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-object-bucket - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - verbs: - - "*" - resources: - - secrets - - configmaps -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - "objectbucket.io" - verbs: - - "*" - resources: - - "*" ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-system -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-provisioner-runner -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-provisioner-runner -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] diff --git a/cluster/examples/kubernetes/ceph/upgrade-from-v1.3-crds.yaml b/cluster/examples/kubernetes/ceph/upgrade-from-v1.3-crds.yaml deleted file mode 100644 index c48303246..000000000 --- a/cluster/examples/kubernetes/ceph/upgrade-from-v1.3-crds.yaml +++ /dev/null @@ -1,706 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephclusters.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephCluster - listKind: CephClusterList - plural: cephclusters - singular: cephcluster - scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: - properties: - annotations: {} - cephVersion: - properties: - allowUnsupported: - type: boolean - image: - type: string - dashboard: - properties: - enabled: - type: boolean - urlPrefix: - type: string - port: - type: integer - minimum: 0 - maximum: 65535 - ssl: - type: boolean - dataDirHostPath: - pattern: ^/(\S+) - type: string - disruptionManagement: - properties: - machineDisruptionBudgetNamespace: - type: string - managePodBudgets: - type: boolean - osdMaintenanceTimeout: - type: integer - manageMachineDisruptionBudgets: - type: boolean - skipUpgradeChecks: - type: boolean - continueUpgradeAfterChecksEvenIfNotHealthy: - type: boolean - mon: - properties: - allowMultiplePerNode: - type: boolean - count: - maximum: 9 - minimum: 0 - type: integer - volumeClaimTemplate: {} - mgr: - properties: - modules: - items: - properties: - name: - type: string - enabled: - type: boolean - network: - properties: - hostNetwork: - type: boolean - provider: - type: string - selectors: {} - storage: - properties: - disruptionManagement: - properties: - machineDisruptionBudgetNamespace: - type: string - managePodBudgets: - type: boolean - osdMaintenanceTimeout: - type: integer - manageMachineDisruptionBudgets: - type: boolean - useAllNodes: - type: boolean - nodes: - items: - properties: - name: - type: string - config: - properties: - metadataDevice: - type: string - storeType: - type: string - pattern: ^(bluestore)$ - databaseSizeMB: - type: string - walSizeMB: - type: string - journalSizeMB: - type: string - osdsPerDevice: - type: string - encryptedDevice: - type: string - pattern: ^(true|false)$ - useAllDevices: - type: boolean - deviceFilter: - type: string - devicePathFilter: - type: string - devices: - type: array - items: - properties: - name: - type: string - config: {} - resources: {} - type: array - useAllDevices: - type: boolean - deviceFilter: - type: string - devicePathFilter: - type: string - config: {} - storageClassDeviceSets: {} - driveGroups: - type: array - nullable: true - items: - properties: - name: - type: string - spec: {} - placement: {} - required: - - name - - spec - monitoring: - properties: - enabled: - type: boolean - rulesNamespace: - type: string - externalMgrEndpoints: - type: array - items: - properties: - ip: - type: string - removeOSDsIfOutAndSafeToRemove: - type: boolean - external: - properties: - enable: - type: boolean - cleanupPolicy: - properties: - confirmation: - type: string - pattern: ^$|^yes-really-destroy-data$ - sanitizeDisks: - properties: - method: - type: string - pattern: ^(complete|quick)$ - dataSource: - type: string - pattern: ^(zero|random)$ - iteration: - type: integer - format: int32 - placement: {} - resources: {} - healthCheck: {} - subresources: - status: {} - additionalPrinterColumns: - - name: DataDirHostPath - type: string - description: Directory used on the K8s nodes - JSONPath: .spec.dataDirHostPath - - name: MonCount - type: string - description: Number of MONs - JSONPath: .spec.mon.count - - name: Age - type: date - JSONPath: .metadata.creationTimestamp - - name: Phase - type: string - description: Phase - JSONPath: .status.phase - - name: Message - type: string - description: Message - JSONPath: .status.message - - name: Health - type: string - description: Ceph Health - JSONPath: .status.ceph.health ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephclients.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephClient - listKind: CephClientList - plural: cephclients - singular: cephclient - scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: - properties: - caps: - type: object - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephrbdmirrors.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephRBDMirror - listKind: CephRBDMirrorList - plural: cephrbdmirrors - singular: cephrbdmirror - scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: - properties: - count: - type: integer - minimum: 1 - maximum: 100 - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephfilesystems.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephFilesystem - listKind: CephFilesystemList - plural: cephfilesystems - singular: cephfilesystem - scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: - properties: - metadataServer: - properties: - activeCount: - minimum: 1 - maximum: 10 - type: integer - activeStandby: - type: boolean - annotations: {} - placement: {} - resources: {} - metadataPool: - properties: - failureDomain: - type: string - crushRoot: - type: string - replicated: - properties: - size: - minimum: 0 - maximum: 10 - type: integer - requireSafeReplicaSize: - type: boolean - erasureCoded: - properties: - dataChunks: - minimum: 0 - maximum: 10 - type: integer - codingChunks: - minimum: 0 - maximum: 10 - type: integer - compressionMode: - type: string - enum: - - "" - - none - - passive - - aggressive - - force - dataPools: - type: array - items: - properties: - failureDomain: - type: string - crushRoot: - type: string - replicated: - properties: - size: - minimum: 0 - maximum: 10 - type: integer - requireSafeReplicaSize: - type: boolean - erasureCoded: - properties: - dataChunks: - minimum: 0 - maximum: 10 - type: integer - codingChunks: - minimum: 0 - maximum: 10 - type: integer - compressionMode: - type: string - enum: - - "" - - none - - passive - - aggressive - - force - parameters: - type: object - preservePoolsOnDelete: - type: boolean - preserveFilesystemOnDelete: - type: boolean - additionalPrinterColumns: - - name: ActiveMDS - type: string - description: Number of desired active MDS daemons - JSONPath: .spec.metadataServer.activeCount - - name: Age - type: date - JSONPath: .metadata.creationTimestamp - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephnfses.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephNFS - listKind: CephNFSList - plural: cephnfses - singular: cephnfs - shortNames: - - nfs - scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: - properties: - rados: - properties: - pool: - type: string - namespace: - type: string - server: - properties: - active: - type: integer - annotations: {} - placement: {} - resources: {} - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephobjectstores.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephObjectStore - listKind: CephObjectStoreList - plural: cephobjectstores - singular: cephobjectstore - scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: - properties: - gateway: - properties: - type: - type: string - sslCertificateRef: {} - port: - type: integer - minimum: 1 - maximum: 65535 - securePort: {} - instances: - type: integer - externalRgwEndpoints: - type: array - items: - properties: - ip: - type: string - annotations: {} - placement: {} - resources: {} - metadataPool: - properties: - failureDomain: - type: string - crushRoot: - type: string - replicated: - properties: - size: - type: integer - requireSafeReplicaSize: - type: boolean - erasureCoded: - properties: - dataChunks: - type: integer - codingChunks: - type: integer - compressionMode: - type: string - enum: - - "" - - none - - passive - - aggressive - - force - parameters: - type: object - dataPool: - properties: - failureDomain: - type: string - crushRoot: - type: string - replicated: - properties: - size: - type: integer - requireSafeReplicaSize: - type: boolean - erasureCoded: - properties: - dataChunks: - type: integer - codingChunks: - type: integer - compressionMode: - type: string - enum: - - "" - - none - - passive - - aggressive - - force - parameters: - type: object - preservePoolsOnDelete: - type: boolean - healthCheck: - properties: - bucket: - properties: - enabled: - type: boolean - interval: - type: string - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephobjectstoreusers.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephObjectStoreUser - listKind: CephObjectStoreUserList - plural: cephobjectstoreusers - singular: cephobjectstoreuser - shortNames: - - rcou - - objectuser - scope: Namespaced - version: v1 - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephobjectrealms.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephObjectRealm - listKind: CephObjectRealmList - plural: cephobjectrealms - singular: cephobjectrealm - scope: Namespaced - version: v1 - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephobjectzonegroups.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephObjectZoneGroup - listKind: CephObjectZoneGroupList - plural: cephobjectzonegroups - singular: cephobjectzonegroup - scope: Namespaced - version: v1 - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephobjectzones.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephObjectZone - listKind: CephObjectZoneList - plural: cephobjectzones - singular: cephobjectzone - scope: Namespaced - version: v1 - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephblockpools.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephBlockPool - listKind: CephBlockPoolList - plural: cephblockpools - singular: cephblockpool - scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: - properties: - failureDomain: - type: string - crushRoot: - type: string - replicated: - properties: - size: - type: integer - minimum: 0 - maximum: 9 - targetSizeRatio: - type: number - requireSafeReplicaSize: - type: boolean - erasureCoded: - properties: - dataChunks: - type: integer - minimum: 0 - maximum: 9 - codingChunks: - type: integer - minimum: 0 - maximum: 9 - compressionMode: - type: string - enum: - - "" - - none - - passive - - aggressive - - force - enableRBDStats: - description: EnableRBDStats is used to enable gathering of statistics - for all RBD images in the pool - type: boolean - parameters: - type: object - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: volumes.rook.io -spec: - group: rook.io - names: - kind: Volume - listKind: VolumeList - plural: volumes - singular: volume - shortNames: - - rv - scope: Namespaced - version: v1alpha2 - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: objectbuckets.objectbucket.io -spec: - group: objectbucket.io - versions: - - name: v1alpha1 - served: true - storage: true - names: - kind: ObjectBucket - listKind: ObjectBucketList - plural: objectbuckets - singular: objectbucket - shortNames: - - ob - - obs - scope: Cluster - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: objectbucketclaims.objectbucket.io -spec: - versions: - - name: v1alpha1 - served: true - storage: true - group: objectbucket.io - names: - kind: ObjectBucketClaim - listKind: ObjectBucketClaimList - plural: objectbucketclaims - singular: objectbucketclaim - shortNames: - - obc - - obcs - scope: Namespaced - subresources: - status: {} diff --git a/cluster/examples/kubernetes/ceph/upgrade-from-v1.3-delete.yaml b/cluster/examples/kubernetes/ceph/upgrade-from-v1.3-delete.yaml deleted file mode 100644 index 4f092e2d2..000000000 --- a/cluster/examples/kubernetes/ceph/upgrade-from-v1.3-delete.yaml +++ /dev/null @@ -1,79 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: cephfs-csi-nodeplugin-rules ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: cephfs-external-provisioner-runner-rules ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rbd-csi-nodeplugin-rules ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rbd-external-provisioner-runner-rules ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rook-ceph-cluster-mgmt-rules ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rook-ceph-global-rules ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rook-ceph-mgr-cluster-rules ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rook-ceph-mgr-system-rules ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: cephfs-csi-nodeplugin ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: cephfs-external-provisioner-runner ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rbd-csi-nodeplugin ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rbd-external-provisioner-runner ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rook-ceph-cluster-mgmt ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rook-ceph-global ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rook-ceph-mgr-cluster ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rook-ceph-mgr-system