forked from rook/rook
Merge pull request #8740 from leseb/examples-layout
core: change directory layout
This commit is contained in:
Sébastien Han
@@ -1,8 +1,8 @@
|
||||
extends: default
|
||||
ignore: |
|
||||
cluster/examples/kubernetes/ceph/csi/template
|
||||
cluster/examples/kubernetes/ceph/crds.yaml
|
||||
cluster/examples/kubernetes/ceph/monitoring/
|
||||
deploy/examples/csi/template
|
||||
deploy/examples/crds.yaml
|
||||
deploy/examples/monitoring/
|
||||
rules:
|
||||
line-length: disable
|
||||
new-lines: disable
|
||||
|
||||
@@ -55,8 +55,8 @@ jobs:
|
||||
mgr_raw=$(kubectl -n rook-ceph exec $toolbox -- ceph mgr dump -f json|jq --raw-output .active_addr)
|
||||
timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- curl --silent --show-error ${mgr_raw%%:*}:9283; do echo 'waiting for mgr prometheus exporter to be ready' && sleep 1; done"
|
||||
kubectl -n rook-ceph exec $toolbox -- mkdir -p /etc/ceph/test-data
|
||||
kubectl -n rook-ceph cp cluster/examples/kubernetes/ceph/test-data/ceph-status-out $toolbox:/etc/ceph/test-data/
|
||||
kubectl -n rook-ceph cp cluster/examples/kubernetes/ceph/create-external-cluster-resources.py $toolbox:/etc/ceph
|
||||
kubectl -n rook-ceph cp tests/ceph-status-out $toolbox:/etc/ceph/test-data/
|
||||
kubectl -n rook-ceph cp deploy/examples/create-external-cluster-resources.py $toolbox:/etc/ceph
|
||||
timeout 10 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool; do echo 'waiting for script to succeed' && sleep 1; done"
|
||||
|
||||
- name: run external script create-external-cluster-resources.py unit tests
|
||||
@@ -73,11 +73,11 @@ jobs:
|
||||
run: |
|
||||
kubectl -n rook-ceph delete deploy/rook-ceph-operator
|
||||
kubectl -n rook-ceph delete deploy/rook-ceph-osd-1 --grace-period=0 --force
|
||||
sed -i 's/<OSD-IDs>/1/' cluster/examples/kubernetes/ceph/osd-purge.yaml
|
||||
sed -i 's/<OSD-IDs>/1/' deploy/examples/osd-purge.yaml
|
||||
# the CI must force the deletion since we use replica 1 on 2 OSDs
|
||||
sed -i 's/false/true/' cluster/examples/kubernetes/ceph/osd-purge.yaml
|
||||
sed -i 's|rook/ceph:master|rook/ceph:local-build|' cluster/examples/kubernetes/ceph/osd-purge.yaml
|
||||
kubectl -n rook-ceph create -f cluster/examples/kubernetes/ceph/osd-purge.yaml
|
||||
sed -i 's/false/true/' deploy/examples/osd-purge.yaml
|
||||
sed -i 's|rook/ceph:master|rook/ceph:local-build|' deploy/examples/osd-purge.yaml
|
||||
kubectl -n rook-ceph create -f deploy/examples/osd-purge.yaml
|
||||
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
||||
kubectl -n rook-ceph exec $toolbox -- ceph status
|
||||
timeout 120 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph osd tree|grep -qE 'osd.1.*.destroyed'; do echo 'waiting for ceph osd 1 to be destroyed'; sleep 1; done"
|
||||
@@ -122,12 +122,12 @@ jobs:
|
||||
|
||||
- name: deploy cluster
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/operator.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
||||
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
|
||||
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
|
||||
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
|
||||
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/toolbox.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
||||
|
||||
- name: wait for prepare pod
|
||||
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod
|
||||
@@ -174,11 +174,11 @@ jobs:
|
||||
|
||||
- name: deploy cluster
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/operator.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
||||
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
|
||||
cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
||||
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/toolbox.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
||||
|
||||
- name: wait for prepare pod
|
||||
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod
|
||||
@@ -222,12 +222,12 @@ jobs:
|
||||
|
||||
- name: deploy rook
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/operator.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
||||
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
|
||||
cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
||||
cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
||||
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/toolbox.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
||||
|
||||
- name: wait for prepare pod
|
||||
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod
|
||||
@@ -274,11 +274,11 @@ jobs:
|
||||
|
||||
- name: deploy cluster
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/operator.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
||||
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
|
||||
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
|
||||
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/toolbox.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
||||
|
||||
- name: wait for prepare pod
|
||||
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod
|
||||
@@ -325,10 +325,10 @@ jobs:
|
||||
|
||||
- name: deploy cluster
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/operator.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
||||
cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
||||
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/toolbox.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
||||
|
||||
- name: wait for prepare pod
|
||||
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod
|
||||
@@ -375,11 +375,11 @@ jobs:
|
||||
|
||||
- name: deploy rook
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/operator.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
||||
cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
||||
cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
||||
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/toolbox.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
||||
|
||||
- name: wait for prepare pod
|
||||
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod
|
||||
@@ -430,7 +430,7 @@ jobs:
|
||||
|
||||
- name: deploy cluster
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/operator.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
||||
cat tests/manifests/test-kms-vault.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
||||
yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml
|
||||
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
|
||||
@@ -439,7 +439,7 @@ jobs:
|
||||
yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml
|
||||
yq write -i tests/manifests/test-object.yaml "spec.security.kms.connectionDetails.VAULT_BACKEND_PATH" rook/ver2
|
||||
kubectl create -f tests/manifests/test-object.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/toolbox.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
||||
|
||||
- name: wait for prepare pod
|
||||
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod
|
||||
@@ -500,12 +500,12 @@ jobs:
|
||||
|
||||
- name: deploy cluster
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/operator.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
||||
yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec-k8s-auth.yaml
|
||||
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
|
||||
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
|
||||
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/toolbox.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
||||
|
||||
- name: wait for prepare pod
|
||||
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod
|
||||
@@ -554,10 +554,10 @@ jobs:
|
||||
|
||||
- name: deploy cluster
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/operator.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
||||
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
|
||||
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/toolbox.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
||||
|
||||
- name: wait for prepare pod
|
||||
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod
|
||||
@@ -603,7 +603,7 @@ jobs:
|
||||
- name: deploy first cluster rook
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh deploy_first_rook_cluster
|
||||
cd cluster/examples/kubernetes/ceph/
|
||||
cd deploy/examples/
|
||||
kubectl create -f rbdmirror.yaml -f filesystem-mirror.yaml
|
||||
|
||||
# cephfs-mirroring is a push operation
|
||||
@@ -611,7 +611,7 @@ jobs:
|
||||
- name: deploy second cluster rook
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh deploy_second_rook_cluster
|
||||
cd cluster/examples/kubernetes/ceph/
|
||||
cd deploy/examples/
|
||||
sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' rbdmirror.yaml
|
||||
kubectl create -f rbdmirror.yaml
|
||||
|
||||
@@ -623,7 +623,7 @@ jobs:
|
||||
|
||||
- name: create replicated mirrored pool on cluster 1
|
||||
run: |
|
||||
cd cluster/examples/kubernetes/ceph/
|
||||
cd deploy/examples/
|
||||
yq w -i pool-test.yaml spec.mirroring.enabled true
|
||||
yq w -i pool-test.yaml spec.mirroring.mode image
|
||||
kubectl create -f pool-test.yaml
|
||||
@@ -631,7 +631,7 @@ jobs:
|
||||
|
||||
- name: create replicated mirrored pool 2 on cluster 1
|
||||
run: |
|
||||
cd cluster/examples/kubernetes/ceph/
|
||||
cd deploy/examples/
|
||||
yq w -i pool-test.yaml metadata.name replicapool2
|
||||
kubectl create -f pool-test.yaml
|
||||
timeout 60 sh -c 'until [ "$(kubectl -n rook-ceph get cephblockpool replicapool2 -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool2 to created on cluster 2" && sleep 1; done'
|
||||
@@ -639,14 +639,14 @@ jobs:
|
||||
|
||||
- name: create replicated mirrored pool on cluster 2
|
||||
run: |
|
||||
cd cluster/examples/kubernetes/ceph/
|
||||
cd deploy/examples/
|
||||
yq w -i pool-test.yaml metadata.namespace rook-ceph-secondary
|
||||
kubectl create -f pool-test.yaml
|
||||
timeout 60 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool to created on cluster 1" && sleep 1; done'
|
||||
|
||||
- name: create replicated mirrored pool 2 on cluster 2
|
||||
run: |
|
||||
cd cluster/examples/kubernetes/ceph/
|
||||
cd deploy/examples/
|
||||
yq w -i pool-test.yaml metadata.name replicapool2
|
||||
kubectl create -f pool-test.yaml
|
||||
timeout 60 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool2 to created on cluster 2" && sleep 1; done'
|
||||
@@ -719,15 +719,15 @@ jobs:
|
||||
|
||||
- name: create replicated mirrored filesystem on cluster 1
|
||||
run: |
|
||||
PRIMARY_YAML=cluster/examples/kubernetes/ceph/filesystem-test-primary.yaml
|
||||
cp cluster/examples/kubernetes/ceph/filesystem-test.yaml "$PRIMARY_YAML"
|
||||
PRIMARY_YAML=deploy/examples/filesystem-test-primary.yaml
|
||||
cp deploy/examples/filesystem-test.yaml "$PRIMARY_YAML"
|
||||
yq merge --inplace --arrays append "$PRIMARY_YAML" tests/manifests/test-fs-mirror-spec.yaml
|
||||
kubectl create -f "$PRIMARY_YAML"
|
||||
timeout 120 sh -c 'until [ "$(kubectl -n rook-ceph get cephfilesystem myfs -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for filesystem to be created" && sleep 1; done'
|
||||
|
||||
- name: create replicated mirrored filesystem on cluster 2
|
||||
run: |
|
||||
cd cluster/examples/kubernetes/ceph/
|
||||
cd deploy/examples/
|
||||
yq w -i filesystem-test.yaml metadata.namespace rook-ceph-secondary
|
||||
yq w -i filesystem-test.yaml spec.mirroring.enabled true
|
||||
kubectl create -f filesystem-test.yaml
|
||||
|
||||
@@ -58,12 +58,12 @@ jobs:
|
||||
# Use the official build images for the nightly arm tests instead of rebuilding
|
||||
export USE_LOCAL_BUILD=false
|
||||
# removing liveness probes since the env is slow and the probe is killing the daemons
|
||||
yq write -d1 -i cluster/examples/kubernetes/ceph/cluster-test.yaml "spec.healthCheck.livenessProbe.mon.disabled" true
|
||||
yq write -d1 -i cluster/examples/kubernetes/ceph/cluster-test.yaml "spec.healthCheck.livenessProbe.mgr.disabled" true
|
||||
yq write -d1 -i cluster/examples/kubernetes/ceph/cluster-test.yaml "spec.healthCheck.livenessProbe.osd.disabled" true
|
||||
yq write -d1 -i deploy/examples/cluster-test.yaml "spec.healthCheck.livenessProbe.mon.disabled" true
|
||||
yq write -d1 -i deploy/examples/cluster-test.yaml "spec.healthCheck.livenessProbe.mgr.disabled" true
|
||||
yq write -d1 -i deploy/examples/cluster-test.yaml "spec.healthCheck.livenessProbe.osd.disabled" true
|
||||
tests/scripts/github-action-helper.sh deploy_cluster
|
||||
# there are no package for arm64 nfs-ganesha
|
||||
kubectl delete -f cluster/examples/kubernetes/ceph/nfs-test.yaml
|
||||
kubectl delete -f deploy/examples/nfs-test.yaml
|
||||
|
||||
- name: wait for prepare pod
|
||||
run: timeout 900 sh -c 'until kubectl -n rook-ceph logs -f $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}'); do sleep 5; done' || kubectl -n rook-ceph get all && kubectl logs -n rook-ceph deploy/rook-ceph-operator
|
||||
|
||||
@@ -35,4 +35,4 @@ jobs:
|
||||
uses: helm/chart-testing-action@v2.1.0
|
||||
|
||||
- name: Run chart-testing (lint)
|
||||
run: ct lint --charts=./cluster/charts/rook-ceph --validate-yaml=false --validate-maintainers=false
|
||||
run: ct lint --charts=./deploy/charts/rook-ceph --validate-yaml=false --validate-maintainers=false
|
||||
|
||||
@@ -33,20 +33,20 @@ runs:
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh replace_ceph_image \
|
||||
"cluster/examples/kubernetes/ceph/cluster-test.yaml" "${{ inputs.ceph-image }}"
|
||||
"deploy/examples/cluster-test.yaml" "${{ inputs.ceph-image }}"
|
||||
|
||||
- name: deploy first cluster rook
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh deploy_first_rook_cluster
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-test.yaml
|
||||
kubectl create -f deploy/examples/object-multisite-test.yaml
|
||||
# wait for multisite store to be created
|
||||
tests/scripts/github-action-helper.sh wait_for_rgw rook-ceph
|
||||
|
||||
- name: prep second cluster pull realm config
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
run: |
|
||||
cd cluster/examples/kubernetes/ceph/
|
||||
cd deploy/examples/
|
||||
IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}")
|
||||
yq w -i -d1 object-multisite-pull-realm-test.yaml spec.pull.endpoint http://${IP_ADDR}:80
|
||||
BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}")
|
||||
@@ -58,7 +58,7 @@ runs:
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh deploy_second_rook_cluster
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml
|
||||
kubectl create -f deploy/examples/object-multisite-pull-realm-test.yaml
|
||||
# wait for realms to be pulled and zone-b-multisite-store to be created
|
||||
tests/scripts/github-action-helper.sh wait_for_rgw rook-ceph-secondary
|
||||
|
||||
|
||||
@@ -28,4 +28,4 @@ jobs:
|
||||
run: pip install yamllint
|
||||
|
||||
- name: Lint YAML files
|
||||
run: yamllint -c .github/workflows/.yamllint cluster/examples/kubernetes/ --no-warnings
|
||||
run: yamllint -c .github/workflows/.yamllint deploy/examples/ --no-warnings
|
||||
|
||||
+2
-2
@@ -8,5 +8,5 @@
|
||||
/tests/integration/rook-test/
|
||||
|
||||
# OLM related stuff
|
||||
cluster/olm/ceph/deploy/*
|
||||
cluster/olm/ceph/templates/*
|
||||
deploy/olm/deploy/*
|
||||
deploy/olm/templates/*
|
||||
|
||||
@@ -19,13 +19,14 @@ This script will help us achieve the following tasks
|
||||
2. Creates ValidatingWebhookConfig and fills the CA bundle with the appropriate value from the cluster.
|
||||
|
||||
Run the following commands:
|
||||
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/crds.yaml -f cluster/examples/kubernetes/ceph/common.yaml
|
||||
kubectl create -f deploy/examples/crds.yaml -f deploy/examples/common.yaml
|
||||
tests/scripts/deploy_admission_controller.sh
|
||||
```
|
||||
Now that the Secrets have been deployed, we can deploy the operator:
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml
|
||||
kubectl create -f deploy/examples/operator.yaml
|
||||
```
|
||||
|
||||
At this point the operator will start the admission controller Deployment automatically and the Webhook will start intercepting requests for Rook resources.
|
||||
|
||||
@@ -41,8 +41,8 @@ The service accounts are:
|
||||
* `rook-ceph-mgr` (namespace: `rook-ceph`): Will affect the MGR pods in the `rook-ceph` namespace.
|
||||
* `rook-ceph-osd` (namespace: `rook-ceph`): Will affect the OSD pods in the `rook-ceph` namespace.
|
||||
|
||||
You can do it either via e.g. `kubectl -n <namespace> edit serviceaccount default` or by modifying the [`operator.yaml`](https://github.com/rook/rook/blob/master/cluster/examples/kubernetes/ceph/operator.yaml)
|
||||
and [`cluster.yaml`](https://github.com/rook/rook/blob/master/cluster/examples/kubernetes/ceph/cluster.yaml) before deploying them.
|
||||
You can do it either via e.g. `kubectl -n <namespace> edit serviceaccount default` or by modifying the [`operator.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/operator.yaml)
|
||||
and [`cluster.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/cluster.yaml) before deploying them.
|
||||
|
||||
Since it's the same procedure for all service accounts, here is just one example:
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ This will help you manage namespaces more easily, but you should still make sure
|
||||
configured to your liking.
|
||||
|
||||
```sh
|
||||
cd cluster/examples/kubernetes/ceph
|
||||
cd deploy/examples
|
||||
|
||||
export ROOK_OPERATOR_NAMESPACE="rook-ceph"
|
||||
export ROOK_CLUSTER_NAMESPACE="rook-ceph"
|
||||
@@ -68,7 +68,7 @@ kubectl apply -f common.yaml -f operator.yaml -f cluster.yaml # add other files
|
||||
If you wish to create a new CephCluster in a different namespace than `rook-ceph` while using a single operator to manage both clusters execute the following:
|
||||
|
||||
```sh
|
||||
cd cluster/examples/kubernetes/ceph
|
||||
cd deploy/examples
|
||||
|
||||
NAMESPACE=rook-ceph-secondary envsubst < common-second-cluster.yaml | kubectl create -f -
|
||||
```
|
||||
|
||||
@@ -95,7 +95,7 @@ provisioner value should be "my-namespace.rbd.csi.ceph.com".
|
||||
Create the storage class.
|
||||
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
|
||||
kubectl create -f deploy/examples/csi/rbd/storageclass.yaml
|
||||
```
|
||||
|
||||
> **NOTE**: As [specified by Kubernetes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#retain), when using the `Retain` reclaim policy, any Ceph RBD image that is backed by a `PersistentVolume` will continue to exist even after the `PersistentVolume` has been deleted. These Ceph RBD images will need to be cleaned up manually using `rbd rm`.
|
||||
@@ -105,7 +105,7 @@ kubectl create -f cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
|
||||
We create a sample app to consume the block storage provisioned by Rook with the classic wordpress and mysql apps.
|
||||
Both of these apps will make use of block volumes provisioned by Rook.
|
||||
|
||||
Start mysql and wordpress from the `cluster/examples/kubernetes` folder:
|
||||
Start mysql and wordpress from the `deploy/examples` folder:
|
||||
|
||||
```console
|
||||
kubectl create -f mysql.yaml
|
||||
@@ -175,5 +175,10 @@ The OSDs must be located on different nodes, because the [`failureDomain`](ceph-
|
||||
### Erasure Coded CSI Driver
|
||||
|
||||
The erasure coded pool must be set as the `dataPool` parameter in
|
||||
[`storageclass-ec.yaml`](https://github.com/rook/rook/blob/{{ branchName
|
||||
}}/cluster/examples/kubernetes/ceph/csi/rbd/storage-class-ec.yaml) It is used for the data of the RBD images.
|
||||
[`storageclass-ec.yaml`](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/csi/rbd/storage-class-ec.yaml) It is used for the data of the RBD images.
|
||||
|
||||
### Erasure Coded Flex Driver
|
||||
|
||||
The erasure coded pool must be set as the `dataBlockPool` parameter in
|
||||
[`storageclass-ec.yaml`](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/flex/storage-class-ec.yaml). It is used for
|
||||
the data of the RBD images.
|
||||
|
||||
@@ -541,7 +541,7 @@ A Placement configuration is specified (according to the kubernetes PodSpec) as:
|
||||
* `tolerations`: list of kubernetes [Toleration](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/)
|
||||
* `topologySpreadConstraints`: kubernetes [TopologySpreadConstraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/)
|
||||
|
||||
If you use `labelSelector` for `osd` pods, you must write two rules both for `rook-ceph-osd` and `rook-ceph-osd-prepare` like [the example configuration](https://github.com/rook/rook/blob/master/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml#L68). It comes from the design that there are these two pods for an OSD. For more detail, see the [osd design doc](https://github.com/rook/rook/blob/master/design/ceph/dedicated-osd-pod.md) and [the related issue](https://github.com/rook/rook/issues/4582).
|
||||
If you use `labelSelector` for `osd` pods, you must write two rules both for `rook-ceph-osd` and `rook-ceph-osd-prepare` like [the example configuration](https://github.com/rook/rook/blob/master/deploy/examples/cluster-on-pvc.yaml#L68). It comes from the design that there are these two pods for an OSD. For more detail, see the [osd design doc](https://github.com/rook/rook/blob/master/design/ceph/dedicated-osd-pod.md) and [the related issue](https://github.com/rook/rook/issues/4582).
|
||||
|
||||
The Rook Ceph operator creates a Job called `rook-ceph-detect-version` to detect the full Ceph version used by the given `cephVersion.image`. The placement from the `mon` section is used for the Job except for the `PodAntiAffinity` field.
|
||||
|
||||
@@ -1279,7 +1279,7 @@ The features available from the external cluster will vary depending on the vers
|
||||
#### Pre-requisites
|
||||
|
||||
In order to configure an external Ceph cluster with Rook, we need to inject some information in order to connect to that cluster.
|
||||
You can use the `cluster/examples/kubernetes/ceph/import-external-cluster.sh` script to achieve that.
|
||||
You can use the `deploy/examples/import-external-cluster.sh` script to achieve that.
|
||||
The script will look for the following populated environment variables:
|
||||
|
||||
* `NAMESPACE`: the namespace where the configmap and secrets should be injected
|
||||
@@ -1308,14 +1308,14 @@ If the Ceph admin key is not provided, the following script needs to be executed
|
||||
On that machine, run:
|
||||
|
||||
```sh
|
||||
. cluster/examples/kubernetes/ceph/create-external-cluster-resources.sh
|
||||
. deploy/examples/create-external-cluster-resources.sh
|
||||
```
|
||||
|
||||
The script will source all the necessary environment variables for you. It assumes the namespace name is `rook-ceph-external`.
|
||||
This can be changed by running the script like (assuming namespace name is `foo` this time):
|
||||
|
||||
```sh
|
||||
ns=foo . cluster/examples/kubernetes/ceph/create-external-cluster-resources.sh
|
||||
ns=foo . deploy/examples/create-external-cluster-resources.sh
|
||||
```
|
||||
|
||||
When done you can execute: `import-external-cluster.sh` to inject them in your Kubernetes cluster.
|
||||
@@ -1342,12 +1342,12 @@ In this example, you can simply export RGW_POOL_PREFIX before executing the scri
|
||||
export RGW_POOL_PREFIX=my-store
|
||||
```
|
||||
|
||||
The script will automatically create users and keys with the lowest possible privileges and populate the necessary environment variables for `cluster/examples/kubernetes/ceph/import-external-cluster.sh` to work correctly.
|
||||
The script will automatically create users and keys with the lowest possible privileges and populate the necessary environment variables for `deploy/examples/import-external-cluster.sh` to work correctly.
|
||||
|
||||
Finally, you can simply execute the script like this from a machine that has access to your Kubernetes cluster:
|
||||
|
||||
```console
|
||||
bash cluster/examples/kubernetes/ceph/import-external-cluster.sh
|
||||
bash deploy/examples/import-external-cluster.sh
|
||||
```
|
||||
|
||||
#### CephCluster example (consumer)
|
||||
@@ -1380,7 +1380,7 @@ Additionally, you now need to inject `common-external.yaml` too.
|
||||
You can now create it like this:
|
||||
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/cluster-external.yaml
|
||||
kubectl create -f deploy/examples/cluster-external.yaml
|
||||
```
|
||||
|
||||
If the previous section has not been completed, the Rook Operator will still acknowledge the CR creation but will wait forever to receive connection information.
|
||||
|
||||
@@ -122,7 +122,7 @@ provisioned and used by the pod with its lifecycle. The volumes are provisioned
|
||||
when pod get spawned and destroyed at time of pod delete.
|
||||
|
||||
Refer to [ephemeral-doc](https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes) for more info.
|
||||
Also, See the example manifests for an [RBD ephemeral volume](https://github.com/rook/rook/tree/{{ branchName }}/cluster/examples/kubernetes/ceph/csi/rbd/pod-ephemeral.yaml) and a [CephFS ephemeral volume](https://github.com/rook/rook/tree/{{ branchName }}/cluster/examples/kubernetes/ceph/csi/cephfs/pod-ephemeral.yaml).
|
||||
Also, See the example manifests for an [RBD ephemeral volume](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/rbd/pod-ephemeral.yaml) and a [CephFS ephemeral volume](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/cephfs/pod-ephemeral.yaml).
|
||||
|
||||
### Prerequisites
|
||||
Kubernetes version 1.21 or greater is required.
|
||||
|
||||
@@ -26,12 +26,11 @@ In short, as the documentation describes it:
|
||||
|
||||
If your Kubernetes version is updated to a newer version of the snapshot API, follow the upgrade guide [here](https://github.com/kubernetes-csi/external-snapshotter/tree/v4.0.0#upgrade) to upgrade from v1alpha1 to v1beta1, or v1beta1 to v1.
|
||||
|
||||
|
||||
## RBD Snapshots
|
||||
|
||||
### VolumeSnapshotClass
|
||||
|
||||
In [VolumeSnapshotClass](https://github.com/rook/rook/tree/{{ branchName }}/cluster/examples/kubernetes/ceph/csi/rbd/snapshotclass.yaml),
|
||||
In [VolumeSnapshotClass](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/rbd/snapshotclass.yaml),
|
||||
the `csi.storage.k8s.io/snapshotter-secret-name` parameter should reference the
|
||||
name of the secret created for the rbdplugin and `pool` to reflect the Ceph pool name.
|
||||
|
||||
@@ -41,18 +40,18 @@ maintain a configmap whose contents will match this key. By default this is
|
||||
"rook-ceph".
|
||||
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/csi/rbd/snapshotclass.yaml
|
||||
kubectl create -f deploy/examples/csi/rbd/snapshotclass.yaml
|
||||
```
|
||||
|
||||
### Volumesnapshot
|
||||
|
||||
In [snapshot](https://github.com/rook/rook/tree/{{ branchName }}/cluster/examples/kubernetes/ceph/csi/rbd/snapshot.yaml),
|
||||
In [snapshot](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/rbd/snapshot.yaml),
|
||||
`volumeSnapshotClassName` should be the name of the `VolumeSnapshotClass`
|
||||
previously created. The `persistentVolumeClaimName` should be the name of the
|
||||
PVC which is already created by the RBD CSI driver.
|
||||
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/csi/rbd/snapshot.yaml
|
||||
kubectl create -f deploy/examples/csi/rbd/snapshot.yaml
|
||||
```
|
||||
|
||||
### Verify RBD Snapshot Creation
|
||||
@@ -81,14 +80,14 @@ The snapshot will be ready to restore to a new PVC when the `READYTOUSE` field o
|
||||
### Restore the snapshot to a new PVC
|
||||
|
||||
In
|
||||
[pvc-restore](https://github.com/rook/rook/tree/{{ branchName }}/cluster/examples/kubernetes/ceph/csi/rbd/pvc-restore.yaml),
|
||||
[pvc-restore](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/rbd/pvc-restore.yaml),
|
||||
`dataSource` should be the name of the `VolumeSnapshot` previously
|
||||
created. The `dataSource` kind should be the `VolumeSnapshot`.
|
||||
|
||||
Create a new PVC from the snapshot
|
||||
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/csi/rbd/pvc-restore.yaml
|
||||
kubectl create -f deploy/examples/csi/rbd/pvc-restore.yaml
|
||||
```
|
||||
|
||||
### Verify RBD Clone PVC Creation
|
||||
@@ -107,16 +106,16 @@ kubectl get pvc
|
||||
To clean your cluster of the resources created by this example, run the following:
|
||||
|
||||
```console
|
||||
kubectl delete -f cluster/examples/kubernetes/ceph/csi/rbd/pvc-restore.yaml
|
||||
kubectl delete -f cluster/examples/kubernetes/ceph/csi/rbd/snapshot.yaml
|
||||
kubectl delete -f cluster/examples/kubernetes/ceph/csi/rbd/snapshotclass.yaml
|
||||
kubectl delete -f deploy/examples/csi/rbd/pvc-restore.yaml
|
||||
kubectl delete -f deploy/examples/csi/rbd/snapshot.yaml
|
||||
kubectl delete -f deploy/examples/csi/rbd/snapshotclass.yaml
|
||||
```
|
||||
|
||||
## CephFS Snapshots
|
||||
|
||||
### VolumeSnapshotClass
|
||||
|
||||
In [VolumeSnapshotClass](https://github.com/rook/rook/tree/{{ branchName }}/cluster/examples/kubernetes/ceph/csi/cephfs/snapshotclass.yaml),
|
||||
In [VolumeSnapshotClass](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/cephfs/snapshotclass.yaml),
|
||||
the `csi.storage.k8s.io/snapshotter-secret-name` parameter should reference the
|
||||
name of the secret created for the cephfsplugin.
|
||||
|
||||
@@ -127,18 +126,18 @@ maintain a configmap whose contents will match this key. By default this is
|
||||
|
||||
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/csi/cephfs/snapshotclass.yaml
|
||||
kubectl create -f deploy/examples/csi/cephfs/snapshotclass.yaml
|
||||
```
|
||||
|
||||
### Volumesnapshot
|
||||
|
||||
In [snapshot](https://github.com/rook/rook/tree/{{ branchName }}/cluster/examples/kubernetes/ceph/csi/cephfs/snapshot.yaml),
|
||||
In [snapshot](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/cephfs/snapshot.yaml),
|
||||
`volumeSnapshotClassName` should be the name of the `VolumeSnapshotClass`
|
||||
previously created. The `persistentVolumeClaimName` should be the name of the
|
||||
PVC which is already created by the CephFS CSI driver.
|
||||
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/csi/cephfs/snapshot.yaml
|
||||
kubectl create -f deploy/examples/csi/cephfs/snapshot.yaml
|
||||
```
|
||||
|
||||
### Verify CephFS Snapshot Creation
|
||||
@@ -165,14 +164,14 @@ The snapshot will be ready to restore to a new PVC when `READYTOUSE` field of th
|
||||
### Restore the snapshot to a new PVC
|
||||
|
||||
In
|
||||
[pvc-restore](https://github.com/rook/rook/tree/{{ branchName }}/cluster/examples/kubernetes/ceph/csi/cephfs/pvc-restore.yaml),
|
||||
[pvc-restore](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/cephfs/pvc-restore.yaml),
|
||||
`dataSource` should be the name of the `VolumeSnapshot` previously
|
||||
created. The `dataSource` kind should be the `VolumeSnapshot`.
|
||||
|
||||
Create a new PVC from the snapshot
|
||||
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/csi/cephfs/pvc-restore.yaml
|
||||
kubectl create -f deploy/examples/csi/cephfs/pvc-restore.yaml
|
||||
```
|
||||
|
||||
### Verify CephFS Restore PVC Creation
|
||||
@@ -192,9 +191,9 @@ kubectl get pvc
|
||||
To clean your cluster of the resources created by this example, run the following:
|
||||
|
||||
```console
|
||||
kubectl delete -f cluster/examples/kubernetes/ceph/csi/cephfs/pvc-restore.yaml
|
||||
kubectl delete -f cluster/examples/kubernetes/ceph/csi/cephfs/snapshot.yaml
|
||||
kubectl delete -f cluster/examples/kubernetes/ceph/csi/cephfs/snapshotclass.yaml
|
||||
kubectl delete -f deploy/examples/csi/cephfs/pvc-restore.yaml
|
||||
kubectl delete -f deploy/examples/csi/cephfs/snapshot.yaml
|
||||
kubectl delete -f deploy/examples/csi/cephfs/snapshotclass.yaml
|
||||
```
|
||||
|
||||
## Limitations
|
||||
|
||||
@@ -25,7 +25,7 @@ for more info.
|
||||
### Volume Cloning
|
||||
|
||||
In
|
||||
[pvc-clone](https://github.com/rook/rook/tree/{{ branchName }}/cluster/examples/kubernetes/ceph/csi/rbd/pvc-clone.yaml),
|
||||
[pvc-clone](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/rbd/pvc-clone.yaml),
|
||||
`dataSource` should be the name of the `PVC` which is already created by RBD
|
||||
CSI driver. The `dataSource` kind should be the `PersistentVolumeClaim` and also storageclass
|
||||
should be same as the source `PVC`.
|
||||
@@ -33,7 +33,7 @@ should be same as the source `PVC`.
|
||||
Create a new PVC Clone from the PVC
|
||||
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/csi/rbd/pvc-clone.yaml
|
||||
kubectl create -f deploy/examples/csi/rbd/pvc-clone.yaml
|
||||
```
|
||||
|
||||
### Verify RBD volume Clone PVC Creation
|
||||
@@ -53,7 +53,7 @@ kubectl get pvc
|
||||
To clean your cluster of the resources created by this example, run the following:
|
||||
|
||||
```console
|
||||
kubectl delete -f cluster/examples/kubernetes/ceph/csi/rbd/pvc-clone.yaml
|
||||
kubectl delete -f deploy/examples/csi/rbd/pvc-clone.yaml
|
||||
```
|
||||
|
||||
## CephFS Volume Cloning
|
||||
@@ -66,7 +66,7 @@ kubectl delete -f cluster/examples/kubernetes/ceph/csi/rbd/pvc-clone.yaml
|
||||
### Volume Cloning
|
||||
|
||||
In
|
||||
[pvc-clone](https://github.com/rook/rook/tree/{{ branchName }}/cluster/examples/kubernetes/ceph/csi/cephfs/pvc-clone.yaml),
|
||||
[pvc-clone](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/cephfs/pvc-clone.yaml),
|
||||
`dataSource` should be the name of the `PVC` which is already created by CephFS
|
||||
CSI driver. The `dataSource` kind should be the `PersistentVolumeClaim` and also storageclass
|
||||
should be same as the source `PVC`.
|
||||
@@ -74,7 +74,7 @@ should be same as the source `PVC`.
|
||||
Create a new PVC Clone from the PVC
|
||||
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/csi/cephfs/pvc-clone.yaml
|
||||
kubectl create -f deploy/examples/csi/cephfs/pvc-clone.yaml
|
||||
```
|
||||
|
||||
### Verify CephFS volume Clone PVC Creation
|
||||
@@ -94,5 +94,5 @@ kubectl get pvc
|
||||
To clean your cluster of the resources created by this example, run the following:
|
||||
|
||||
```console
|
||||
kubectl delete -f cluster/examples/kubernetes/ceph/csi/cephfs/pvc-clone.yaml
|
||||
kubectl delete -f deploy/examples/csi/cephfs/pvc-clone.yaml
|
||||
```
|
||||
|
||||
@@ -9,13 +9,13 @@ indent: true
|
||||
|
||||
Configuration for Rook and Ceph can be configured in multiple ways to provide block devices, shared filesystem volumes or object storage in a kubernetes namespace. We have provided several examples to simplify storage setup, but remember there are many tunables and you will need to decide what settings work for your use case and environment.
|
||||
|
||||
See the **[example yaml files](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph)** folder for all the rook/ceph setup example spec files.
|
||||
See the **[example yaml files](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples)** folder for all the rook/ceph setup example spec files.
|
||||
|
||||
## Common Resources
|
||||
|
||||
The first step to deploy Rook is to create the CRDs and other common resources. The configuration for these resources will be the same for most deployments.
|
||||
The [crds.yaml](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/crds.yaml) and
|
||||
[common.yaml](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/common.yaml) sets these resources up.
|
||||
The [crds.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/crds.yaml) and
|
||||
[common.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/common.yaml) sets these resources up.
|
||||
|
||||
```console
|
||||
kubectl create -f crds.yaml -f common.yaml
|
||||
@@ -25,14 +25,14 @@ The examples all assume the operator and all Ceph daemons will be started in the
|
||||
|
||||
## Operator
|
||||
|
||||
After the common resources are created, the next step is to create the Operator deployment. Several spec file examples are provided in [this directory](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/):
|
||||
After the common resources are created, the next step is to create the Operator deployment. Several spec file examples are provided in [this directory](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/):
|
||||
|
||||
* `operator.yaml`: The most common settings for production deployments
|
||||
* `kubectl create -f operator.yaml`
|
||||
* `operator-openshift.yaml`: Includes all of the operator settings for running a basic Rook cluster in an OpenShift environment. You will also want to review the [OpenShift Prerequisites](ceph-openshift.md) to confirm the settings.
|
||||
* `oc create -f operator-openshift.yaml`
|
||||
|
||||
Settings for the operator are configured through environment variables on the operator deployment. The individual settings are documented in [operator.yaml](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/operator.yaml).
|
||||
Settings for the operator are configured through environment variables on the operator deployment. The individual settings are documented in [operator.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/operator.yaml).
|
||||
|
||||
## Cluster CRD
|
||||
|
||||
@@ -97,13 +97,13 @@ See the [Object Store CRD](ceph-object-store-crd.md) topic for more details on t
|
||||
|
||||
### Object Storage User
|
||||
|
||||
* `object-user.yaml`: Creates a simple object storage user and generates credentials for the S3 API
|
||||
- `object-user.yaml`: Creates a simple object storage user and generates credentials for the S3 API
|
||||
|
||||
### Object Storage Buckets
|
||||
|
||||
The Ceph operator also runs an object store bucket provisioner which can grant access to existing buckets or dynamically provision new buckets.
|
||||
|
||||
* [object-bucket-claim-retain.yaml](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/object-bucket-claim-retain.yaml) Creates a request for a new bucket by referencing a StorageClass which saves the bucket when the initiating OBC is deleted.
|
||||
* [object-bucket-claim-delete.yaml](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/object-bucket-claim-delete.yaml) Creates a request for a new bucket by referencing a StorageClass which deletes the bucket when the initiating OBC is deleted.
|
||||
* [storageclass-bucket-retain.yaml](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/storageclass-bucket-retain.yaml) Creates a new StorageClass which defines the Ceph Object Store, a region, and retains the bucket after the initiating OBC is deleted.
|
||||
* [storageclass-bucket-delete.yaml](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/storageclass-bucket-delete.yaml) Creates a new StorageClass which defines the Ceph Object Store, a region, and deletes the bucket after the initiating OBC is deleted.
|
||||
* [object-bucket-claim-retain.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/object-bucket-claim-retain.yaml) Creates a request for a new bucket by referencing a StorageClass which saves the bucket when the initiating OBC is deleted.
|
||||
* [object-bucket-claim-delete.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/object-bucket-claim-delete.yaml) Creates a request for a new bucket by referencing a StorageClass which deletes the bucket when the initiating OBC is deleted.
|
||||
* [storageclass-bucket-retain.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/storageclass-bucket-retain.yaml) Creates a new StorageClass which defines the Ceph Object Store, a region, and retains the bucket after the initiating OBC is deleted.
|
||||
* [storageclass-bucket-delete.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/storageclass-bucket-delete.yaml) Creates a new StorageClass which defines the Ceph Object Store, a region, and deletes the bucket after the initiating OBC is deleted.
|
||||
|
||||
@@ -65,7 +65,7 @@ spec:
|
||||
# memory: "1024Mi"
|
||||
```
|
||||
|
||||
(These definitions can also be found in the [`filesystem.yaml`](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/filesystem.yaml) file)
|
||||
(These definitions can also be found in the [`filesystem.yaml`](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/filesystem.yaml) file)
|
||||
|
||||
### Erasure Coded
|
||||
|
||||
@@ -94,8 +94,8 @@ spec:
|
||||
activeStandby: true
|
||||
```
|
||||
|
||||
(These definitions can also be found in the [`filesystem-ec.yaml`](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/filesystem-ec.yaml) file.
|
||||
Also see an example in the [`storageclass-ec.yaml`](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/csi/cephfs/storageclass-ec.yaml) for how to configure the volume.)
|
||||
(These definitions can also be found in the [`filesystem-ec.yaml`](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/filesystem-ec.yaml) file.
|
||||
Also see an example in the [`storageclass-ec.yaml`](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/csi/cephfs/storageclass-ec.yaml) for how to configure the volume.)
|
||||
|
||||
### Mirroring
|
||||
|
||||
@@ -211,7 +211,7 @@ The metadata server settings correspond to the MDS daemon settings.
|
||||
* `duration`:
|
||||
* `annotations`: Key value pair list of annotations to add.
|
||||
* `labels`: Key value pair list of labels to add.
|
||||
* `placement`: The mds pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/cluster.yaml).
|
||||
* `placement`: The mds pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/cluster.yaml).
|
||||
* `resources`: Set resource requests/limits for the Filesystem MDS Pod(s), see [MDS Resources Configuration Settings](#mds-resources-configuration-settings)
|
||||
* `priorityClassName`: Set priority class name for the Filesystem MDS Pod(s)
|
||||
|
||||
|
||||
@@ -120,7 +120,7 @@ provisioner value should be "rook-op.rbd.csi.ceph.com".
|
||||
Create the storage class.
|
||||
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/csi/cephfs/storageclass.yaml
|
||||
kubectl create -f deploy/examples/csi/cephfs/storageclass.yaml
|
||||
```
|
||||
|
||||
## Quotas
|
||||
@@ -213,7 +213,7 @@ spec:
|
||||
Create the Kube registry deployment:
|
||||
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/csi/cephfs/kube-registry.yaml
|
||||
kubectl create -f deploy/examples/csi/cephfs/kube-registry.yaml
|
||||
```
|
||||
|
||||
You now have a docker registry which is HA with persistent storage.
|
||||
|
||||
@@ -38,7 +38,7 @@ If any setting is unspecified, a suitable default will be used automatically.
|
||||
|
||||
### FilesystemMirror Settings
|
||||
|
||||
- `placement`: The cephfs-mirror pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/cluster.yaml).
|
||||
- `placement`: The cephfs-mirror pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/cluster.yaml).
|
||||
- `annotations`: Key value pair list of annotations to add.
|
||||
- `labels`: Key value pair list of labels to add.
|
||||
- `resources`: The resource requirements for the cephfs-mirror pods.
|
||||
|
||||
@@ -39,7 +39,7 @@ From the root of your locally cloned Rook repo, go the monitoring directory:
|
||||
|
||||
```console
|
||||
$ git clone --single-branch --branch {{ branchName }} https://github.com/rook/rook.git
|
||||
cd rook/cluster/examples/kubernetes/ceph/monitoring
|
||||
cd rook/deploy/examples/monitoring
|
||||
```
|
||||
|
||||
Create the service monitor as well as the Prometheus server pod and service:
|
||||
@@ -100,7 +100,7 @@ To enable the Ceph Prometheus alerts follow these steps:
|
||||
1. Create the RBAC rules to enable monitoring.
|
||||
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/monitoring/rbac.yaml
|
||||
kubectl create -f deploy/examples/monitoring/rbac.yaml
|
||||
```
|
||||
|
||||
2. Make following changes to your CephCluster object (e.g., `cluster.yaml`).
|
||||
@@ -151,7 +151,7 @@ with each update or upgrade. This should be done at the same time you update Roo
|
||||
like `common.yaml`.
|
||||
|
||||
```console
|
||||
kubectl apply -f cluster/examples/kubernetes/ceph/monitoring/rbac.yaml
|
||||
kubectl apply -f deploy/examples/monitoring/rbac.yaml
|
||||
```
|
||||
|
||||
> This is updated automatically if you are upgrading via the helm chart
|
||||
|
||||
@@ -33,7 +33,8 @@ If an admin wants to set up multisite on a Rook Ceph cluster, the admin should c
|
||||
1. A [zone](/Documentation/ceph-object-multisite-crd.md#object-zone-settings)
|
||||
1. An [object-store](/Documentation/ceph-object-store-crd.md#zone-settings) with the `zone` section
|
||||
|
||||
object-multisite.yaml in the [examples](/cluster/examples/kubernetes/ceph/) directory can be used to create the multisite CRDs.
|
||||
object-multisite.yaml in the [examples](/deploy/examples/) directory can be used to create the multisite CRDs.
|
||||
|
||||
```console
|
||||
kubectl create -f object-multisite.yaml
|
||||
```
|
||||
@@ -169,7 +170,8 @@ Once the admin knows the endpoint and the secret for the keys has been created,
|
||||
1. A [CephObjectZone](/design/ceph/object/zone.md) referring to the CephObjectZoneGroup created above.
|
||||
1. A [CephObjectStore](/design/ceph/object/store.md) referring to the new CephObjectZone resource.
|
||||
|
||||
object-multisite-pull-realm.yaml (with changes) in the [examples](/cluster/examples/kubernetes/ceph/) directory can be used to create the multisite CRDs.
|
||||
object-multisite-pull-realm.yaml (with changes) in the [examples](/deploy/examples/) directory can be used to create the multisite CRDs.
|
||||
|
||||
```console
|
||||
kubectl create -f object-multisite-pull-realm.yaml
|
||||
```
|
||||
|
||||
@@ -12,7 +12,7 @@ OpenShift Console uses OpenShift Prometheus for monitoring and populating data i
|
||||
|
||||
1. Change the monitoring namespace to `openshift-monitoring`
|
||||
|
||||
Change the namespace of the RoleBinding `rook-ceph-metrics` from `rook-ceph` to `openshift-monitoring` for the `prometheus-k8s` ServiceAccount in [rbac.yaml](https://github.com/rook/rook/blob/master/cluster/examples/kubernetes/ceph/monitoring/rbac.yaml#L70).
|
||||
Change the namespace of the RoleBinding `rook-ceph-metrics` from `rook-ceph` to `openshift-monitoring` for the `prometheus-k8s` ServiceAccount in [rbac.yaml](https://github.com/rook/rook/blob/master/deploy/examples/monitoring/rbac.yaml#L70).
|
||||
|
||||
```
|
||||
subjects:
|
||||
|
||||
@@ -9,7 +9,7 @@ indent: true
|
||||
|
||||
[OpenShift](https://www.openshift.com/) adds a number of security and other enhancements to Kubernetes. In particular, [security context constraints](https://blog.openshift.com/understanding-service-accounts-sccs/) allow the cluster admin to define exactly which permissions are allowed to pods running in the cluster. You will need to define those permissions that allow the Rook pods to run.
|
||||
|
||||
The settings for Rook in OpenShift are described below, and are also included in the [example yaml files](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph):
|
||||
The settings for Rook in OpenShift are described below, and are also included in the [example yaml files](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples):
|
||||
|
||||
* `operator-openshift.yaml`: Creates the security context constraints and starts the operator deployment
|
||||
* `object-openshift.yaml`: Creates an object store with rgw listening on a valid port number for OpenShift
|
||||
@@ -35,7 +35,7 @@ To orchestrate the storage platform, Rook requires the following access in the c
|
||||
|
||||
## Security Context Constraints
|
||||
|
||||
Before starting the Rook operator or cluster, create the security context constraints needed by the Rook pods. The following yaml is found in `operator-openshift.yaml` under `/cluster/examples/kubernetes/ceph`.
|
||||
Before starting the Rook operator or cluster, create the security context constraints needed by the Rook pods. The following yaml is found in `operator-openshift.yaml` under `/deploy/examples`.
|
||||
|
||||
> **NOTE**: Older versions of OpenShift may require `apiVersion: v1`.
|
||||
|
||||
|
||||
@@ -53,6 +53,7 @@ to the updated cluster CR.
|
||||
|
||||
To remove an OSD due to a failed disk or other re-configuration, consider the following to ensure the health of the data
|
||||
through the removal process:
|
||||
|
||||
- Confirm you will have enough space on your cluster after removing your OSDs to properly handle the deletion
|
||||
- Confirm the remaining OSDs and their placement groups (PGs) are healthy in order to handle the rebalancing of the data
|
||||
- Do not remove too many OSDs at once
|
||||
@@ -106,7 +107,7 @@ in the toolbox may show which OSD is `down`. If you want to remove a healthy OSD
|
||||
|
||||
### Purge the OSD from the Ceph cluster
|
||||
|
||||
OSD removal can be automated with the example found in the [rook-ceph-purge-osd job](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/osd-purge.yaml).
|
||||
OSD removal can be automated with the example found in the [rook-ceph-purge-osd job](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/osd-purge.yaml).
|
||||
In the osd-purge.yaml, change the `<OSD-IDs>` to the ID(s) of the OSDs you want to remove.
|
||||
|
||||
1. Run the job: `kubectl create -f osd-purge.yaml`
|
||||
|
||||
@@ -41,7 +41,7 @@ If any setting is unspecified, a suitable default will be used automatically.
|
||||
### RBDMirror Settings
|
||||
|
||||
* `count`: The number of rbd mirror instance to run.
|
||||
* `placement`: The rbd mirror pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/cluster.yaml).
|
||||
* `placement`: The rbd mirror pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/cluster.yaml)..
|
||||
* `annotations`: Key value pair list of annotations to add.
|
||||
* `labels`: Key value pair list of labels to add.
|
||||
* `resources`: The resource requirements for the rbd mirror pods.
|
||||
|
||||
@@ -23,7 +23,7 @@ run arbitrary Ceph commands.
|
||||
Launch the rook-ceph-tools pod:
|
||||
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/toolbox.yaml
|
||||
kubectl create -f deploy/examples/toolbox.yaml
|
||||
```
|
||||
|
||||
Wait for the toolbox pod to download its container and get to the `running` state:
|
||||
@@ -63,7 +63,7 @@ In this example, the `ceph status` command is executed when the job is created.
|
||||
Create the toolbox job:
|
||||
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/toolbox-job.yaml
|
||||
kubectl create -f deploy/examples/toolbox-job.yaml
|
||||
```
|
||||
|
||||
After the job completes, see the results of the script:
|
||||
|
||||
@@ -59,7 +59,7 @@ the following:
|
||||
First get the latest common resources manifests that contain the latest changes for Rook v1.7.
|
||||
```sh
|
||||
git clone --single-branch --depth=1 --branch v1.7.1 https://github.com/rook/rook.git
|
||||
cd rook/cluster/examples/kubernetes/ceph
|
||||
cd rook/deploy/examples
|
||||
```
|
||||
|
||||
If you have deployed the Rook Operator or the Ceph cluster into a different namespace than
|
||||
@@ -103,7 +103,7 @@ time without compatibility support and without prior notice.
|
||||
We will do all our work in the Ceph example manifests directory.
|
||||
|
||||
```sh
|
||||
$ cd $YOUR_ROOK_REPO/cluster/examples/kubernetes/ceph/
|
||||
$ cd $YOUR_ROOK_REPO/deploy/examples/
|
||||
```
|
||||
|
||||
Unless your Rook cluster was created with customized namespaces, namespaces for Rook clusters are
|
||||
@@ -275,7 +275,7 @@ needed by the Operator. Also update the Custom Resource Definitions (CRDs).
|
||||
Get the latest common resources manifests that contain the latest changes.
|
||||
```sh
|
||||
git clone --single-branch --depth=1 --branch v1.7.0 https://github.com/rook/rook.git
|
||||
cd rook/cluster/examples/kubernetes/ceph
|
||||
cd rook/deploy/examples
|
||||
```
|
||||
|
||||
If you have deployed the Rook Operator or the Ceph cluster into a different namespace than
|
||||
@@ -299,7 +299,7 @@ If you have [Prometheus monitoring](ceph-monitoring.md) enabled, follow the
|
||||
step to upgrade the Prometheus RBAC resources as well.
|
||||
|
||||
```sh
|
||||
kubectl apply -f cluster/examples/kubernetes/ceph/monitoring/rbac.yaml
|
||||
kubectl apply -f deploy/examples/monitoring/rbac.yaml
|
||||
```
|
||||
|
||||
### **2. Update Ceph CSI versions**
|
||||
|
||||
@@ -16,7 +16,7 @@ If your pod dies, your mount will die with it.
|
||||
To test mounting your Ceph volumes, start a pod with the necessary mounts. An example is provided in the examples test directory:
|
||||
|
||||
```console
|
||||
kubectl create -f cluster/examples/kubernetes/ceph/direct-mount.yaml
|
||||
kubectl create -f deploy/examples/direct-mount.yaml
|
||||
```
|
||||
|
||||
After the pod is started, connect to it like this:
|
||||
|
||||
@@ -33,7 +33,7 @@ Rook currently publishes builds of this chart to the `release` and `master` chan
|
||||
**Before installing, review the values.yaml to confirm if the default settings need to be updated.**
|
||||
* If the operator was installed in a namespace other than `rook-ceph`, the namespace
|
||||
must be set in the `operatorNamespace` variable.
|
||||
* Set the desired settings in the `cephClusterSpec`. The [defaults](https://github.com/rook/rook/tree/{{ branchName }}/cluster/charts/rook-ceph-cluster/values.yaml)
|
||||
* Set the desired settings in the `cephClusterSpec`. The [defaults](https://github.com/rook/rook/tree/{{ branchName }}/deploy/charts/rook-ceph-cluster/values.yaml)
|
||||
are only an example and not likely to apply to your cluster.
|
||||
* The `monitoring` section should be removed from the `cephClusterSpec`, as it is specified separately in the helm settings.
|
||||
* The default values for `cephBlockPools`, `cephFileSystems`, and `CephObjectStores` will create one of each, and their corresponding storage classes.
|
||||
@@ -143,7 +143,7 @@ chart to start managing the cluster:
|
||||
To deploy from a local build from your development environment:
|
||||
|
||||
```console
|
||||
cd cluster/charts/rook-ceph-cluster
|
||||
cd deploy/charts/rook-ceph-cluster
|
||||
helm install --create-namespace --namespace rook-ceph rook-ceph-cluster -f values-override.yaml .
|
||||
```
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ To deploy from a local build from your development environment:
|
||||
1. Install the helm chart:
|
||||
|
||||
```console
|
||||
cd cluster/charts/rook-ceph
|
||||
cd deploy/charts/rook-ceph
|
||||
helm install --create-namespace --namespace rook-ceph rook-ceph .
|
||||
```
|
||||
|
||||
@@ -163,4 +163,4 @@ Alternatively, a yaml file that specifies the values for the above parameters (`
|
||||
helm install --namespace rook-ceph rook-ceph rook-release/rook-ceph -f values.yaml
|
||||
```
|
||||
|
||||
For example settings, see [values.yaml](https://github.com/rook/rook/tree/{{ branchName }}/cluster/charts/rook-ceph/values.yaml)
|
||||
For example settings, see [values.yaml](https://github.com/rook/rook/tree/{{ branchName }}/deploy/charts/rook-ceph/values.yaml)
|
||||
|
||||
@@ -14,5 +14,5 @@ Rook has published the following Helm charts for the Ceph storage provider:
|
||||
|
||||
The Helm charts are intended to simplify deployment and upgrades.
|
||||
Configuring the Rook resources without Helm is also fully supported by creating the
|
||||
[manifests](https://github.com/rook/rook/tree/{{ branchName }}/cluster/examples/kubernetes)
|
||||
[manifests](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples)
|
||||
directly.
|
||||
|
||||
@@ -12,7 +12,7 @@ please review this document. By default, Kubernetes clusters do not have PSPs en
|
||||
be able to skip this document.
|
||||
|
||||
If you are configuring Ceph on OpenShift, the Ceph walkthrough will configure the PSPs as well
|
||||
when you start the operator with [operator-openshift.yaml](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/operator-openshift.yaml).
|
||||
when you start the operator with [operator-openshift.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/operator-openshift.yaml).
|
||||
|
||||
Creating the Rook operator requires privileges for setting up RBAC. To launch the operator you need to have created your user certificate that is bound to ClusterRole `cluster-admin`.
|
||||
|
||||
@@ -23,7 +23,7 @@ using [PodSecurityPolicies](https://kubernetes.io/docs/concepts/policy/pod-secur
|
||||
for the different `ServiceAccounts` Rook uses to start the Rook Storage Pods.
|
||||
|
||||
Security policies will differ for different backends. See Ceph's Pod Security Policies set up in
|
||||
[common.yaml](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/common.yaml)
|
||||
[common.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/common.yaml)
|
||||
for an example of how this is done in practice.
|
||||
|
||||
### PodSecurityPolicy
|
||||
|
||||
@@ -31,11 +31,11 @@ In order to configure the Ceph storage cluster, at least one of these local stor
|
||||
|
||||
## TL;DR
|
||||
|
||||
A simple Rook cluster can be created with the following kubectl commands and [example manifests](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph).
|
||||
A simple Rook cluster can be created with the following kubectl commands and [example manifests](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples).
|
||||
|
||||
```console
|
||||
$ git clone --single-branch --branch {{ branchName }} https://github.com/rook/rook.git
|
||||
cd rook/cluster/examples/kubernetes/ceph
|
||||
cd rook/deploy/examples
|
||||
kubectl create -f crds.yaml -f common.yaml -f operator.yaml
|
||||
kubectl create -f cluster.yaml
|
||||
```
|
||||
@@ -44,10 +44,10 @@ After the cluster is running, you can create [block, object, or file](#storage)
|
||||
|
||||
## Deploy the Rook Operator
|
||||
|
||||
The first step is to deploy the Rook operator. Check that you are using the [example yaml files](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph) that correspond to your release of Rook. For more options, see the [examples documentation](ceph-examples.md).
|
||||
The first step is to deploy the Rook operator. Check that you are using the [example yaml files](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples) that correspond to your release of Rook. For more options, see the [examples documentation](ceph-examples.md).
|
||||
|
||||
```console
|
||||
cd cluster/examples/kubernetes/ceph
|
||||
cd deploy/examples
|
||||
kubectl create -f crds.yaml -f common.yaml -f operator.yaml
|
||||
|
||||
# verify the rook-ceph-operator is in the `Running` state before proceeding
|
||||
@@ -57,7 +57,7 @@ kubectl -n rook-ceph get pod
|
||||
You can also deploy the operator with the [Rook Helm Chart](helm-operator.md).
|
||||
|
||||
Before you start the operator in production, there are some settings that you may want to consider:
|
||||
1. Consider if you want to enable certain Rook features that are disabled by default. See the [operator.yaml](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/operator.yaml) for these and other advanced settings.
|
||||
1. Consider if you want to enable certain Rook features that are disabled by default. See the [operator.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/operator.yaml) for these and other advanced settings.
|
||||
1. Device discovery: Rook will watch for new devices to configure if the `ROOK_ENABLE_DISCOVERY_DAEMON` setting is enabled, commonly used in bare metal clusters.
|
||||
2. Node affinity and tolerations: The CSI driver by default will run on any node in the cluster. To configure the CSI driver affinity, several settings are available.
|
||||
|
||||
@@ -68,9 +68,10 @@ If you wish to deploy into a namespace other than the default `rook-ceph`, see t
|
||||
|
||||
The Rook documentation is focused around starting Rook in a production environment. Examples are also
|
||||
provided to relax some settings for test environments. When creating the cluster later in this guide, consider these example cluster manifests:
|
||||
- [cluster.yaml](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/cluster.yaml): Cluster settings for a production cluster running on bare metal. Requires at least three worker nodes.
|
||||
- [cluster-on-pvc.yaml](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml): Cluster settings for a production cluster running in a dynamic cloud environment.
|
||||
- [cluster-test.yaml](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/cluster-test.yaml): Cluster settings for a test environment such as minikube.
|
||||
|
||||
- [cluster.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/cluster.yaml): Cluster settings for a production cluster running on bare metal. Requires at least three worker nodes.
|
||||
- [cluster-on-pvc.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/cluster-on-pvc.yaml): Cluster settings for a production cluster running in a dynamic cloud environment.
|
||||
- [cluster-test.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/cluster-test.yaml): Cluster settings for a test environment such as minikube.
|
||||
|
||||
See the [Ceph examples](ceph-examples.md) for more details.
|
||||
|
||||
|
||||
@@ -267,7 +267,7 @@ Below guide assumes that we have a PVC (rbd-pvc) in BOUND state; created using
|
||||
In this case, we create a Volume Replication Class on cluster-1
|
||||
|
||||
```bash
|
||||
[cluster-1]$ kubectl apply -f cluster/examples/kubernetes/ceph/volume-replication-class.yaml
|
||||
[cluster-1]$ kubectl apply -f deploy/examples/volume-replication-class.yaml
|
||||
```
|
||||
|
||||
> **Note:** The `schedulingInterval` can be specified in formats of
|
||||
@@ -281,7 +281,7 @@ In this case, we create a Volume Replication Class on cluster-1
|
||||
the PVC which we intend to replicate to secondary cluster.
|
||||
|
||||
```bash
|
||||
[cluster-1]$ kubectl apply -f cluster/examples/kubernetes/ceph/volume-replication.yaml
|
||||
[cluster-1]$ kubectl apply -f deploy/examples/volume-replication.yaml
|
||||
```
|
||||
|
||||
>:memo: *VolumeReplication* is a namespace scoped object. Thus,
|
||||
@@ -373,7 +373,7 @@ Here, we take a backup of PVC and PV object on one site, so that they can be res
|
||||
* Create VolumeReplicationClass on the secondary cluster
|
||||
|
||||
```bash
|
||||
[cluster-1]$ kubectl apply -f cluster/examples/kubernetes/ceph/volume-replication-class.yaml
|
||||
[cluster-1]$ kubectl apply -f deploy/examples/volume-replication-class.yaml
|
||||
```
|
||||
|
||||
> ```bash
|
||||
|
||||
@@ -12,6 +12,9 @@ v1.8...
|
||||
you will need to convert them to csi volumes. See the flex conversion tool.
|
||||
- Min supported version of K8s is now 1.16. If running on an older version of K8s it is recommended
|
||||
to update to a newer version before updating to Rook v1.8.
|
||||
- Directory structure of the YAML examples has changed. Files are now in `deploy/examples` and subdirectories.
|
||||
|
||||
### Ceph
|
||||
|
||||
## Features
|
||||
|
||||
|
||||
@@ -32,9 +32,9 @@ if [[ -n "$BUILD_CRDS_INTO_DIR" ]]; then
|
||||
echo "Generating CRDs into dir $BUILD_CRDS_INTO_DIR"
|
||||
DESTINATION_ROOT="$BUILD_CRDS_INTO_DIR"
|
||||
fi
|
||||
OLM_CATALOG_DIR="${DESTINATION_ROOT}/cluster/olm/ceph/deploy/crds"
|
||||
CEPH_CRDS_FILE_PATH="${DESTINATION_ROOT}/cluster/examples/kubernetes/ceph/crds.yaml"
|
||||
CEPH_HELM_CRDS_FILE_PATH="${DESTINATION_ROOT}/cluster/charts/rook-ceph/templates/resources.yaml"
|
||||
OLM_CATALOG_DIR="${DESTINATION_ROOT}/deploy/olm/deploy/crds"
|
||||
CEPH_CRDS_FILE_PATH="${DESTINATION_ROOT}/deploy/examples/crds.yaml"
|
||||
CEPH_HELM_CRDS_FILE_PATH="${DESTINATION_ROOT}/deploy/charts/rook-ceph/templates/resources.yaml"
|
||||
|
||||
#############
|
||||
# FUNCTIONS #
|
||||
@@ -42,8 +42,8 @@ CEPH_HELM_CRDS_FILE_PATH="${DESTINATION_ROOT}/cluster/charts/rook-ceph/templates
|
||||
|
||||
copy_ob_obc_crds() {
|
||||
mkdir -p "$OLM_CATALOG_DIR"
|
||||
cp -f "${SCRIPT_ROOT}/cluster/olm/ceph/assemble/objectbucket.io_objectbucketclaims.yaml" "$OLM_CATALOG_DIR"
|
||||
cp -f "${SCRIPT_ROOT}/cluster/olm/ceph/assemble/objectbucket.io_objectbuckets.yaml" "$OLM_CATALOG_DIR"
|
||||
cp -f "${SCRIPT_ROOT}/deploy/olm/assemble/objectbucket.io_objectbucketclaims.yaml" "$OLM_CATALOG_DIR"
|
||||
cp -f "${SCRIPT_ROOT}/deploy/olm/assemble/objectbucket.io_objectbuckets.yaml" "$OLM_CATALOG_DIR"
|
||||
}
|
||||
|
||||
generating_crds_v1() {
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
HELM_CHARTS ?= rook-ceph rook-ceph-cluster
|
||||
HELM_BASE_URL ?= https://charts.rook.io
|
||||
HELM_S3_BUCKET ?= rook.chart
|
||||
HELM_CHARTS_DIR ?= $(ROOT_DIR)/cluster/charts
|
||||
HELM_CHARTS_DIR ?= $(ROOT_DIR)/deploy/charts
|
||||
HELM_OUTPUT_DIR ?= $(OUTPUT_DIR)/charts
|
||||
|
||||
HELM_HOME := $(abspath $(CACHE_DIR)/helm)
|
||||
|
||||
@@ -6,7 +6,7 @@ set -eEuox pipefail
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
|
||||
pushd "$SCRIPT_DIR"
|
||||
|
||||
${HELM} template ../../cluster/charts/rook-ceph \
|
||||
${HELM} template ../../deploy/charts/rook-ceph \
|
||||
--namespace rook-ceph \
|
||||
--set crds.enabled=false | ./keep-rbac-yaml.py > rbac.yaml
|
||||
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
# Build Rook's CSV file
|
||||
|
||||
Just run `make CSV_VERSION=1.0.0 csv-ceph` like this:
|
||||
|
||||
```console
|
||||
make csv-ceph CSV_VERSION=1.0.1 CSV_PLATFORM=k8s ROOK_OP_VERSION=rook/ceph:v1.0.1
|
||||
```
|
||||
|
||||
>```
|
||||
>INFO[0000] Generating CSV manifest version 1.0.1
|
||||
>INFO[0000] Fill in the following required fields in file deploy/olm-catalog/ceph.csv.yaml:
|
||||
> spec.keywords
|
||||
> spec.maintainers
|
||||
> spec.provider
|
||||
> spec.labels
|
||||
>INFO[0000] Create deploy/olm-catalog/ceph.csv.yaml
|
||||
>INFO[0000] Create deploy/olm-catalog/_generated.concat_crd.yaml
|
||||
>
|
||||
>Congratulations!
|
||||
>Your Rook CSV 1.0.1 file for k8s is ready at: cluster/olm/ceph/deploy/olm-catalog/rook-ceph.v1.0.1.clusterserviceversion.yaml
|
||||
>Push it to https://github.com/operator-framework/community-operators as well as the CRDs files from cluster/olm/ceph/deploy/crds and the package file cluster/olm/ceph/assemble/rook-ceph.package.yaml.
|
||||
>```
|
||||
|
||||
Or for OpenShift use: `make CSV_VERSION=1.0.0 CSV_PLATFORM=ocp csv-ceph`.
|
||||
@@ -10,7 +10,7 @@ image:
|
||||
|
||||
crds:
|
||||
# Whether the helm chart should create and update the CRDs. If false, the CRDs must be
|
||||
# managed independently with cluster/examples/kubernetes/ceph/crds.yaml.
|
||||
# managed independently with deploy/examples/crds.yaml.
|
||||
# **WARNING** Only set during first deployment. If later disabled the cluster may be DESTROYED.
|
||||
# If the CRDs are deleted in this case, see the disaster recovery guide to restore them.
|
||||
# https://rook.github.io/docs/rook/latest/ceph-disaster-recovery.html#restoring-crds-after-deletion
|
||||
+3
-3
@@ -2,7 +2,7 @@
|
||||
# this script creates all the users/keys on the external cluster
|
||||
# those keys will be injected via the import-external-cluster.sh once this one is done running
|
||||
# so you can run import-external-cluster.sh right after this script
|
||||
# run me like: . cluster/examples/kubernetes/ceph/create-external-cluster-resources.sh
|
||||
# run me like: . deploy/examples/create-external-cluster-resources.sh
|
||||
set -e
|
||||
|
||||
#############
|
||||
@@ -26,12 +26,12 @@ function checkEnv() {
|
||||
echo "'ceph' binary is expected'"
|
||||
return 1
|
||||
fi
|
||||
|
||||
|
||||
if ! is_available jq; then
|
||||
echo "'jq' binary is expected'"
|
||||
return 1
|
||||
fi
|
||||
|
||||
|
||||
if ! ceph -s 1>/dev/null; then
|
||||
echo "cannot connect to the ceph cluster"
|
||||
return 1
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user