feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

Merge pull request #14772 from jhoblitt/bugfix/cephobjectstore-existing-pools

Browse Source 
rgw: fix CephObjectStore failing with pre-existing pools
This commit is contained in:
Travis Nielsen authored and GitHub committed 2024-10-03 11:50:32 -06:00
2 parents fc2ac66d65 581fd5c197
commit 049953471d
4 files changed
+297 -21
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/canary-integration-test.yml
+67 -5
View File
@@ -49,7 +49,9 @@ jobs:
tests/scripts/github-action-helper.sh create_partitions_for_osds
- name: deploy cluster
run: tests/scripts/github-action-helper.sh deploy_cluster
run: |
tests/scripts/github-action-helper.sh deploy_cluster
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
- name: setup csi-addons
run: tests/scripts/csiaddons.sh setup_csiaddons
@@ -364,6 +366,7 @@ jobs:
run: |
export ALLOW_LOOP_DEVICES=true
tests/scripts/github-action-helper.sh deploy_cluster loop
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
tests/scripts/github-action-helper.sh create_operator_toolbox
- name: wait for prepare pod
@@ -433,7 +436,9 @@ jobs:
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only
- name: deploy cluster
run: tests/scripts/github-action-helper.sh deploy_cluster two_osds_in_device
run: |
tests/scripts/github-action-helper.sh deploy_cluster two_osds_in_device
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
- name: wait for prepare pod
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
@@ -482,6 +487,7 @@ jobs:
- name: deploy cluster
run: |
tests/scripts/github-action-helper.sh deploy_cluster osd_with_metadata_partition_device
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
- name: wait for prepare pod
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
@@ -537,7 +543,9 @@ jobs:
tests/scripts/github-action-helper.sh create_LV_on_disk $(sudo losetup --find --show test-rook.img)
- name: deploy cluster
run: tests/scripts/github-action-helper.sh deploy_cluster osd_with_metadata_device
run: |
tests/scripts/github-action-helper.sh deploy_cluster osd_with_metadata_device
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
- name: wait for prepare pod
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
@@ -587,7 +595,9 @@ jobs:
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only
- name: deploy cluster
run: tests/scripts/github-action-helper.sh deploy_cluster encryption
run: |
tests/scripts/github-action-helper.sh deploy_cluster encryption
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster encryption
- name: wait for prepare pod
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
@@ -642,7 +652,9 @@ jobs:
tests/scripts/github-action-helper.sh create_LV_on_disk $BLOCK
- name: deploy cluster
run: tests/scripts/github-action-helper.sh deploy_cluster lvm
run: |
tests/scripts/github-action-helper.sh deploy_cluster lvm
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
- name: wait for prepare pod
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
@@ -1698,3 +1710,53 @@ jobs:
uses: ./.github/workflows/collect-logs
with:
name: ${{ github.job }}-${{ matrix.ceph-image }}
object-with-cephblockpools:
runs-on: ubuntu-22.04
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
strategy:
matrix:
ceph-image: ${{ fromJson(inputs.ceph_images) }}
steps:
- name: checkout
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
with:
fetch-depth: 0
- name: consider debugging
uses: ./.github/workflows/tmate_debug
with:
use-tmate: ${{ secrets.USE_TMATE }}
- name: setup cluster resources
uses: ./.github/workflows/canary-test-config
- name: set Ceph version in CephCluster manifest
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}"
- name: validate-yaml
run: tests/scripts/github-action-helper.sh validate_yaml
- name: use local disk and create partitions for osds
run: |
tests/scripts/github-action-helper.sh use_local_disk
tests/scripts/github-action-helper.sh create_partitions_for_osds
- name: deploy cluster
run: tests/scripts/github-action-helper.sh deploy_cluster
- name: create CephBlockPool(s) and CephObjectStore
shell: bash --noprofile --norc -eo pipefail -x {0}
run: kubectl create -f deploy/examples/object-with-cephblockpools-test.yaml
- name: wait for CephObjectStore to be ready
run: tests/scripts/validate_cluster.sh rgw object-with-cephblockpools
- name: check for pools created by RGW that are unexpected
run: tests/scripts/github-action-helper.sh test_object_with_cephblockpools_extra_pools
- name: collect common logs
if: always()
uses: ./.github/workflows/collect-logs
with:
name: ${{ github.job }}-${{ matrix.ceph-image }}
deploy/examples/object-with-cephblockpools-test.yaml
+135
View File
@@ -0,0 +1,135 @@
##########################################################
# Create an object store using pre-created pools
# kubectl create -f object-with-cephblockpools-test.yaml
##########################################################
---
apiVersion: ceph.rook.io/v1
kind: CephObjectStore
metadata:
name: object-with-cephblockpools
namespace: rook-ceph # namespace:cluster
spec:
gateway:
port: 80
instances: 1
---
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
name: rgw.root
namespace: rook-ceph # namespace:cluster
spec:
application: rgw
failureDomain: osd
name: .rgw.root
parameters:
pg_autoscale_mode: "off"
pg_num: "1"
replicated:
requireSafeReplicaSize: false
size: 1
---
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
name: object-with-cephblockpools.rgw.control
namespace: rook-ceph # namespace:cluster
spec:
application: rgw
failureDomain: osd
parameters:
pg_autoscale_mode: "off"
pg_num: "1"
replicated:
requireSafeReplicaSize: false
size: 1
---
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
name: object-with-cephblockpools.rgw.meta
namespace: rook-ceph # namespace:cluster
spec:
application: rgw
failureDomain: osd
parameters:
pg_autoscale_mode: "off"
pg_num: "1"
replicated:
requireSafeReplicaSize: false
size: 1
---
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
name: object-with-cephblockpools.rgw.log
namespace: rook-ceph # namespace:cluster
spec:
application: rgw
failureDomain: osd
parameters:
pg_autoscale_mode: "off"
pg_num: "1"
replicated:
requireSafeReplicaSize: false
size: 1
---
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
name: object-with-cephblockpools.rgw.buckets.index
namespace: rook-ceph # namespace:cluster
spec:
application: rgw
failureDomain: osd
parameters:
pg_autoscale_mode: "off"
pg_num: "1"
replicated:
requireSafeReplicaSize: false
size: 1
---
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
name: object-with-cephblockpools.rgw.buckets.non-ec
namespace: rook-ceph # namespace:cluster
spec:
application: rgw
failureDomain: osd
parameters:
pg_autoscale_mode: "off"
pg_num: "1"
replicated:
requireSafeReplicaSize: false
size: 1
---
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
name: object-with-cephblockpools.rgw.otp
namespace: rook-ceph # namespace:cluster
spec:
application: rgw
failureDomain: osd
parameters:
pg_autoscale_mode: "off"
pg_num: "1"
replicated:
requireSafeReplicaSize: false
size: 1
---
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
name: object-with-cephblockpools.rgw.buckets.data
namespace: rook-ceph # namespace:cluster
spec:
application: rgw
failureDomain: osd
parameters:
pg_autoscale_mode: "off"
pg_num: "1"
replicated:
requireSafeReplicaSize: false
size: 1
pkg/operator/ceph/object/objectstore.go
+3
View File
@@ -758,6 +758,9 @@ func CreateObjectStorePools(context *Context, cluster *cephv1.ClusterSpec, metad
if len(missingPools) > 0 {
return fmt.Errorf("CR store pools are missing: %v", missingPools)
}
// pools exist, nothing to do
return nil
}
// get the default PG count for rgw metadata pools
tests/scripts/github-action-helper.sh
+92 -16
View File
@@ -20,6 +20,8 @@ set -xeEo pipefail
# VARIABLES #
#############
REPO_DIR="$(readlink -f -- "${BASH_SOURCE%/*}/../..")"
function find_extra_block_dev() {
# shellcheck disable=SC2005 # redirect doesn't work with sudo, so use echo
echo "$(sudo lsblk)" >/dev/stderr # print lsblk output to stderr for debugging in case of future errors
@@ -213,7 +215,7 @@ function build_rook_all() {
}
function validate_yaml() {
cd deploy/examples
cd "${REPO_DIR}/deploy/examples"
kubectl create -f crds.yaml -f common.yaml -f csi/nfs/rbac.yaml
# create the volume replication CRDs
@@ -240,7 +242,7 @@ function validate_yaml() {
function create_cluster_prerequisites() {
# this might be called from another function that has already done a cd
(cd deploy/examples && kubectl create -f crds.yaml -f common.yaml -f csi/nfs/rbac.yaml)
(cd "${REPO_DIR}/deploy/examples" && kubectl create -f crds.yaml -f common.yaml -f csi/nfs/rbac.yaml)
}
function deploy_manifest_with_local_build() {
@@ -257,6 +259,7 @@ function deploy_manifest_with_local_build() {
# Deploy toolbox with same ceph version as the cluster-test for ci
function deploy_toolbox() {
cd "${REPO_DIR}/deploy/examples"
sed -i 's/image: quay\.io\/ceph\/ceph:.*/image: quay.io\/ceph\/ceph:v18/' toolbox.yaml
kubectl create -f toolbox.yaml
}
@@ -271,9 +274,15 @@ function replace_ceph_image() {
sed -i "s|image: .*ceph/ceph:.*|image: ${ceph_image}|g" "${file}"
}
# Deploy the operator, a CephCluster, and the toolbox. This is intended to be a
# minimal deployment generic enough to be used by most canary tests. Each
# canary test should be installing its own set of resources as a job step or
# using dedicated helper functions.
function deploy_cluster() {
cd deploy/examples
cd "${REPO_DIR}/deploy/examples"
deploy_manifest_with_local_build operator.yaml
if [ $# == 0 ]; then
sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}|g" cluster-test.yaml
elif [ "$1" = "two_osds_in_device" ]; then
@@ -281,8 +290,8 @@ function deploy_cluster() {
elif [ "$1" = "osd_with_metadata_device" ]; then
sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}\n config:\n metadataDevice: /dev/test-rook-vg/test-rook-lv|g" cluster-test.yaml
elif [ "$1" = "osd_with_metadata_partition_device" ]; then
yq w -i -d0 cluster-test.yaml spec.storage.devices[0].name ${BLOCK}2
yq w -i -d0 cluster-test.yaml spec.storage.devices[0].config.metadataDevice ${BLOCK}1
yq w -i -d0 cluster-test.yaml spec.storage.devices[0].name "${BLOCK}2"
yq w -i -d0 cluster-test.yaml spec.storage.devices[0].config.metadataDevice "${BLOCK}1"
elif [ "$1" = "encryption" ]; then
sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}\n config:\n encryptedDevice: \"true\"|g" cluster-test.yaml
elif [ "$1" = "lvm" ]; then
@@ -294,13 +303,28 @@ function deploy_cluster() {
echo "invalid argument: $*" >&2
exit 1
fi
# enable monitoring
yq w -i -d0 cluster-test.yaml spec.monitoring.enabled true
kubectl create -f https://raw.githubusercontent.com/coreos/prometheus-operator/v0.71.1/bundle.yaml
kubectl create -f monitoring/rbac.yaml
# create the cluster resources
kubectl create -f cluster-test.yaml
deploy_toolbox
}
# These resources were extracted from the original deploy_cluster(), which was
# deploying a smorgasbord of resources used by multiple canary tests.
#
# Use of this function is discouraged. Existing users should migrate away and
# deploy only the necessary resources for the test scenario in their own job
# steps.
#
# The addition of new resources this function is forbidden!
function deploy_all_additional_resources_on_cluster() {
cd "${REPO_DIR}/deploy/examples"
kubectl create -f object-shared-pools-test.yaml
kubectl create -f object-a.yaml
kubectl create -f object-b.yaml
@@ -312,12 +336,11 @@ function deploy_cluster() {
kubectl create -f filesystem-mirror.yaml
kubectl create -f nfs-test.yaml
kubectl create -f subvolumegroup.yaml
deploy_toolbox
}
function deploy_csi_hostnetwork_disabled_cluster() {
create_cluster_prerequisites
cd deploy/examples
cd "${REPO_DIR}/deploy/examples"
sed -i 's/.*CSI_ENABLE_HOST_NETWORK:.*/ CSI_ENABLE_HOST_NETWORK: \"false\"/g' operator.yaml
deploy_manifest_with_local_build operator.yaml
if [ $# == 0 ]; then
@@ -424,7 +447,7 @@ function deploy_first_rook_cluster() {
DEVICE_NAME="$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
export BLOCK="/dev/${DEVICE_NAME}"
create_cluster_prerequisites
cd deploy/examples/
cd "${REPO_DIR}/deploy/examples"
deploy_manifest_with_local_build operator.yaml
yq w -i -d0 cluster-test.yaml spec.dashboard.enabled false
@@ -437,7 +460,7 @@ function deploy_first_rook_cluster() {
function deploy_second_rook_cluster() {
DEVICE_NAME="$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
export BLOCK="/dev/${DEVICE_NAME}"
cd deploy/examples/
cd "${REPO_DIR}/deploy/examples"
NAMESPACE=rook-ceph-secondary envsubst <common-second-cluster.yaml | kubectl create -f -
sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' cluster-test.yaml
yq w -i -d0 cluster-test.yaml spec.storage.deviceFilter "${DEVICE_NAME}"2
@@ -561,7 +584,7 @@ function test_multisite_object_replication() {
local cluster_2_ip
cluster_2_ip=$(get_clusterip rook-ceph-secondary rook-ceph-rgw-zone-b-multisite-store)
cd deploy/examples
cd "${REPO_DIR}/deploy/examples"
cat <<-EOF >s3cfg
[default]
host_bucket = no.way
@@ -620,7 +643,7 @@ EOF
}
function deploy_multus_cluster() {
cd deploy/examples
cd "${REPO_DIR}/deploy/examples"
sed -i 's/.*ROOK_CSI_ENABLE_NFS:.*/ ROOK_CSI_ENABLE_NFS: \"true\"/g' operator.yaml
deploy_manifest_with_local_build operator.yaml
deploy_toolbox
@@ -639,7 +662,7 @@ function test_multus_connections() {
}
function create_operator_toolbox() {
cd deploy/examples
cd "${REPO_DIR}/deploy/examples"
sed -i "s|image: docker.io/rook/ceph:.*|image: docker.io/rook/ceph:local-build|g" toolbox-operator-image.yaml
kubectl create -f toolbox-operator-image.yaml
}
@@ -666,7 +689,7 @@ EOF
}
function test_csi_rbd_workload {
cd deploy/examples/csi/rbd
cd "${REPO_DIR}/deploy/examples/csi/rbd"
sed -i 's|size: 3|size: 1|g' storageclass.yaml
sed -i 's|requireSafeReplicaSize: true|requireSafeReplicaSize: false|g' storageclass.yaml
kubectl create -f storageclass.yaml
@@ -681,7 +704,7 @@ function test_csi_rbd_workload {
}
function test_csi_cephfs_workload {
cd deploy/examples/csi/cephfs
cd "${REPO_DIR}/deploy/examples/csi/cephfs"
kubectl create -f storageclass.yaml
kubectl create -f pvc.yaml
kubectl create -f pod.yaml
@@ -694,7 +717,7 @@ function test_csi_cephfs_workload {
}
function test_csi_nfs_workload {
cd deploy/examples/csi/nfs
cd "${REPO_DIR}/deploy/examples/csi/nfs"
sed -i "s|#- debug|- nolock|" storageclass.yaml
kubectl create -f storageclass.yaml
kubectl create -f pvc.yaml
@@ -738,6 +761,59 @@ function install_minikube_with_none_driver() {
minikube start --kubernetes-version="$1" --driver=none --memory 6g --cpus=2 --addons ingress --cni=calico
}
function toolbox() {
kubectl -n rook-ceph exec -it "$(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[0].metadata.name}')" -- "$@"
}
function ceph() {
toolbox ceph "$@"
}
function rbd() {
toolbox rbd "$@"
}
function radosgw-admin() {
toolbox radosgw-admin "$@"
}
function test_object_with_cephblockpools_extra_pools() {
expected_pools=(
.mgr
.rgw.root
object-with-cephblockpools.rgw.control
object-with-cephblockpools.rgw.meta
object-with-cephblockpools.rgw.log
object-with-cephblockpools.rgw.buckets.index
object-with-cephblockpools.rgw.buckets.non-ec
object-with-cephblockpools.rgw.otp
object-with-cephblockpools.rgw.buckets.data
)
output=$(ceph osd pool ls)
readarray -t live_pools < <(printf '%s' "$output")
errors=0
for l in "${live_pools[@]}"; do
found=false
for e in "${expected_pools[@]}"; do
if [[ "$l" == "$e" ]]; then
found=true
break
fi
done
if [[ "$found" == false ]]; then
echo "Live pool $l is not an expected pool"
errors=$((errors+1))
fi
done
if [[ $errors -gt 0 ]]; then
echo "Found $errors errors"
exit $errors
fi
}
FUNCTION="$1"
shift # remove function arg now that we've recorded it
# call the function with the remainder of the user-provided args