diff --git a/configure-rgw-ec-user-defaults.sh b/configure-rgw-ec-user-defaults.sh new file mode 100755 index 000000000..0be1dc6d7 --- /dev/null +++ b/configure-rgw-ec-user-defaults.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -euo pipefail + +# The user default is copied when a bucket is created; it does not change +# existing buckets. Those need S3 lifecycle rules or explicit upload headers. +# Existing STANDARD objects still live in the replicated pool, so do not remap +# STANDARD itself until every old object has been migrated and verified. +apply=false +if [[ "${1:-}" == --apply ]]; then + apply=true + shift +fi +if [[ $# -ne 0 ]]; then + echo "Usage: $0 [--apply]" >&2 + exit 64 +fi + +expected_context=kubernetes-admin@kubernetes +context=$(kubectl config current-context) +if [[ "$context" != "$expected_context" ]]; then + echo "Refusing RGW changes in context $context" >&2 + exit 77 +fi + +namespace=rook-ceph +store=my-store +storage_class=STANDARD_IA +placement=default-placement +pool=my-store.rgw.buckets.ec +tools_pod=$(kubectl -n "$namespace" get pods -l app=rook-ceph-tools \ + --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}') +[[ -n "$tools_pod" ]] || { echo 'No running Rook toolbox pod' >&2; exit 69; } + +pool_type=$(kubectl -n "$namespace" get cephblockpool "$pool" \ + -o jsonpath='{.status.info.type}') +[[ "$pool_type" == 'Erasure Coded' ]] || { echo "$pool is not EC" >&2; exit 65; } +zone=$(kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin zone get --rgw-zone "$store") +jq -e --arg pool "$pool" --arg class "$storage_class" ' + any(.placement_pools[]; .key == "default-placement" and .val.storage_classes[$class].data_pool == $pool) +' <<<"$zone" >/dev/null || { echo 'RGW class does not point to the expected EC pool' >&2; exit 65; } +unset zone + +users_json=$(kubectl -n "$namespace" get cephobjectstoreusers -o json) +mapfile -t users < <(jq -r --arg store "$store" \ + '.items[] | select(.spec.store == $store) | .metadata.name' <<<"$users_json" | sort) +unset users_json +for user in "${users[@]}"; do + info=$(kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin user info --uid "$user") + current=$(jq -r '.default_storage_class // ""' <<<"$info") + current_placement=$(jq -r '.default_placement // ""' <<<"$info") + unset info + if [[ "$current" == "$storage_class" && "$current_placement" == "$placement" ]]; then + echo "$user: already $storage_class" + continue + fi + if [[ -n "$current_placement" && "$current_placement" != "$placement" ]]; then + echo "$user: refusing to replace existing placement $current_placement" >&2 + exit 65 + fi + if [[ -n "$current" ]]; then + echo "$user: refusing to replace existing default class $current" >&2 + exit 65 + fi + if [[ "$apply" != true ]]; then + echo "$user: would set $storage_class" + continue + fi + # The command's JSON response contains the user's S3 keys; never print it. + kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin user modify \ + --uid "$user" --placement-id "$placement" --storage-class "$storage_class" >/dev/null + verified=$(kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin user info --uid "$user" | \ + jq -r '[.default_placement, .default_storage_class] | join("/")') + [[ "$verified" == "$placement/$storage_class" ]] || { echo "$user: verification failed" >&2; exit 1; } + echo "$user: set $storage_class" +done diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml index bf34975f8..bc260894a 100644 --- a/deploy/examples/cluster.yaml +++ b/deploy/examples/cluster.yaml @@ -78,9 +78,13 @@ spec: prometheusEndpoint: http://prometheus-kube-prometheus-prometheus.prometheus.svc:9090 # Whether SSL should be verified if the Prometheus server is using https # prometheusEndpointSSLVerify: false - # Persist dashboard integrations which do not have first-class fields above. + # Persist OSD discard settings and dashboard integrations. # Backend URLs use cluster DNS; the Grafana frontend URL must be browser-accessible. cephConfig: + osd: + # Return freed blocks to the SSD without blocking the OSD on each discard. + bdev_enable_discard: "true" + bdev_async_discard_threads: "1" mgr: mgr/dashboard/ALERTMANAGER_API_HOST: "http://prometheus-kube-prometheus-alertmanager.prometheus.svc:9093" mgr/dashboard/GRAFANA_API_URL: "http://grafana.default.svc:80" diff --git a/finish-rgw-ec-conversion.sh b/finish-rgw-ec-conversion.sh new file mode 100755 index 000000000..8a9d30560 --- /dev/null +++ b/finish-rgw-ec-conversion.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail + +# One-time continuation for the large existing RGW buckets. The S3 lifecycle +# rules are the source of truth and RGW's own LC threads remain enabled. This +# script only forces a single bucket at a time, without changing pool mappings. +expected_context=kubernetes-admin@kubernetes +namespace=rook-ceph +tools_pod=$(kubectl -n "$namespace" get pod -l app=rook-ceph-tools \ + --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}') +[[ -n "$tools_pod" ]] || { echo 'No running Rook toolbox pod' >&2; exit 69; } + +check_context() { + local context + context=$(kubectl config current-context) + [[ "$context" == "$expected_context" ]] || { + echo "Refusing RGW changes in context $context" >&2 + exit 77 + } +} + +bucket_status() { + kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin lc list | \ + jq -r --arg bucket "$1" \ + '[.[] | select(.bucket | startswith(":" + $bucket + ":")) | .status] | first // "MISSING"' +} + +check_context +for bucket in synapse quay nextcloud; do + # Another manual pass may still be running. Avoid forcing concurrent LC + # work while the OSDs are reporting slow operations. + while kubectl -n "$namespace" exec "$tools_pod" -- ps -eo args | \ + grep -Eq '^radosgw-admin lc process --bucket '; do + echo "Waiting for active RGW lifecycle pass before $bucket" + sleep 60 + done + + check_context + health=$(kubectl -n "$namespace" exec "$tools_pod" -- ceph health) + [[ "$health" != HEALTH_ERR* ]] || { + echo "Ceph health is $health; refusing forced lifecycle work" >&2 + exit 1 + } + status=$(bucket_status "$bucket") + if [[ "$status" == COMPLETE ]]; then + echo "$bucket: already COMPLETE" + continue + fi + [[ "$status" != MISSING ]] || { echo "$bucket: lifecycle rule missing" >&2; exit 1; } + + echo "$bucket: starting forced lifecycle pass (previous status: $status)" + kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin lc process --bucket "$bucket" + status=$(bucket_status "$bucket") + [[ "$status" == COMPLETE ]] || { + echo "$bucket: pass ended with status $status" >&2 + exit 1 + } + echo "$bucket: COMPLETE" +done + +kubectl -n "$namespace" exec "$tools_pod" -- ceph df -f json | \ + jq -r '.pools[] | select(.name == "my-store.rgw.buckets.data" or .name == "my-store.rgw.buckets.ec") | [.name,.stats.stored,.stats.bytes_used] | @tsv' diff --git a/rbd-nodeplugin-arschrock.yaml b/rbd-nodeplugin-arschrock.yaml new file mode 100644 index 000000000..6bc9b5e98 --- /dev/null +++ b/rbd-nodeplugin-arschrock.yaml @@ -0,0 +1,18 @@ +apiVersion: csi.ceph.io/v1 +kind: Driver +metadata: + name: rook-ceph.rbd.csi.ceph.com + namespace: rook-ceph +spec: + fsGroupPolicy: File + controllerPlugin: {} + nodePlugin: + tolerations: + # arschrock is the control-plane node and hosts only explicitly pinned + # application workloads. The node plugin is required to attach RBD for + # those workloads; this does not make other pods schedulable there. + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + updateStrategy: + type: RollingUpdate