Implement OSD trim to improve SSD speeds

This commit is contained in:
feedc0de committed 2026-10-11 15:25:34 +02:00
1 parent 77ee09b06b
commit 88028ba6d5
4 files changed
+160 -1

No files matched your search

+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env bash
set -euo pipefail
# The user default is copied when a bucket is created; it does not change
# existing buckets. Those need S3 lifecycle rules or explicit upload headers.
# Existing STANDARD objects still live in the replicated pool, so do not remap
# STANDARD itself until every old object has been migrated and verified.
apply=false
if [[ "${1:-}" == --apply ]]; then
apply=true
shift
fi
if [[ $# -ne 0 ]]; then
echo "Usage: $0 [--apply]" >&2
exit 64
fi
expected_context=kubernetes-admin@kubernetes
context=$(kubectl config current-context)
if [[ "$context" != "$expected_context" ]]; then
echo "Refusing RGW changes in context $context" >&2
exit 77
fi
namespace=rook-ceph
store=my-store
storage_class=STANDARD_IA
placement=default-placement
pool=my-store.rgw.buckets.ec
tools_pod=$(kubectl -n "$namespace" get pods -l app=rook-ceph-tools \
--field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}')
[[ -n "$tools_pod" ]] || { echo 'No running Rook toolbox pod' >&2; exit 69; }
pool_type=$(kubectl -n "$namespace" get cephblockpool "$pool" \
-o jsonpath='{.status.info.type}')
[[ "$pool_type" == 'Erasure Coded' ]] || { echo "$pool is not EC" >&2; exit 65; }
zone=$(kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin zone get --rgw-zone "$store")
jq -e --arg pool "$pool" --arg class "$storage_class" '
any(.placement_pools[]; .key == "default-placement" and .val.storage_classes[$class].data_pool == $pool)
' <<<"$zone" >/dev/null || { echo 'RGW class does not point to the expected EC pool' >&2; exit 65; }
unset zone
users_json=$(kubectl -n "$namespace" get cephobjectstoreusers -o json)
mapfile -t users < <(jq -r --arg store "$store" \
'.items[] | select(.spec.store == $store) | .metadata.name' <<<"$users_json" | sort)
unset users_json
for user in "${users[@]}"; do
info=$(kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin user info --uid "$user")
current=$(jq -r '.default_storage_class // ""' <<<"$info")
current_placement=$(jq -r '.default_placement // ""' <<<"$info")
unset info
if [[ "$current" == "$storage_class" && "$current_placement" == "$placement" ]]; then
echo "$user: already $storage_class"
continue
fi
if [[ -n "$current_placement" && "$current_placement" != "$placement" ]]; then
echo "$user: refusing to replace existing placement $current_placement" >&2
exit 65
fi
if [[ -n "$current" ]]; then
echo "$user: refusing to replace existing default class $current" >&2
exit 65
fi
if [[ "$apply" != true ]]; then
echo "$user: would set $storage_class"
continue
fi
# The command's JSON response contains the user's S3 keys; never print it.
kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin user modify \
--uid "$user" --placement-id "$placement" --storage-class "$storage_class" >/dev/null
verified=$(kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin user info --uid "$user" | \
jq -r '[.default_placement, .default_storage_class] | join("/")')
[[ "$verified" == "$placement/$storage_class" ]] || { echo "$user: verification failed" >&2; exit 1; }
echo "$user: set $storage_class"
done
+5 -1
View File
@@ -78,9 +78,13 @@ spec:
prometheusEndpoint: http://prometheus-kube-prometheus-prometheus.prometheus.svc:9090
# Whether SSL should be verified if the Prometheus server is using https
# prometheusEndpointSSLVerify: false
# Persist dashboard integrations which do not have first-class fields above.
# Persist OSD discard settings and dashboard integrations.
# Backend URLs use cluster DNS; the Grafana frontend URL must be browser-accessible.
cephConfig:
osd:
# Return freed blocks to the SSD without blocking the OSD on each discard.
bdev_enable_discard: "true"
bdev_async_discard_threads: "1"
mgr:
mgr/dashboard/ALERTMANAGER_API_HOST: "http://prometheus-kube-prometheus-alertmanager.prometheus.svc:9093"
mgr/dashboard/GRAFANA_API_URL: "http://grafana.default.svc:80"
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env bash
set -euo pipefail
# One-time continuation for the large existing RGW buckets. The S3 lifecycle
# rules are the source of truth and RGW's own LC threads remain enabled. This
# script only forces a single bucket at a time, without changing pool mappings.
expected_context=kubernetes-admin@kubernetes
namespace=rook-ceph
tools_pod=$(kubectl -n "$namespace" get pod -l app=rook-ceph-tools \
--field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}')
[[ -n "$tools_pod" ]] || { echo 'No running Rook toolbox pod' >&2; exit 69; }
check_context() {
local context
context=$(kubectl config current-context)
[[ "$context" == "$expected_context" ]] || {
echo "Refusing RGW changes in context $context" >&2
exit 77
}
}
bucket_status() {
kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin lc list | \
jq -r --arg bucket "$1" \
'[.[] | select(.bucket | startswith(":" + $bucket + ":")) | .status] | first // "MISSING"'
}
check_context
for bucket in synapse quay nextcloud; do
# Another manual pass may still be running. Avoid forcing concurrent LC
# work while the OSDs are reporting slow operations.
while kubectl -n "$namespace" exec "$tools_pod" -- ps -eo args | \
grep -Eq '^radosgw-admin lc process --bucket '; do
echo "Waiting for active RGW lifecycle pass before $bucket"
sleep 60
done
check_context
health=$(kubectl -n "$namespace" exec "$tools_pod" -- ceph health)
[[ "$health" != HEALTH_ERR* ]] || {
echo "Ceph health is $health; refusing forced lifecycle work" >&2
exit 1
}
status=$(bucket_status "$bucket")
if [[ "$status" == COMPLETE ]]; then
echo "$bucket: already COMPLETE"
continue
fi
[[ "$status" != MISSING ]] || { echo "$bucket: lifecycle rule missing" >&2; exit 1; }
echo "$bucket: starting forced lifecycle pass (previous status: $status)"
kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin lc process --bucket "$bucket"
status=$(bucket_status "$bucket")
[[ "$status" == COMPLETE ]] || {
echo "$bucket: pass ended with status $status" >&2
exit 1
}
echo "$bucket: COMPLETE"
done
kubectl -n "$namespace" exec "$tools_pod" -- ceph df -f json | \
jq -r '.pools[] | select(.name == "my-store.rgw.buckets.data" or .name == "my-store.rgw.buckets.ec") | [.name,.stats.stored,.stats.bytes_used] | @tsv'
+18
View File
@@ -0,0 +1,18 @@
apiVersion: csi.ceph.io/v1
kind: Driver
metadata:
name: rook-ceph.rbd.csi.ceph.com
namespace: rook-ceph
spec:
fsGroupPolicy: File
controllerPlugin: {}
nodePlugin:
tolerations:
# arschrock is the control-plane node and hosts only explicitly pinned
# application workloads. The node plugin is required to attach RBD for
# those workloads; this does not make other pods schedulable there.
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
updateStrategy:
type: RollingUpdate