Implement OSD trim to improve SSD speeds
This commit is contained in:
1 parent
77ee09b06b
commit
88028ba6d5
4 files changed
+160
-1
No files matched your search
Executable
+75
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# The user default is copied when a bucket is created; it does not change
|
||||
# existing buckets. Those need S3 lifecycle rules or explicit upload headers.
|
||||
# Existing STANDARD objects still live in the replicated pool, so do not remap
|
||||
# STANDARD itself until every old object has been migrated and verified.
|
||||
apply=false
|
||||
if [[ "${1:-}" == --apply ]]; then
|
||||
apply=true
|
||||
shift
|
||||
fi
|
||||
if [[ $# -ne 0 ]]; then
|
||||
echo "Usage: $0 [--apply]" >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
expected_context=kubernetes-admin@kubernetes
|
||||
context=$(kubectl config current-context)
|
||||
if [[ "$context" != "$expected_context" ]]; then
|
||||
echo "Refusing RGW changes in context $context" >&2
|
||||
exit 77
|
||||
fi
|
||||
|
||||
namespace=rook-ceph
|
||||
store=my-store
|
||||
storage_class=STANDARD_IA
|
||||
placement=default-placement
|
||||
pool=my-store.rgw.buckets.ec
|
||||
tools_pod=$(kubectl -n "$namespace" get pods -l app=rook-ceph-tools \
|
||||
--field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}')
|
||||
[[ -n "$tools_pod" ]] || { echo 'No running Rook toolbox pod' >&2; exit 69; }
|
||||
|
||||
pool_type=$(kubectl -n "$namespace" get cephblockpool "$pool" \
|
||||
-o jsonpath='{.status.info.type}')
|
||||
[[ "$pool_type" == 'Erasure Coded' ]] || { echo "$pool is not EC" >&2; exit 65; }
|
||||
zone=$(kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin zone get --rgw-zone "$store")
|
||||
jq -e --arg pool "$pool" --arg class "$storage_class" '
|
||||
any(.placement_pools[]; .key == "default-placement" and .val.storage_classes[$class].data_pool == $pool)
|
||||
' <<<"$zone" >/dev/null || { echo 'RGW class does not point to the expected EC pool' >&2; exit 65; }
|
||||
unset zone
|
||||
|
||||
users_json=$(kubectl -n "$namespace" get cephobjectstoreusers -o json)
|
||||
mapfile -t users < <(jq -r --arg store "$store" \
|
||||
'.items[] | select(.spec.store == $store) | .metadata.name' <<<"$users_json" | sort)
|
||||
unset users_json
|
||||
for user in "${users[@]}"; do
|
||||
info=$(kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin user info --uid "$user")
|
||||
current=$(jq -r '.default_storage_class // ""' <<<"$info")
|
||||
current_placement=$(jq -r '.default_placement // ""' <<<"$info")
|
||||
unset info
|
||||
if [[ "$current" == "$storage_class" && "$current_placement" == "$placement" ]]; then
|
||||
echo "$user: already $storage_class"
|
||||
continue
|
||||
fi
|
||||
if [[ -n "$current_placement" && "$current_placement" != "$placement" ]]; then
|
||||
echo "$user: refusing to replace existing placement $current_placement" >&2
|
||||
exit 65
|
||||
fi
|
||||
if [[ -n "$current" ]]; then
|
||||
echo "$user: refusing to replace existing default class $current" >&2
|
||||
exit 65
|
||||
fi
|
||||
if [[ "$apply" != true ]]; then
|
||||
echo "$user: would set $storage_class"
|
||||
continue
|
||||
fi
|
||||
# The command's JSON response contains the user's S3 keys; never print it.
|
||||
kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin user modify \
|
||||
--uid "$user" --placement-id "$placement" --storage-class "$storage_class" >/dev/null
|
||||
verified=$(kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin user info --uid "$user" | \
|
||||
jq -r '[.default_placement, .default_storage_class] | join("/")')
|
||||
[[ "$verified" == "$placement/$storage_class" ]] || { echo "$user: verification failed" >&2; exit 1; }
|
||||
echo "$user: set $storage_class"
|
||||
done
|
||||
@@ -78,9 +78,13 @@ spec:
|
||||
prometheusEndpoint: http://prometheus-kube-prometheus-prometheus.prometheus.svc:9090
|
||||
# Whether SSL should be verified if the Prometheus server is using https
|
||||
# prometheusEndpointSSLVerify: false
|
||||
# Persist dashboard integrations which do not have first-class fields above.
|
||||
# Persist OSD discard settings and dashboard integrations.
|
||||
# Backend URLs use cluster DNS; the Grafana frontend URL must be browser-accessible.
|
||||
cephConfig:
|
||||
osd:
|
||||
# Return freed blocks to the SSD without blocking the OSD on each discard.
|
||||
bdev_enable_discard: "true"
|
||||
bdev_async_discard_threads: "1"
|
||||
mgr:
|
||||
mgr/dashboard/ALERTMANAGER_API_HOST: "http://prometheus-kube-prometheus-alertmanager.prometheus.svc:9093"
|
||||
mgr/dashboard/GRAFANA_API_URL: "http://grafana.default.svc:80"
|
||||
|
||||
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# One-time continuation for the large existing RGW buckets. The S3 lifecycle
|
||||
# rules are the source of truth and RGW's own LC threads remain enabled. This
|
||||
# script only forces a single bucket at a time, without changing pool mappings.
|
||||
expected_context=kubernetes-admin@kubernetes
|
||||
namespace=rook-ceph
|
||||
tools_pod=$(kubectl -n "$namespace" get pod -l app=rook-ceph-tools \
|
||||
--field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}')
|
||||
[[ -n "$tools_pod" ]] || { echo 'No running Rook toolbox pod' >&2; exit 69; }
|
||||
|
||||
check_context() {
|
||||
local context
|
||||
context=$(kubectl config current-context)
|
||||
[[ "$context" == "$expected_context" ]] || {
|
||||
echo "Refusing RGW changes in context $context" >&2
|
||||
exit 77
|
||||
}
|
||||
}
|
||||
|
||||
bucket_status() {
|
||||
kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin lc list | \
|
||||
jq -r --arg bucket "$1" \
|
||||
'[.[] | select(.bucket | startswith(":" + $bucket + ":")) | .status] | first // "MISSING"'
|
||||
}
|
||||
|
||||
check_context
|
||||
for bucket in synapse quay nextcloud; do
|
||||
# Another manual pass may still be running. Avoid forcing concurrent LC
|
||||
# work while the OSDs are reporting slow operations.
|
||||
while kubectl -n "$namespace" exec "$tools_pod" -- ps -eo args | \
|
||||
grep -Eq '^radosgw-admin lc process --bucket '; do
|
||||
echo "Waiting for active RGW lifecycle pass before $bucket"
|
||||
sleep 60
|
||||
done
|
||||
|
||||
check_context
|
||||
health=$(kubectl -n "$namespace" exec "$tools_pod" -- ceph health)
|
||||
[[ "$health" != HEALTH_ERR* ]] || {
|
||||
echo "Ceph health is $health; refusing forced lifecycle work" >&2
|
||||
exit 1
|
||||
}
|
||||
status=$(bucket_status "$bucket")
|
||||
if [[ "$status" == COMPLETE ]]; then
|
||||
echo "$bucket: already COMPLETE"
|
||||
continue
|
||||
fi
|
||||
[[ "$status" != MISSING ]] || { echo "$bucket: lifecycle rule missing" >&2; exit 1; }
|
||||
|
||||
echo "$bucket: starting forced lifecycle pass (previous status: $status)"
|
||||
kubectl -n "$namespace" exec "$tools_pod" -- radosgw-admin lc process --bucket "$bucket"
|
||||
status=$(bucket_status "$bucket")
|
||||
[[ "$status" == COMPLETE ]] || {
|
||||
echo "$bucket: pass ended with status $status" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "$bucket: COMPLETE"
|
||||
done
|
||||
|
||||
kubectl -n "$namespace" exec "$tools_pod" -- ceph df -f json | \
|
||||
jq -r '.pools[] | select(.name == "my-store.rgw.buckets.data" or .name == "my-store.rgw.buckets.ec") | [.name,.stats.stored,.stats.bytes_used] | @tsv'
|
||||
@@ -0,0 +1,18 @@
|
||||
apiVersion: csi.ceph.io/v1
|
||||
kind: Driver
|
||||
metadata:
|
||||
name: rook-ceph.rbd.csi.ceph.com
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
fsGroupPolicy: File
|
||||
controllerPlugin: {}
|
||||
nodePlugin:
|
||||
tolerations:
|
||||
# arschrock is the control-plane node and hosts only explicitly pinned
|
||||
# application workloads. The node plugin is required to attach RBD for
|
||||
# those workloads; this does not make other pods schedulable there.
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
updateStrategy:
|
||||
type: RollingUpdate
|
||||
Reference in new issue
Block a user