From 827ee1d8794db0bf2a47dd9587bfecac73155dcc Mon Sep 17 00:00:00 2001 From: parth-gr Date: Tue, 8 Jun 2021 20:30:59 +0530 Subject: [PATCH] ceph: auto grow OSDs size on PVCs When an OSD reaches OSD_NEARFULL state, we have to manually increase the PVC volume claim or manually increase the count of OSDs in the device set Added a script auto-grow-storage.sh which will i)automatically increase claim volume ii)automatically add number of OSDs Closes: https://github.com/rook/rook/issues/6101 Signed-off-by: parth-gr --- Documentation/ceph-advanced-configuration.md | 37 +++ tests/scripts/auto-grow-storage.sh | 263 +++++++++++++++++++ 2 files changed, 300 insertions(+) create mode 100755 tests/scripts/auto-grow-storage.sh diff --git a/Documentation/ceph-advanced-configuration.md b/Documentation/ceph-advanced-configuration.md index 906f3c1d4..f2913f0d2 100644 --- a/Documentation/ceph-advanced-configuration.md +++ b/Documentation/ceph-advanced-configuration.md @@ -22,6 +22,7 @@ storage cluster. * [OSD Dedicated Network](#osd-dedicated-network) * [Phantom OSD Removal](#phantom-osd-removal) * [Change Failure Domain](#change-failure-domain) +* [Auto Expansion of OSDs](#auto-expansion-of-OSDs) ## Prerequisites @@ -590,3 +591,39 @@ ceph osd pool get replicapool crush_rule If the cluster's health was `HEALTH_OK` when we performed this change, immediately, the new rule is applied to the cluster transparently without service disruption. Exactly the same approach can be used to change from `host` back to `osd`. + +## Auto Expansion of OSDs + +### Prerequisites + +1) A [PVC-based cluster](ceph-cluster-crd.md#pvc-based-cluster) deployed in dynamic provisioning environment with a `storageClassDeviceSet`. + +2) Create the Rook [Toolbox](ceph-toolbox.md). + +>Note: [Prometheus Operator](ceph-monitoring.md#prometheus-operator) and [Prometheus Instances](ceph-monitoring.md#prometheus-instances) are Prerequisites that are created by the auto-grow-storage script. + +### To scale OSDs Vertically + +Run the following script to auto-grow the size of OSDs on a PVC-based Rook-Ceph cluster whenever the OSDs have reached the storage near-full threshold. +```console +tests/scripts/auto-grow-storage.sh size --max maxSize --growth-rate percent +``` +>growth-rate percentage represents the percent increase you want in the OSD capacity and maxSize represent the maximum disk size. + +For example, if you need to increase the size of OSD by 30% and max disk size is 1Ti +```console +./auto-grow-storage.sh size --max 1Ti --growth-rate 30 +``` + +### To scale OSDs Horizontally + +Run the following script to auto-grow the number of OSDs on a PVC-based Rook-Ceph cluster whenever the OSDs have reached the storage near-full threshold. +```console +tests/scripts/auto-grow-storage.sh count --max maxCount --count rate +``` +>Count of OSD represents the number of OSDs you need to add and maxCount represents the number of disks a storage cluster will support. + +For example, if you need to increase the number of OSDs by 3 and maxCount is 10 +```console +./auto-grow-storage.sh count --max 10 --count 3 +``` diff --git a/tests/scripts/auto-grow-storage.sh b/tests/scripts/auto-grow-storage.sh new file mode 100755 index 000000000..15a6a2649 --- /dev/null +++ b/tests/scripts/auto-grow-storage.sh @@ -0,0 +1,263 @@ +#!/usr/bin/env bash + +############# +# FUNCTIONS # +############# + +function calculateSize() { + local currentsize=$2 + local unit=$1 + rawsizeValue=0 # rawsizeValue is a global variable + + if [[ "$currentsize" == *"Mi" ]] + then + rawSize=${currentsize//Mi} # rawSize is a global variable + unitSize="Mi" + rawsizeValue=$rawSize + elif [[ "$currentsize" == *"Gi" ]] + then + rawSize=${currentsize//Gi} + unitSize="Gi" + rawsizeValue=$(( rawSize * 1000 )) + elif [[ "$currentsize" == *"Ti" ]] + then + rawSize=${currentsize//Ti} + unitSize="Ti" + rawsizeValue=$(( rawSize * 1000000 )) + else + echo "Unknown unit of $unit : ${currentsize}" + echo "Supported units are 'Mi','Gi','Ti'" + exit 1 + fi +} + +function compareSizes() { + local newsize=$1 + local maxsize=$2 + calculateSize newsize "${newsize}" # rawsizeValue is calculated and used for further process + local newsize=$rawsizeValue + calculateSize maxsize "${maxsize}" + local maxsize=$rawsizeValue + if [ "${newsize}" -ge "${maxsize}" ] + then + return "1" + fi + return "0" +} + +function growVertically() { + local growRate=$1 + local pvc=$2 + local ns=$3 + local maxSize=$4 + local currentSize + currentSize=$(kubectl get pvc "${pvc}" -n "${ns}" -o json | jq -r '.spec.resources.requests.storage') + echo "PVC(OSD) current size is ${currentSize} and will be increased by ${growRate}%." + + calculateSize "${pvc}" "${currentSize}" # rawSize is calculated and used for further process + + if ! [[ "${rawSize}" =~ ^[0-9]+$ ]] + then + echo "disk size should be an integer" + else + newSize=$(echo "${rawSize}+(${rawSize} * ${growRate})/100" | bc | cut -f1 -d'.') + if [ "${newSize}" = "${rawSize}" ] + then + newSize=$(( rawSize + 1 )) + echo "New adjusted calculated size for the PVC is ${newSize}${unitSize}" + else + echo "New calculated size for the PVC is ${newSize}${unitSize}" + fi + + compareSizes ${newSize}${unitSize} "${maxSize}" + if [ "1" = $? ] + then + newSize=${maxSize} + echo "Disk has reached it's MAX capacity ${maxSize}, add a new disk to it" + result=$(kubectl patch pvc "${pvc}" -n "${ns}" --type json --patch "[{ op: replace, path: /spec/resources/requests/storage, value: ${newSize} }]") + else + result=$(kubectl patch pvc "${pvc}" -n "${ns}" --type json --patch "[{ op: replace, path: /spec/resources/requests/storage, value: ${newSize}${unitSize} }]") + fi + echo "${result}" + fi +} + +function growHorizontally() { + local increaseOSDCount=$1 + local pvc=$2 + local ns=$3 + local maxOSDCount=$4 + local deviceSetName + local cluster="" + local deviceSet="" + local currentOSDCount=0 + local clusterCount=0 + local deviceSetCount=0 + deviceSetName=$(kubectl get pvc "${pvc}" -n "${ns}" -o json | jq -r '.metadata.labels."ceph.rook.io/DeviceSet"') + while [ "$cluster" != "null" ] + do + cluster=$(kubectl get CephCluster -n "${ns}" -o json | jq -r ".items[${clusterCount}]") + while [ "$deviceSet" != "null" ] + do + deviceSet=$(kubectl get CephCluster -n "${ns}" -o json | jq -r ".items[${clusterCount}].spec.storage.storageClassDeviceSets[${deviceSetCount}].name") + if [[ $deviceSet == "${deviceSetName}" ]] + then + currentOSDCount=$(kubectl get CephCluster -n "${ns}" -o json | jq -r ".items[${clusterCount}].spec.storage.storageClassDeviceSets[${deviceSetCount}].count") + finalCount=$(( "${currentOSDCount}" + "${increaseOSDCount}" )) + echo "OSD count: ${currentOSDCount}. OSD count will be increased by ${increaseOSDCount}." + if [ "${finalCount}" -ge "${maxOSDCount}" ] + then + finalCount=${maxOSDCount} + echo "DeviceSet ${deviceSet} capacity is full, cannot add more OSD to it" + fi + echo "Total count of OSDs for deviceset ${deviceSetName} is set to ${finalCount}." + clusterName=$(kubectl get CephCluster -n "${ns}" -o json | jq -r ".items[${clusterCount}].metadata.name" ) + result=$(kubectl patch CephCluster "${clusterName}" -n "${ns}" --type json --patch "[{ op: replace, path: /spec/storage/storageClassDeviceSets/${deviceSetCount}/count, value: ${finalCount} }]") + echo "${result}" + break + fi + deviceSetCount=$((deviceSetCount+1)) + deviceSet=$(kubectl get CephCluster -n "${ns}" -o json | jq -r ".items[${clusterCount}].spec.storage.storageClassDeviceSets[${deviceSetCount}].name") + done + clusterCount=$((clusterCount+1)) + cluster=$(kubectl get CephCluster -n "${ns}" -o json | jq -r ".items[${clusterCount}]") + done +} + +function growOSD(){ + itr=0 + alertmanagerroute=$(kubectl -n rook-ceph -o jsonpath="{.status.hostIP}" get pod prometheus-rook-prometheus-0) + route=${alertmanagerroute}:30900 + toolbox=$(kubectl get pods -n rook-ceph | grep -i rook-ceph-tools | awk '{ print $1 }') + alerts=$(kubectl exec -it "${toolbox}" -n rook-ceph -- bash -c "curl -s http://${route}/api/v1/alerts") + export total_alerts + total_alerts=$( jq '.data.alerts | length' <<< "${alerts}") + echo "Looping at $(date +"%Y-%m-%d %H:%M:%S")" + + while true + do + if [ "${total_alerts}" == "" ] + then + echo "Alert manager not configured,re-run the script" + exit 1 + fi + export entry + entry=$( jq ".data.alerts[$itr]" <<< "${alerts}") + thename=$(echo "${entry}" | jq -r '.labels.alertname') + if [ "${thename}" = "CephOSDNearFull" ] || [ "${thename}" = "CephOSDCriticallyFull" ] + then + echo "${entry}" + ns=$(echo "${entry}" | jq -r '.labels.namespace') + osdID=$(echo "${entry}" | jq -r '.labels.ceph_daemon') + osdID=${osdID/./-} + pvc=$(kubectl get deployment -n "${ns}" rook-ceph-"${osdID}" -o json | jq -r '.metadata.labels."ceph.rook.io/pvc"') + if [[ $pvc == null ]] + then + echo "PVC not found, script can only run on PVC-based cluster" + exit 1 + fi + echo "Processing NearFull or Full alert for PVC ${pvc} in namespace ${ns}" + if [[ $1 == "count" ]] + then + growHorizontally "$2" "${pvc}" "${ns}" "$3" + else + growVertically "$2" "${pvc}" "${ns}" "$3" + fi + fi + (( itr = itr + 1 )) + if [[ "${itr}" == "${total_alerts}" ]] || [[ "${total_alerts}" == "0" ]] + then + sleep 600 + alerts=$(kubectl exec -it "${toolbox}" -n rook-ceph -- bash -c "curl -s http://${route}/api/v1/alerts") + total_alerts=$( jq '.data.alerts | length' <<< "${alerts}") + itr=0 + echo "Looping at $(date +"%Y-%m-%d %H:%M:%S")" + fi + done +} + +function creatingPrerequisites(){ + echo "creating Prerequisites deployments - Prometheus Operator and Prometheus Instances" + # creating Prometheus operator + kubectl apply -f https://raw.githubusercontent.com/coreos/prometheus-operator/v0.40.0/bundle.yaml + # waitng for Prometheus operator to get ready + timeout 30 sh -c "until [ $(kubectl get pod -l app.kubernetes.'io/name'=prometheus-operator -o json | jq -r '.items[0].status.phase') = Running ]; do echo 'waiting for prometheus-operator to get created' && sleep 1; done" + # creating a service monitor that will watch the Rook cluster and collect metrics regularly + kubectl create -f https://raw.githubusercontent.com/rook/rook/master/cluster/examples/kubernetes/ceph/monitoring/service-monitor.yaml + # create the PrometheusRule for Rook alerts. + kubectl create -f https://raw.githubusercontent.com/rook/rook/master/cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules.yaml + # create prometheus-rook-prometheus-0 pod + kubectl create -f https://raw.githubusercontent.com/rook/rook/master/cluster/examples/kubernetes/ceph/monitoring/prometheus.yaml + # create prometheus-service + kubectl create -f https://raw.githubusercontent.com/rook/rook/master/cluster/examples/kubernetes/ceph/monitoring/prometheus-service.yaml + # waitng for prometheus-rook-prometheus-0 pod to get ready + timeout 60 sh -c "until [ $(kubectl get pod -l prometheus=rook-prometheus -nrook-ceph -o json | jq -r '.items[0].status.phase') = Running ]; do echo 'waiting for prometheus-rook-prometheus-0 pod to get created' && sleep 1; done" + if [ "$(kubectl get pod -l prometheus=rook-prometheus -nrook-ceph)" == "" ] + then + echo "prometheus-rook-prometheus-0 pod not created, re-run the script" + exit 1 + fi + echo "Prerequisites deployments created" +} + +function invalidCall(){ + echo " $0 [command] +Available Commands for normal cluster: + ./auto-grow-storage.sh count --max maxCount --count rate Scale horizontally by adding more OSDs to the cluster + ./auto-grow-storage.sh size --max maxSize --growth-rate percent Scale vertically by increasing the size of existing OSDs +" >&2 +} + +case "${1:-}" in +count) + if [[ $# -ne 5 ]]; then + echo "incorrect command to run the script" + invalidCall + exit 1 + fi + max=$3 + count=$5 + if ! [[ "${max}" =~ ^[0-9]+$ ]] + then + echo "maxCount should be an integer" + invalidCall + exit 1 + fi + if ! [[ "${count}" =~ ^[0-9]+$ ]] + then + echo "rate should be an integer" + invalidCall + exit 1 + fi + creatingPrerequisites + echo "Adding on nearfull and full alert and number of OSD to add is ${count}" + growOSD count "${count}" "${max}" + ;; +size) + if [[ $# -ne 5 ]]; then + echo "incorrect command to run the script" + invalidCall + exit 1 + fi + max=$3 + growRate=$5 + if [[ "${max}" =~ ^[0-9]+$ ]] + then + echo "maxSize should be an string" + invalidCall + exit 1 + fi + if ! [[ "${growRate}" =~ ^[0-9]+$ ]] + then + echo "growth-rate should be an integer" + invalidCall + exit 1 + fi + creatingPrerequisites + echo "Resizing on nearfull and full alert and Expansion percentage set to ${growRate}%" + growOSD size "${growRate}" "${max}" + ;; +*) + invalidCall + ;; +esac