forked from rook/rook
Oded Viner
5d143d990c
this PR updates the Prometheus Operator URL references from version v0.71.1 to the latest release v0.81.0 in documentation and integration test scripts. This ensures we are aligned with the latest features and improvements from the Prometheus Operator project. Signed-off-by: Oded Viner <oviner@redhat.com>
264 lines
9.8 KiB
Bash
Executable File
264 lines
9.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
#############
|
|
# FUNCTIONS #
|
|
#############
|
|
|
|
function calculateSize() {
|
|
local currentsize=$2
|
|
local unit=$1
|
|
rawsizeValue=0 # rawsizeValue is a global variable
|
|
|
|
if [[ "$currentsize" == *"Mi" ]]
|
|
then
|
|
rawSize=${currentsize//Mi} # rawSize is a global variable
|
|
unitSize="Mi"
|
|
rawsizeValue=$rawSize
|
|
elif [[ "$currentsize" == *"Gi" ]]
|
|
then
|
|
rawSize=${currentsize//Gi}
|
|
unitSize="Gi"
|
|
rawsizeValue=$(( rawSize * 1000 ))
|
|
elif [[ "$currentsize" == *"Ti" ]]
|
|
then
|
|
rawSize=${currentsize//Ti}
|
|
unitSize="Ti"
|
|
rawsizeValue=$(( rawSize * 1000000 ))
|
|
else
|
|
echo "Unknown unit of $unit : ${currentsize}"
|
|
echo "Supported units are 'Mi','Gi','Ti'"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
function compareSizes() {
|
|
local newsize=$1
|
|
local maxsize=$2
|
|
calculateSize newsize "${newsize}" # rawsizeValue is calculated and used for further process
|
|
local newsize=$rawsizeValue
|
|
calculateSize maxsize "${maxsize}"
|
|
local maxsize=$rawsizeValue
|
|
if [ "${newsize}" -ge "${maxsize}" ]
|
|
then
|
|
return "1"
|
|
fi
|
|
return "0"
|
|
}
|
|
|
|
function growVertically() {
|
|
local growRate=$1
|
|
local pvc=$2
|
|
local ns=$3
|
|
local maxSize=$4
|
|
local currentSize
|
|
currentSize=$(kubectl get pvc "${pvc}" -n "${ns}" -o json | jq -r '.spec.resources.requests.storage')
|
|
echo "PVC(OSD) current size is ${currentSize} and will be increased by ${growRate}%."
|
|
|
|
calculateSize "${pvc}" "${currentSize}" # rawSize is calculated and used for further process
|
|
|
|
if ! [[ "${rawSize}" =~ ^[0-9]+$ ]]
|
|
then
|
|
echo "disk size should be an integer"
|
|
else
|
|
newSize=$(echo "${rawSize}+(${rawSize} * ${growRate})/100" | bc | cut -f1 -d'.')
|
|
if [ "${newSize}" = "${rawSize}" ]
|
|
then
|
|
newSize=$(( rawSize + 1 ))
|
|
echo "New adjusted calculated size for the PVC is ${newSize}${unitSize}"
|
|
else
|
|
echo "New calculated size for the PVC is ${newSize}${unitSize}"
|
|
fi
|
|
|
|
compareSizes ${newSize}${unitSize} "${maxSize}"
|
|
if [ "1" = $? ]
|
|
then
|
|
newSize=${maxSize}
|
|
echo "Disk has reached it's MAX capacity ${maxSize}, add a new disk to it"
|
|
result=$(kubectl patch pvc "${pvc}" -n "${ns}" --type json --patch "[{ op: replace, path: /spec/resources/requests/storage, value: ${newSize} }]")
|
|
else
|
|
result=$(kubectl patch pvc "${pvc}" -n "${ns}" --type json --patch "[{ op: replace, path: /spec/resources/requests/storage, value: ${newSize}${unitSize} }]")
|
|
fi
|
|
echo "${result}"
|
|
fi
|
|
}
|
|
|
|
function growHorizontally() {
|
|
local increaseOSDCount=$1
|
|
local pvc=$2
|
|
local ns=$3
|
|
local maxOSDCount=$4
|
|
local deviceSetName
|
|
local cluster=""
|
|
local deviceSet=""
|
|
local currentOSDCount=0
|
|
local clusterCount=0
|
|
local deviceSetCount=0
|
|
deviceSetName=$(kubectl get pvc "${pvc}" -n "${ns}" -o json | jq -r '.metadata.labels."ceph.rook.io/DeviceSet"')
|
|
while [ "$cluster" != "null" ]
|
|
do
|
|
cluster=$(kubectl get CephCluster -n "${ns}" -o json | jq -r ".items[${clusterCount}]")
|
|
while [ "$deviceSet" != "null" ]
|
|
do
|
|
deviceSet=$(kubectl get CephCluster -n "${ns}" -o json | jq -r ".items[${clusterCount}].spec.storage.storageClassDeviceSets[${deviceSetCount}].name")
|
|
if [[ $deviceSet == "${deviceSetName}" ]]
|
|
then
|
|
currentOSDCount=$(kubectl get CephCluster -n "${ns}" -o json | jq -r ".items[${clusterCount}].spec.storage.storageClassDeviceSets[${deviceSetCount}].count")
|
|
finalCount=$(( "${currentOSDCount}" + "${increaseOSDCount}" ))
|
|
echo "OSD count: ${currentOSDCount}. OSD count will be increased by ${increaseOSDCount}."
|
|
if [ "${finalCount}" -ge "${maxOSDCount}" ]
|
|
then
|
|
finalCount=${maxOSDCount}
|
|
echo "DeviceSet ${deviceSet} capacity is full, cannot add more OSD to it"
|
|
fi
|
|
echo "Total count of OSDs for deviceset ${deviceSetName} is set to ${finalCount}."
|
|
clusterName=$(kubectl get CephCluster -n "${ns}" -o json | jq -r ".items[${clusterCount}].metadata.name" )
|
|
result=$(kubectl patch CephCluster "${clusterName}" -n "${ns}" --type json --patch "[{ op: replace, path: /spec/storage/storageClassDeviceSets/${deviceSetCount}/count, value: ${finalCount} }]")
|
|
echo "${result}"
|
|
break
|
|
fi
|
|
deviceSetCount=$((deviceSetCount+1))
|
|
deviceSet=$(kubectl get CephCluster -n "${ns}" -o json | jq -r ".items[${clusterCount}].spec.storage.storageClassDeviceSets[${deviceSetCount}].name")
|
|
done
|
|
clusterCount=$((clusterCount+1))
|
|
cluster=$(kubectl get CephCluster -n "${ns}" -o json | jq -r ".items[${clusterCount}]")
|
|
done
|
|
}
|
|
|
|
function growOSD(){
|
|
itr=0
|
|
alertmanagerroute=$(kubectl -n rook-ceph -o jsonpath="{.status.hostIP}" get pod prometheus-rook-prometheus-0)
|
|
route=${alertmanagerroute}:30900
|
|
toolbox=$(kubectl get pods -n rook-ceph | grep -i rook-ceph-tools | awk '{ print $1 }')
|
|
alerts=$(kubectl exec -it "${toolbox}" -n rook-ceph -- bash -c "curl -s http://${route}/api/v1/alerts")
|
|
export total_alerts
|
|
total_alerts=$( jq '.data.alerts | length' <<< "${alerts}")
|
|
echo "Looping at $(date +"%Y-%m-%d %H:%M:%S")"
|
|
|
|
while true
|
|
do
|
|
if [ "${total_alerts}" == "" ]
|
|
then
|
|
echo "Alert manager not configured,re-run the script"
|
|
exit 1
|
|
fi
|
|
export entry
|
|
entry=$( jq ".data.alerts[$itr]" <<< "${alerts}")
|
|
thename=$(echo "${entry}" | jq -r '.labels.alertname')
|
|
if [ "${thename}" = "CephOSDNearFull" ] || [ "${thename}" = "CephOSDCriticallyFull" ]
|
|
then
|
|
echo "${entry}"
|
|
ns=$(echo "${entry}" | jq -r '.labels.namespace')
|
|
osdID=$(echo "${entry}" | jq -r '.labels.ceph_daemon')
|
|
osdID=${osdID/./-}
|
|
pvc=$(kubectl get deployment -n "${ns}" rook-ceph-"${osdID}" -o json | jq -r '.metadata.labels."ceph.rook.io/pvc"')
|
|
if [[ $pvc == null ]]
|
|
then
|
|
echo "PVC not found, script can only run on PVC-based cluster"
|
|
exit 1
|
|
fi
|
|
echo "Processing NearFull or Full alert for PVC ${pvc} in namespace ${ns}"
|
|
if [[ $1 == "count" ]]
|
|
then
|
|
growHorizontally "$2" "${pvc}" "${ns}" "$3"
|
|
else
|
|
growVertically "$2" "${pvc}" "${ns}" "$3"
|
|
fi
|
|
fi
|
|
(( itr = itr + 1 ))
|
|
if [[ "${itr}" == "${total_alerts}" ]] || [[ "${total_alerts}" == "0" ]]
|
|
then
|
|
sleep 600
|
|
alerts=$(kubectl exec -it "${toolbox}" -n rook-ceph -- bash -c "curl -s http://${route}/api/v1/alerts")
|
|
total_alerts=$( jq '.data.alerts | length' <<< "${alerts}")
|
|
itr=0
|
|
echo "Looping at $(date +"%Y-%m-%d %H:%M:%S")"
|
|
fi
|
|
done
|
|
}
|
|
|
|
function creatingPrerequisites(){
|
|
echo "creating Prerequisites deployments - Prometheus Operator and Prometheus Instances"
|
|
# creating Prometheus operator
|
|
kubectl create -f https://raw.githubusercontent.com/coreos/prometheus-operator/v0.82.0/bundle.yaml
|
|
# waiting for Prometheus operator to get ready
|
|
timeout 30 sh -c "until [ $(kubectl get pod -l app.kubernetes.'io/name'=prometheus-operator -o json | jq -r '.items[0].status.phase') = Running ]; do echo 'waiting for prometheus-operator to get created' && sleep 1; done"
|
|
# creating a service monitor that will watch the Rook cluster and collect metrics regularly
|
|
kubectl create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/monitoring/service-monitor.yaml
|
|
# create the PrometheusRule for Rook alerts.
|
|
kubectl create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/monitoring/prometheus-ceph-v14-rules.yaml
|
|
# create prometheus-rook-prometheus-0 pod
|
|
kubectl create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/monitoring/prometheus.yaml
|
|
# create prometheus-service
|
|
kubectl create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/monitoring/prometheus-service.yaml
|
|
# waiting for prometheus-rook-prometheus-0 pod to get ready
|
|
timeout 60 sh -c "until [ $(kubectl get pod -l prometheus=rook-prometheus -nrook-ceph -o json | jq -r '.items[0].status.phase') = Running ]; do echo 'waiting for prometheus-rook-prometheus-0 pod to get created' && sleep 1; done"
|
|
if [ "$(kubectl get pod -l prometheus=rook-prometheus -nrook-ceph)" == "" ]
|
|
then
|
|
echo "prometheus-rook-prometheus-0 pod not created, re-run the script"
|
|
exit 1
|
|
fi
|
|
echo "Prerequisites deployments created"
|
|
}
|
|
|
|
function invalidCall(){
|
|
echo " $0 [command]
|
|
Available Commands for normal cluster:
|
|
./auto-grow-storage.sh count --max maxCount --count rate Scale horizontally by adding more OSDs to the cluster
|
|
./auto-grow-storage.sh size --max maxSize --growth-rate percent Scale vertically by increasing the size of existing OSDs
|
|
" >&2
|
|
}
|
|
|
|
case "${1:-}" in
|
|
count)
|
|
if [[ $# -ne 5 ]]; then
|
|
echo "incorrect command to run the script"
|
|
invalidCall
|
|
exit 1
|
|
fi
|
|
max=$3
|
|
count=$5
|
|
if ! [[ "${max}" =~ ^[0-9]+$ ]]
|
|
then
|
|
echo "maxCount should be an integer"
|
|
invalidCall
|
|
exit 1
|
|
fi
|
|
if ! [[ "${count}" =~ ^[0-9]+$ ]]
|
|
then
|
|
echo "rate should be an integer"
|
|
invalidCall
|
|
exit 1
|
|
fi
|
|
creatingPrerequisites
|
|
echo "Adding on nearfull and full alert and number of OSD to add is ${count}"
|
|
growOSD count "${count}" "${max}"
|
|
;;
|
|
size)
|
|
if [[ $# -ne 5 ]]; then
|
|
echo "incorrect command to run the script"
|
|
invalidCall
|
|
exit 1
|
|
fi
|
|
max=$3
|
|
growRate=$5
|
|
if [[ "${max}" =~ ^[0-9]+$ ]]
|
|
then
|
|
echo "maxSize should be an string"
|
|
invalidCall
|
|
exit 1
|
|
fi
|
|
if ! [[ "${growRate}" =~ ^[0-9]+$ ]]
|
|
then
|
|
echo "growth-rate should be an integer"
|
|
invalidCall
|
|
exit 1
|
|
fi
|
|
creatingPrerequisites
|
|
echo "Resizing on nearfull and full alert and Expansion percentage set to ${growRate}%"
|
|
growOSD size "${growRate}" "${max}"
|
|
;;
|
|
*)
|
|
invalidCall
|
|
;;
|
|
esac
|