my-rook-config/tests/scripts/create-dev-cluster.sh

#!/usr/bin/env bash

DEFAULT_NS="rook-ceph"
CLUSTER_FILES="common.yaml operator.yaml cluster-test.yaml cluster-on-pvc-minikube.yaml dashboard-external-http.yaml toolbox.yaml"
MONITORING_FILES="monitoring/prometheus.yaml monitoring/service-monitor.yaml monitoring/exporter-service-monitor.yaml monitoring/prometheus-service.yaml monitoring/rbac.yaml"
SCRIPT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)

# Script arguments: new arguments must be added here (following the same format)
export MINIKUBE_NODES="${MINIKUBE_NODES:=1}" ## Specify the minikube number of nodes to create
export MINIKUBE_DISK_SIZE="${MINIKUBE_DISK_SIZE:=40g}" ## Specify the minikube disk size
export MINIKUBE_EXTRA_DISKS="${MINIKUBE_EXTRA_DISKS:=3}" ## Specify the minikube number of extra disks
export ROOK_PROFILE_NAME="${ROOK_PROFILE_NAME:=rook}" ## Specify the minikube profile name
export ROOK_CLUSTER_NS="${ROOK_CLUSTER_NS:=$DEFAULT_NS}" ## CephCluster namespace
export ROOK_OPERATOR_NS="${ROOK_OPERATOR_NS:=$DEFAULT_NS}" ## Rook operator namespace (if different from CephCluster namespace)
export ROOK_EXAMPLES_DIR="${ROOK_EXAMPLES_DIR:="$SCRIPT_ROOT"/../../deploy/examples}" ## Path to Rook examples directory (i.e github.com/rook/rook/deploy/examples)
export ROOK_CLUSTER_SPEC_FILE="${ROOK_CLUSTER_SPEC_FILE:=cluster-test.yaml}" ## CephCluster manifest file

init_vars(){
    MINIKUBE="minikube --profile $ROOK_PROFILE_NAME"
    KUBECTL="$MINIKUBE kubectl --"

    echo "Using '$(realpath "$ROOK_EXAMPLES_DIR")' as examples directory.."
    echo "Using '$ROOK_CLUSTER_SPEC_FILE' as cluster spec file.."
    echo "Using '$ROOK_PROFILE_NAME' as minikube profile.."
    echo "Using '$ROOK_CLUSTER_NS' as cluster namespace.."
    echo "Using '$ROOK_OPERATOR_NS' as operator namespace.."
}

update_namespaces() {
    if [ "$ROOK_CLUSTER_NS" != "$DEFAULT_NS" ] || [ "$ROOK_OPERATOR_NS" != "$DEFAULT_NS" ]; then
	for file in $CLUSTER_FILES $MONITORING_FILES; do
	    echo "Updating namespace on $file"
	    sed -i.bak \
		-e "s/\(.*\):.*# namespace:operator/\1: $ROOK_OPERATOR_NS # namespace:operator/g" \
		-e "s/\(.*\):.*# namespace:cluster/\1: $ROOK_CLUSTER_NS # namespace:cluster/g" \
		-e "s/\(.*serviceaccount\):.*:\(.*\) # serviceaccount:namespace:operator/\1:$ROOK_OPERATOR_NS:\2 # serviceaccount:namespace:operator/g" \
		-e "s/\(.*serviceaccount\):.*:\(.*\) # serviceaccount:namespace:cluster/\1:$ROOK_CLUSTER_NS:\2 # serviceaccount:namespace:cluster/g" \
		-e "s/\(.*\): [-_A-Za-z0-9]*\.\(.*\) # csi-provisioner-name/\1: $ROOK_OPERATOR_NS.\2 # csi-provisioner-name/g" \
		-e "s/\(.*\): [-_A-Za-z0-9]*\.\(.*\) # driver:namespace:cluster/\1: $ROOK_CLUSTER_NS.\2 # driver:namespace:cluster/g" \
		"$file"
	done
    fi
}

wait_for_ceph_cluster() {
    echo "Waiting for ceph cluster to enter HEALTH_OK"
    WAIT_CEPH_CLUSTER_RUNNING=20
    while ! $KUBECTL get cephclusters.ceph.rook.io -n "$ROOK_CLUSTER_NS" -o jsonpath='{.items[?(@.kind == "CephCluster")].status.ceph.health}' | grep -q "HEALTH_OK"; do
	echo "Waiting for Ceph cluster to enter HEALTH_OK"
	sleep ${WAIT_CEPH_CLUSTER_RUNNING}
    done
    echo "Ceph cluster installed and running"
}

get_minikube_driver() {
    os=$(uname)
    architecture=$(uname -m)
    if [[ "$os" == "Darwin" ]]; then
        if [[ "$architecture" == "x86_64" ]]; then
            echo "hyperkit"
        elif [[ "$architecture" == "arm64" ]]; then
            echo "qemu"
        else
            echo "Unknown Architecture on Apple OS"
	    exit 1
        fi
    elif [[ "$os" == "Linux" ]]; then
        echo "kvm2"
    else
        echo "Unknown/Unsupported OS"
	exit 1
    fi
}

show_info() {
    local monitoring_enabled=$1
    DASHBOARD_PASSWORD=$($KUBECTL -n "$ROOK_CLUSTER_NS" get secret rook-ceph-dashboard-password -o jsonpath="{['data']['password']}" | base64 --decode && echo)
    DASHBOARD_END_POINT=$($MINIKUBE service rook-ceph-mgr-dashboard-external-http -n "$ROOK_CLUSTER_NS" --url)
    BASE_URL="$DASHBOARD_END_POINT"
    echo "==========================="
    echo "Ceph Dashboard:"
    echo "   IP_ADDR  : $BASE_URL"
    echo "   USER     : admin"
    echo "   PASSWORD : $DASHBOARD_PASSWORD"
    if [ "$monitoring_enabled" = true ]; then
	PROMETHEUS_API_HOST="http://$(kubectl -n "$ROOK_CLUSTER_NS" -o jsonpath='{.status.hostIP}' get pod prometheus-rook-prometheus-0):30900"
    echo "Prometheus Dashboard: "
    echo "   API_HOST: $PROMETHEUS_API_HOST"
    fi
    echo "==========================="
    echo " "
    echo " *** To start using your rook cluster please set the following env: "
    echo " "
    echo "   > eval \$($MINIKUBE docker-env)"
    echo "   > alias kubectl=\"$KUBECTL"\"
    echo " "
    echo " *** To access the new cluster with k9s: "
    echo " "
    echo "   > k9s --context $ROOK_PROFILE_NAME"
    echo " "
}

check_minikube_exists() {
    echo "Checking minikube profile '$ROOK_PROFILE_NAME'..."
    if minikube profile list -l 2> /dev/null | grep -qE "\s$ROOK_PROFILE_NAME\s"; then
        echo "A minikube profile '$ROOK_PROFILE_NAME' already exists, please use -f to force the cluster creation."
	exit 1
    fi
}

setup_minikube_env() {
    minikube_driver="$(get_minikube_driver)"
    echo "Setting up minikube env for profile '$ROOK_PROFILE_NAME' (using $minikube_driver driver)"
    $MINIKUBE delete
    $MINIKUBE start --disk-size="$MINIKUBE_DISK_SIZE" --extra-disks="$MINIKUBE_EXTRA_DISKS" --driver "$minikube_driver" -n "$MINIKUBE_NODES" $ROOK_MINIKUBE_EXTRA_ARGS
    eval "$($MINIKUBE docker-env)"
}

create_rook_cluster() {
    echo "Creating cluster"
    # create operator namespace if it doesn't exist
    if ! kubectl get namespace "$ROOK_OPERATOR_NS" &> /dev/null; then
	kubectl create namespace "$ROOK_OPERATOR_NS"
    fi
    $KUBECTL apply -f crds.yaml -f common.yaml -f operator.yaml
    $KUBECTL apply -f "$ROOK_CLUSTER_SPEC_FILE" -f toolbox.yaml
    $KUBECTL apply -f dashboard-external-http.yaml
}

change_to_examples_dir() {
    if [ ! -e "$ROOK_EXAMPLES_DIR" ]; then
	echo "Examples directory '$ROOK_EXAMPLES_DIR' does not exist. Please, provide a valid rook examples directory."
	exit 1
    fi

    CRDS_FILE_PATH=$(realpath "$ROOK_EXAMPLES_DIR/crds.yaml")
    if [ ! -e "$CRDS_FILE_PATH" ]; then
	echo "File '$CRDS_FILE_PATH' does not exist. Please, provide a valid rook examples directory."
	exit 1
    fi

    ROOK_CLUSTER_SPEC_PATH=$(realpath "$ROOK_EXAMPLES_DIR/$ROOK_CLUSTER_SPEC_FILE")
    if [ ! -e "$ROOK_CLUSTER_SPEC_PATH" ]; then
	echo "File '$ROOK_CLUSTER_SPEC_PATH' does not exist. Please, provide a valid cluster spec file."
	exit 1
    fi

    cd "$ROOK_EXAMPLES_DIR" || exit
}

wait_for_rook_operator() {
    echo "Waiting for rook operator..."
    $KUBECTL rollout status deployment rook-ceph-operator -n "$ROOK_OPERATOR_NS" --timeout=180s
    while ! $KUBECTL get cephclusters.ceph.rook.io -n "$ROOK_CLUSTER_NS" -o jsonpath='{.items[?(@.kind == "CephCluster")].status.phase}' | grep -q "Ready"; do
	echo "Waiting for ceph cluster to become ready..."
	sleep 20
    done
}

enable_rook_orchestrator() {
    echo "Enabling rook orchestrator"
    $KUBECTL rollout status deployment rook-ceph-tools -n "$ROOK_CLUSTER_NS" --timeout=90s
    $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph mgr module enable rook
    $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph orch set backend rook
    $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph orch status
}

enable_monitoring() {
    echo "Enabling monitoring"
    $KUBECTL create -f https://raw.githubusercontent.com/coreos/prometheus-operator/v0.71.1/bundle.yaml
    $KUBECTL wait --for=condition=ready pod -l app.kubernetes.io/name=prometheus-operator --timeout=30s
    $KUBECTL apply -f monitoring/rbac.yaml
    $KUBECTL apply -f monitoring/service-monitor.yaml
    $KUBECTL apply -f monitoring/exporter-service-monitor.yaml
    $KUBECTL apply -f monitoring/prometheus.yaml
    $KUBECTL apply -f monitoring/prometheus-service.yaml
    PROMETHEUS_API_HOST="http://$(kubectl -n "$ROOK_CLUSTER_NS" -o jsonpath='{.status.hostIP}' get pod prometheus-rook-prometheus-0):30900"
    $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph dashboard set-prometheus-api-host "$PROMETHEUS_API_HOST"
}

show_usage() {
    echo ""
    echo "Usage: [ARG=VALUE]... $(basename "$0") [-f] [-r] [-m]"
    echo "  -f     Force cluster creation by deleting minikube profile"
    echo "  -r     Enable rook orchestrator"
    echo "  -m     Enable monitoring"
    echo "  Args:"
    sed -n -E "s/^export (.*)=\".*:=.*\" ## (.*)/    \1 (\\$\1):  \2/p;" "$SCRIPT_ROOT"/"$(basename "$0")" | envsubst
    echo ""
}

invocation_error() {
    printf "%s\n" "$*" > /dev/stderr
    show_usage
    exit 1
}

####################################################################
################# MAIN #############################################

while getopts ":hrmf" opt; do
    case $opt in
	h)
	    show_usage
	    exit 0
	    ;;
	r)
	    enable_rook=true
	    ;;
	m)
	    enable_monitoring=true
	    ;;
	f)
	    force_minikube=true
	    ;;
	\?)
	    invocation_error "Invalid option: -$OPTARG"
	    ;;
	:)
	    invocation_error "Option -$OPTARG requires an argument."
	    ;;
    esac
done

# initialization zone
init_vars
change_to_examples_dir
[ -z "$force_minikube" ] && check_minikube_exists
update_namespaces

# cluster creation zone
setup_minikube_env
create_rook_cluster
wait_for_rook_operator
wait_for_ceph_cluster

# final tweaks and ceph cluster tuning
[ "$enable_rook" = true ] && enable_rook_orchestrator
[ "$enable_monitoring" = true ] && enable_monitoring
show_info "$enable_monitoring"

####################################################################
####################################################################