From 46d1483468a535853608c6a902aa5c322b0d5663 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Fri, 27 Oct 2017 11:00:36 -0700 Subject: [PATCH] tools: remove dependence on api service to launch toolbox --- cluster/examples/kubernetes/rook-tools.yaml | 9 ++- cmd/rookctl/rook/rook.go | 16 ++-- images/toolbox/toolbox.sh | 82 ++++++++++++--------- 3 files changed, 62 insertions(+), 45 deletions(-) diff --git a/cluster/examples/kubernetes/rook-tools.yaml b/cluster/examples/kubernetes/rook-tools.yaml index fcde0f7d9..fe4bcfe4d 100644 --- a/cluster/examples/kubernetes/rook-tools.yaml +++ b/cluster/examples/kubernetes/rook-tools.yaml @@ -9,7 +9,6 @@ spec: - name: rook-tools image: rook/toolbox:master imagePullPolicy: IfNotPresent - args: ["sleep", "36500d"] env: - name: ROOK_ADMIN_SECRET valueFrom: @@ -25,6 +24,8 @@ spec: name: sysbus - mountPath: /lib/modules name: libmodules + - name: mon-endpoint-volume + mountPath: /etc/rook hostNetwork: false volumes: - name: dev @@ -36,3 +37,9 @@ spec: - name: libmodules hostPath: path: /lib/modules + - name: mon-endpoint-volume + configMap: + name: rook-ceph-mon-endpoints + items: + - key: data + path: mon-endpoints diff --git a/cmd/rookctl/rook/rook.go b/cmd/rookctl/rook/rook.go index 040a3fa78..d9d15faf0 100644 --- a/cmd/rookctl/rook/rook.go +++ b/cmd/rookctl/rook/rook.go @@ -20,12 +20,10 @@ import ( "io" "net/http" "os" - "strconv" "text/tabwriter" "time" "github.com/coreos/pkg/capnslog" - "github.com/rook/rook/pkg/model" "github.com/rook/rook/pkg/rook/client" "github.com/rook/rook/pkg/util/flags" "github.com/spf13/cobra" @@ -52,15 +50,13 @@ https://github.com/rook/rook`, } func init() { - defaultHost := os.Getenv("ROOK_API_SERVICE_HOST") - if defaultHost == "" { - defaultHost = "127.0.0.1" + host := os.Getenv("ROOK_API_SERVICE_HOST") + port := os.Getenv("ROOK_API_SERVICE_PORT") + defaultEndpoint := fmt.Sprintf("%s:%s", host, port) + if host == "" || port == "" { + fmt.Println("$ROOK_API_SERVICE_HOST or $ROOK_API_SERVICE_PORT are not set. Check if the rook-api service is running.") + defaultEndpoint = "" } - defaultPort := os.Getenv("ROOK_API_SERVICE_PORT") - if defaultPort == "" { - defaultPort = strconv.Itoa(model.Port) - } - defaultEndpoint := fmt.Sprintf("%s:%s", defaultHost, defaultPort) RootCmd.PersistentFlags().StringVar(&APIServerEndpoint, "api-server-endpoint", defaultEndpoint, "IP endpoint of API server instance (required)") RootCmd.PersistentFlags().StringVar(&logLevelRaw, "log-level", "WARNING", "logging level for logging/tracing output (valid values: CRITICAL,ERROR,WARNING,NOTICE,INFO,DEBUG,TRACE)") diff --git a/images/toolbox/toolbox.sh b/images/toolbox/toolbox.sh index fdbbe5f8f..f4d801796 100755 --- a/images/toolbox/toolbox.sh +++ b/images/toolbox/toolbox.sh @@ -21,50 +21,64 @@ fi ROOK_DIR="/var/lib/rook" CEPH_CONFIG="/etc/ceph/ceph.conf" +MON_CONFIG="/etc/rook/mon-endpoints" +KEYRING_FILE="/etc/ceph/keyring" -# attempt to create a ceph config file in its default location so ceph/rados tools can be used +# create a ceph config file in its default location so ceph/rados tools can be used # without specifying any arguments -if [[ -d ${ROOK_DIR} ]]; then - # there is a rook directory, try to find one of rook's ceph config files and copy it - ROOK_CONFIG=$(find ${ROOK_DIR} -regex '.*mon[0-9]+/.*\.config' | head -1) - if [[ ! -z ${ROOK_CONFIG} ]]; then - cp ${ROOK_CONFIG} ${CEPH_CONFIG} - fi -else - if [[ -z ${ROOK_API_SERVER_ENDPOINT} ]]; then - if [[ ! -z ${ROOK_API_SERVICE_HOST} ]] && [[ ! -z ${ROOK_API_SERVICE_PORT} ]]; then - # we have some rook env vars set, get client info from the rook API server and construct - # a ceph config file from that - ROOK_API_SERVER_ENDPOINT=${ROOK_API_SERVICE_HOST}:${ROOK_API_SERVICE_PORT} - else - # default to the dns name of the rook-api svc - ROOK_API_SERVER_ENDPOINT="rook-api:8124" - fi - fi - - # append to the bashrc file so that the rook tool can find the API server easily - cat <> ~/.bashrc - -export ROOK_API_SERVER_ENDPOINT=${ROOK_API_SERVER_ENDPOINT} - -EOF - - CLIENT_INFO=$(curl -s http://${ROOK_API_SERVER_ENDPOINT}/client) - MON_ENDPOINTS=$(echo $CLIENT_INFO | jq -r '.monAddresses[]' | awk -F '/' 'BEGIN { ORS="," }; {print $1}' | sed 's/,$//') - KEYRING_FILE="/etc/ceph/keyring" - +write_endpoints() { + endpoints=$(cat ${MON_CONFIG}) + mon_endpoints=$(echo ${endpoints} | sed 's/rook-ceph-mon[0-9]=//g') + DATE=$(date) + echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}" cat < ${CEPH_CONFIG} [global] -mon_host = ${MON_ENDPOINTS} +mon_host = ${mon_endpoints} [client.admin] keyring = ${KEYRING_FILE} EOF +} - cat < ${KEYRING_FILE} +# watch the endpoints config file and update if the mon endpoints ever change +watch_endpoints() { + # get the timestamp for the target of the soft link + real_path=$(realpath ${MON_CONFIG}) + initial_time=$(stat -c %Z ${real_path}) + while true; do + real_path=$(realpath ${MON_CONFIG}) + latest_time=$(stat -c %Z ${real_path}) + + if [[ "${latest_time}" != "${initial_time}" ]]; then + write_endpoints + initial_time=${latest_time} + fi + sleep 10 + done +} + +if [[ -z ${ROOK_API_SERVER_ENDPOINT} ]]; then + if [[ ! -z ${ROOK_API_SERVICE_HOST} ]] && [[ ! -z ${ROOK_API_SERVICE_PORT} ]]; then + # we have some rook env vars set, get client info from the rook API server and construct + # a ceph config file from that + ROOK_API_SERVER_ENDPOINT=${ROOK_API_SERVICE_HOST}:${ROOK_API_SERVICE_PORT} + fi +fi + +# append to the bashrc file so that the rook tool can find the API server easily +cat <> ~/.bashrc + +export ROOK_API_SERVER_ENDPOINT=${ROOK_API_SERVER_ENDPOINT} +EOF + +# create the keyring file +cat < ${KEYRING_FILE} [client.admin] key = ${ROOK_ADMIN_SECRET} EOF -fi -exec ${ARGS} +# write the initial config file +write_endpoints + +# continuously update the mon endpoints if they fail over +watch_endpoints