Merge pull request #1123 from travisn/launch-tools

Remove dependence on api service to launch toolbox

cluster/examples/kubernetes/rook-tools.yaml

@@ -9,7 +9,6 @@ spec:
   - name: rook-tools
     image: rook/toolbox:master
     imagePullPolicy: IfNotPresent
-    args: ["sleep", "36500d"]
     env:
       - name: ROOK_ADMIN_SECRET
         valueFrom:
@@ -25,6 +24,8 @@ spec:
         name: sysbus
       - mountPath: /lib/modules
         name: libmodules
+      - name: mon-endpoint-volume
+        mountPath: /etc/rook
   hostNetwork: false
   volumes:
     - name: dev
@@ -36,3 +37,9 @@ spec:
     - name: libmodules
       hostPath:
         path: /lib/modules
+    - name: mon-endpoint-volume
+      configMap:
+        name: rook-ceph-mon-endpoints
+        items:
+        - key: data
+          path: mon-endpoints

cmd/rookctl/rook/rook.go

@@ -20,12 +20,10 @@ import (
 	"io"
 	"net/http"
 	"os"
-	"strconv"
 	"text/tabwriter"
 	"time"
 
 	"github.com/coreos/pkg/capnslog"
-	"github.com/rook/rook/pkg/model"
 	"github.com/rook/rook/pkg/rook/client"
 	"github.com/rook/rook/pkg/util/flags"
 	"github.com/spf13/cobra"
@@ -52,15 +50,13 @@ https://github.com/rook/rook`,
 }
 
 func init() {
-	defaultHost := os.Getenv("ROOK_API_SERVICE_HOST")
-	if defaultHost == "" {
-		defaultHost = "127.0.0.1"
+	host := os.Getenv("ROOK_API_SERVICE_HOST")
+	port := os.Getenv("ROOK_API_SERVICE_PORT")
+	defaultEndpoint := fmt.Sprintf("%s:%s", host, port)
+	if host == "" || port == "" {
+		fmt.Println("$ROOK_API_SERVICE_HOST or $ROOK_API_SERVICE_PORT are not set. Check if the rook-api service is running.")
+		defaultEndpoint = ""
 	}
-	defaultPort := os.Getenv("ROOK_API_SERVICE_PORT")
-	if defaultPort == "" {
-		defaultPort = strconv.Itoa(model.Port)
-	}
-	defaultEndpoint := fmt.Sprintf("%s:%s", defaultHost, defaultPort)
 
 	RootCmd.PersistentFlags().StringVar(&APIServerEndpoint, "api-server-endpoint", defaultEndpoint, "IP endpoint of API server instance (required)")
 	RootCmd.PersistentFlags().StringVar(&logLevelRaw, "log-level", "WARNING", "logging level for logging/tracing output (valid values: CRITICAL,ERROR,WARNING,NOTICE,INFO,DEBUG,TRACE)")

images/toolbox/toolbox.sh

@@ -21,50 +21,64 @@ fi
 
 ROOK_DIR="/var/lib/rook"
 CEPH_CONFIG="/etc/ceph/ceph.conf"
+MON_CONFIG="/etc/rook/mon-endpoints"
+KEYRING_FILE="/etc/ceph/keyring"
 
-# attempt to create a ceph config file in its default location so ceph/rados tools can be used
+# create a ceph config file in its default location so ceph/rados tools can be used
 # without specifying any arguments
-if [[ -d ${ROOK_DIR} ]]; then
-    # there is a rook directory, try to find one of rook's ceph config files and copy it
-    ROOK_CONFIG=$(find ${ROOK_DIR} -regex '.*mon[0-9]+/.*\.config' | head -1)
-    if [[ ! -z ${ROOK_CONFIG} ]]; then
-        cp ${ROOK_CONFIG} ${CEPH_CONFIG}
-    fi
-else
-    if [[ -z ${ROOK_API_SERVER_ENDPOINT} ]]; then
-        if [[ ! -z ${ROOK_API_SERVICE_HOST} ]] && [[ ! -z ${ROOK_API_SERVICE_PORT} ]]; then
-            # we have some rook env vars set, get client info from the rook API server and construct
-            # a ceph config file from that
-            ROOK_API_SERVER_ENDPOINT=${ROOK_API_SERVICE_HOST}:${ROOK_API_SERVICE_PORT}
-        else
-            # default to the dns name of the rook-api svc
-            ROOK_API_SERVER_ENDPOINT="rook-api:8124"
-        fi
-    fi
-
-    # append to the bashrc file so that the rook tool can find the API server easily
-    cat <<EOF >> ~/.bashrc
-
-export ROOK_API_SERVER_ENDPOINT=${ROOK_API_SERVER_ENDPOINT}
-
-EOF
-
-    CLIENT_INFO=$(curl -s http://${ROOK_API_SERVER_ENDPOINT}/client)
-    MON_ENDPOINTS=$(echo $CLIENT_INFO | jq -r '.monAddresses[]' | awk -F '/' 'BEGIN { ORS="," }; {print $1}' | sed 's/,$//')
-    KEYRING_FILE="/etc/ceph/keyring"
-
+write_endpoints() {  
+    endpoints=$(cat ${MON_CONFIG})
+    mon_endpoints=$(echo ${endpoints} | sed 's/rook-ceph-mon[0-9]=//g')
+    DATE=$(date)
+    echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}"
     cat <<EOF > ${CEPH_CONFIG}
 [global]
-mon_host = ${MON_ENDPOINTS}
+mon_host = ${mon_endpoints}
 
 [client.admin]
 keyring = ${KEYRING_FILE}
 EOF
+}
 
-    cat <<EOF > ${KEYRING_FILE}
+# watch the endpoints config file and update if the mon endpoints ever change
+watch_endpoints() {
+    # get the timestamp for the target of the soft link
+    real_path=$(realpath ${MON_CONFIG})
+    initial_time=$(stat -c %Z ${real_path})
+    while true; do
+       real_path=$(realpath ${MON_CONFIG})
+       latest_time=$(stat -c %Z ${real_path})
+
+       if [[ "${latest_time}" != "${initial_time}" ]]; then
+         write_endpoints
+         initial_time=${latest_time}
+       fi
+       sleep 10
+    done
+}
+
+if [[ -z ${ROOK_API_SERVER_ENDPOINT} ]]; then
+    if [[ ! -z ${ROOK_API_SERVICE_HOST} ]] && [[ ! -z ${ROOK_API_SERVICE_PORT} ]]; then
+        # we have some rook env vars set, get client info from the rook API server and construct
+        # a ceph config file from that
+        ROOK_API_SERVER_ENDPOINT=${ROOK_API_SERVICE_HOST}:${ROOK_API_SERVICE_PORT}
+    fi
+fi
+
+# append to the bashrc file so that the rook tool can find the API server easily
+cat <<EOF >> ~/.bashrc
+
+export ROOK_API_SERVER_ENDPOINT=${ROOK_API_SERVER_ENDPOINT}
+EOF
+
+# create the keyring file
+cat <<EOF > ${KEYRING_FILE}
 [client.admin]
 key = ${ROOK_ADMIN_SECRET}
 EOF
-fi
 
-exec ${ARGS}
+# write the initial config file
+write_endpoints
+
+# continuously update the mon endpoints if they fail over
+watch_endpoints