From af240f22f8798e8346daa4b0ccdf777575db1bf8 Mon Sep 17 00:00:00 2001
From: hacor <hacornelis@gmail.com>
Date: Tue, 10 Apr 2018 11:58:09 +0200
Subject: [PATCH] Added ceph-reboot-script using the Container Linux Update
 Operator

[skip ci]

Signed-off-by: hacor <hacornelis@gmail.com>
---
 Documentation/container-linux.md              |  84 ++++++++++++
 .../coreos/after-reboot-daemonset.yaml        |  55 ++++++++
 .../coreos/before-reboot-daemonset.yaml       |  55 ++++++++
 .../coreos/ceph-after-reboot-script.yaml      | 127 ++++++++++++++++++
 .../coreos/ceph-before-reboot-script.yaml     | 120 +++++++++++++++++
 cluster/examples/coreos/rbac.yaml             |  29 ++++
 6 files changed, 470 insertions(+)
 create mode 100644 Documentation/container-linux.md
 create mode 100644 cluster/examples/coreos/after-reboot-daemonset.yaml
 create mode 100644 cluster/examples/coreos/before-reboot-daemonset.yaml
 create mode 100644 cluster/examples/coreos/ceph-after-reboot-script.yaml
 create mode 100644 cluster/examples/coreos/ceph-before-reboot-script.yaml
 create mode 100644 cluster/examples/coreos/rbac.yaml

diff --git a/Documentation/container-linux.md b/Documentation/container-linux.md
new file mode 100644
index 000000000..48cb3e029
--- /dev/null
+++ b/Documentation/container-linux.md
@@ -0,0 +1,84 @@
+---
+title: Container Linux
+weight: 80
+indent: true
+---
+# Using the Container Linux Update Operator with Rook
+
+When you are using Container Linux (CoreOS) and have the update engine enabled, it could be that a node reboots quickly after another not leaving enough time for the Rook cluster to rebuild. The [Container Linux Update Operator](https://github.com/coreos/container-linux-update-operator) is the solution for this, you can block your nodes to reboot until the Ceph cluster is healthy.
+
+## Prequisites
+- An operational Container Linux Kubernetes cluster (Successfully tested with 1.8.4)
+- A working rook cluster
+- The update-engine.service systemd unit on each machine should be unmasked, enabled and started in systemd
+- The locksmithd.service systemd unit on each machine should be masked and stopped in systemd
+
+## Start the update operator
+
+Proper reading of the README on the [Container Linux Update Operator](https://github.com/coreos/container-linux-update-operator) is necessary. Clone the repo and go in the `examples` directory.
+
+Look for the file named `update-operator.yaml` and update the `command` part of the container from:
+```yaml
+command:
+- "/bin/update-operator"
+```
+
+to:
+```yaml
+command:
+- "/bin/update-operator"
+- "--before-reboot-annotations"
+- "ceph-before-reboot-check"
+- "--after-reboot-annotations"
+- "ceph-after-reboot-check"
+```
+
+You can also add the `-v 6` argument for more extensive logging.
+
+Now create the update-operator by invoking following commands:
+```
+kubectl create -f namespace.yaml
+kubectl create -f cluster-role.yaml
+kubectl create -f cluster-role-binding.yaml
+kubectl create -f update-operator.yaml
+kubectl create -f update-agent.yaml
+```
+
+These files create a new namespace `reboot-coordinator`, configured to listen for the node annotation `ceph-reboot-check`. Now you can create both files in the `cluster/examples/coreos` folder, here's a short description of what each file does:
+
+- `rbac.yaml`: This file contains the necessary RBAC settings.
+- `ceph-after-reboot-script.yaml`: This file creates a `ConfigMap` containing a bash script which will be mounted in the `rook-toolbox` image as executable file.
+- `ceph-before-reboot-script.yaml`: This file creates a `ConfigMap` containing a bash script which will be mounted in the `rook-toolbox` image as executable file.
+- `before-reboot-daemonset.yaml`: This file creates a `DaemonSet` which waits for a node being labeled `before-reboot=true`, runs and checks the Ceph status. If all is correct, it annotates the node with `ceph-before-reboot-check=true`.
+- `after-reboot-daemonset.yaml`: This file creates a `DaemonSet` which waits for a node being labeled `after-reboot=true`, runs and unsets the `noout` option for the ceph OSDs. If all is correct, it annotates the node with `ceph-after-reboot-check=true`.
+
+The node annotation `ceph-no-noout=true` can be used to avoid `ceph-before-reboot-check` from setting the OSD `noout` flag. This annotation should only be used when deleting a node from a cluster, this way the cluster starts rebalancing immediately, not waiting for the node to come back up.
+
+```
+kubectl create -f rbac.yaml
+kubectl create -f ceph-after-reboot-script.yaml
+kubectl create -f ceph-before-reboot-script.yaml
+kubectl create -f before-reboot-daemonset.yaml
+kubectl create -f after-reboot-daemonset.yaml
+```
+
+## Destroy the update operator
+
+To destroy all elements created in this file, run:
+```
+kubectl delete -f before-reboot-daemonset.yaml
+kubectl delete -f after-reboot-daemonset.yaml
+kubectl delete -f ceph-after-reboot-script.yaml
+kubectl delete -f ceph-before-reboot-script.yaml
+kubectl delete -f rbac.yaml
+```
+
+Then you may safely delete the update operator itself:
+From the directory of the Container Linux Update Operator you cloned earlier, go again into the `examples` folder and run following commands:
+```
+kubectl delete -f update-agent.yaml
+kubectl delete -f update-operator.yaml
+kubectl delete -f cluster-role-binding.yaml
+kubectl delete -f cluster-role.yaml
+kubectl delete -f namespace.yaml
+```
diff --git a/cluster/examples/coreos/after-reboot-daemonset.yaml b/cluster/examples/coreos/after-reboot-daemonset.yaml
new file mode 100644
index 000000000..9fe545e77
--- /dev/null
+++ b/cluster/examples/coreos/after-reboot-daemonset.yaml
@@ -0,0 +1,55 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: ceph-after-reboot-check
+  namespace: rook
+spec:
+  template:
+    metadata:
+      labels:
+        app: ceph-after-reboot-check
+    spec:
+      serviceAccountName: rook-node-annotator
+      nodeSelector:
+        container-linux-update.v1.coreos.com/after-reboot: "true"
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      containers:
+      - name: ceph-after-reboot-check
+        image: rook/toolbox:master
+        imagePullPolicy: IfNotPresent
+        command: ["/scripts/status-check.sh"]
+        env:
+          - name: ROOK_ADMIN_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: rook-ceph-mon
+                key: admin-secret
+          - name: NODE
+            valueFrom:
+               fieldRef:
+                  fieldPath: spec.nodeName
+        volumeMounts:
+          - mountPath: /lib/modules
+            name: libmodules
+          - name: mon-endpoint-volume
+            mountPath: /etc/rook
+          - name: status-script
+            mountPath: /scripts
+      hostNetwork: false
+      volumes:
+        - name: libmodules
+          hostPath:
+            path: /lib/modules
+        - name: mon-endpoint-volume
+          configMap:
+            name: rook-ceph-mon-endpoints
+            items:
+            - key: data
+              path: mon-endpoints
+        - name: status-script
+          configMap:
+            name: ceph-after-reboot-script
+            defaultMode: 0740
diff --git a/cluster/examples/coreos/before-reboot-daemonset.yaml b/cluster/examples/coreos/before-reboot-daemonset.yaml
new file mode 100644
index 000000000..1025935e6
--- /dev/null
+++ b/cluster/examples/coreos/before-reboot-daemonset.yaml
@@ -0,0 +1,55 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: ceph-before-reboot-check
+  namespace: rook
+spec:
+  template:
+    metadata:
+      labels:
+        app: ceph-before-reboot-check
+    spec:
+      serviceAccountName: rook-node-annotator
+      nodeSelector:
+        container-linux-update.v1.coreos.com/before-reboot: "true"
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      containers:
+      - name: ceph-before-reboot-check
+        image: rook/toolbox:master
+        imagePullPolicy: IfNotPresent
+        command: ["/scripts/status-check.sh"]
+        env:
+          - name: ROOK_ADMIN_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: rook-ceph-mon
+                key: admin-secret
+          - name: NODE
+            valueFrom:
+               fieldRef:
+                  fieldPath: spec.nodeName
+        volumeMounts:
+          - mountPath: /lib/modules
+            name: libmodules
+          - name: mon-endpoint-volume
+            mountPath: /etc/rook
+          - name: status-script
+            mountPath: /scripts
+      hostNetwork: false
+      volumes:
+        - name: libmodules
+          hostPath:
+            path: /lib/modules
+        - name: mon-endpoint-volume
+          configMap:
+            name: rook-ceph-mon-endpoints
+            items:
+            - key: data
+              path: mon-endpoints
+        - name: status-script
+          configMap:
+            name: ceph-before-reboot-script
+            defaultMode: 0740
diff --git a/cluster/examples/coreos/ceph-after-reboot-script.yaml b/cluster/examples/coreos/ceph-after-reboot-script.yaml
new file mode 100644
index 000000000..adad7d183
--- /dev/null
+++ b/cluster/examples/coreos/ceph-after-reboot-script.yaml
@@ -0,0 +1,127 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ceph-after-reboot-script
+  namespace: rook
+data:
+  status-check.sh: |-
+    #!/bin/bash
+
+    # preflightCheck checks for existence of "dependencies"
+    preflightCheck() {
+        if [ ! -f "/var/run/secrets/kubernetes.io/serviceaccount/token" ]; then
+            echo "$(date) | No Kubernetes ServiceAccount token found."
+            exit 1
+        else
+            KUBE_TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+            export KUBE_TOKEN
+        fi
+    }
+    # updateNodeRebootAnnotation sets the `ceph-reboot-finished` annotation to `true` on `$NODE`
+    updateNodeRebootAnnotation(){
+        export PATCH="[{ \"op\": \"add\", \"path\": \"/metadata/annotations/ceph-after-reboot-check\", \"value\": \"true\" }]"
+
+        TRIES=0
+        until [ $TRIES -eq 10 ]; do
+            if curl -sSk \
+                --fail \
+                -XPATCH \
+                -H "Authorization: Bearer $KUBE_TOKEN" \
+                -H "Accept: application/json" \
+                -H "Content-Type:application/json-patch+json" \
+                --data "$PATCH" \
+                "https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/nodes/$NODE"; then
+                echo "$(date) | Annotation \"ceph-reboot-finished\" from node $NODE updated to \"true\". Reboot finished!"
+                return 0
+            else
+                echo "$(date) | Kubernetes API server connection error, will retry in 5 seconds..."
+                (( TRIES++ ))
+                /bin/sleep 5
+            fi
+        done
+        return 1
+    }
+
+    # deleteNooutAnnotation deletes the `ceph-no-noout` annotation from `$NODE`
+    deleteNooutAnnotation(){
+        export PATCH="[{ \"op\": \"remove\", \"path\": \"/metadata/annotations/ceph-no-noout\"}]"
+
+        TRIES=0
+        until [ $TRIES -eq 10 ]; do
+            if curl -sSk \
+                --fail \
+                -XPATCH \
+                -H "Authorization: Bearer $KUBE_TOKEN" \
+                -H "Accept: application/json" \
+                -H "Content-Type:application/json-patch+json" \
+                --data "$PATCH" \
+                "https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/nodes/$NODE"; then
+                echo "$(date) | Annotation \"ceph-no-noout\" from node $NODE removed!"
+                return 0
+            else
+                echo "$(date) | Kubernetes API server connection error, will retry in 5 seconds..."
+                (( TRIES++ ))
+                /bin/sleep 5
+            fi
+        done
+        return 1
+    }
+
+    # Check if noout option should be set
+    checkForNoout() {
+
+        # Fetch the annotations list of the node
+        TRIES=0
+        until [ $TRIES -eq 10 ]; do
+            if NODE_ANNOTATIONS=$(curl -sSk \
+                --fail \
+                -H "Authorization: Bearer $KUBE_TOKEN" \
+                -H "Accept: application/json" \
+                "https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/nodes/$NODE" \
+                | jq ".metadata.annotations") ; then
+
+                echo "$(date) | Node annotations collected, looking for \"ceph-no-noout\" annotation"
+                if [ $(echo "$NODE_ANNOTATIONS" | jq '."ceph-no-noout"') != "null" ] ; then
+                    echo "$(date) | Node annotation \"ceph-no-noout\" exists, deleting the annotation on node $NODE"
+                    deleteNooutAnnotation
+                else
+                    echo "$(date) | Node annotation \"ceph-no-noout\" doesn't exist, unsetting Ceph noout flag"
+                    ceph osd unset noout
+                fi
+                return 0
+            else
+                echo "$(date) | Kubernetes API server connection error, will retry in 5 seconds..."
+                (( TRIES++ ))
+                /bin/sleep 5
+            fi
+        done
+        return 1
+    }
+
+    preflightCheck
+
+    echo "$(date) | Running the rook toolbox config initiation script..."
+    /usr/local/bin/toolbox.sh &
+
+    TRIES=0
+    until [ -f /etc/ceph/ceph.conf ]; do
+        [ $TRIES -eq 10 ] && { echo "$(date) | No Ceph config found after 10 tries. Exiting ..."; exit 1; }
+        echo "$(date) | Waiting for Ceph config (try $TRIES from 10) ..."
+        (( TRIES++ ))
+        sleep 3
+    done
+
+    while true; do
+        if checkForNoout; then
+            if updateNodeRebootAnnotation; then
+                echo "$(date) | Reboot from node $NODE completely finished!"
+                exit 0
+            else
+                echo "$(date) | Failed updating annotation for $NODE. Exiting."
+                exit 1
+            fi
+        else
+            echo "$(date) | Failed updating ceph noout flag or node annotation. Exiting."
+            exit 1
+        fi
+    done
diff --git a/cluster/examples/coreos/ceph-before-reboot-script.yaml b/cluster/examples/coreos/ceph-before-reboot-script.yaml
new file mode 100644
index 000000000..357228206
--- /dev/null
+++ b/cluster/examples/coreos/ceph-before-reboot-script.yaml
@@ -0,0 +1,120 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ceph-before-reboot-script
+  namespace: rook
+data:
+  status-check.sh: |-
+    #!/bin/bash
+
+    # preflightCheck checks for existence of "dependencies"
+    preflightCheck() {
+        if [ ! -f "/var/run/secrets/kubernetes.io/serviceaccount/token" ]; then
+            echo "$(date) | No Kubernetes ServiceAccount token found."
+            exit 1
+        else
+            KUBE_TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+            export KUBE_TOKEN
+        fi
+    }
+
+    # updateNodeRebootAnnotation sets the `ceph-reboot-check` annotation to `true` on `$NODE`
+    updateNodeRebootAnnotation(){
+        export PATCH="[{ \"op\": \"add\", \"path\": \"/metadata/annotations/ceph-before-reboot-check\", \"value\": \"true\" }]"
+
+        TRIES=0
+        until [ $TRIES -eq 10 ]; do
+            if curl -sSk \
+                --fail \
+                -XPATCH \
+                -H "Authorization: Bearer $KUBE_TOKEN" \
+                -H "Accept: application/json" \
+                -H "Content-Type:application/json-patch+json" \
+                --data "$PATCH" \
+                "https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/nodes/$NODE"; then
+                echo "$(date) | Annotation \"ceph-reboot-check\" from node $NODE updated to \"true\". Reboot confirmed!"
+                return 0
+            else
+                echo "$(date) | Kubernetes API server connection error, will retry in 5 seconds..."
+                (( TRIES++ ))
+                /bin/sleep 5
+            fi
+        done
+        return 1
+    }
+
+    # Check if noout option should be set
+    checkForNoout() {
+
+        # Fetch the annotations list of the node
+        TRIES=0
+        until [ $TRIES -eq 10 ]; do
+            if NODE_ANNOTATIONS=$(curl -sSk \
+                --fail \
+                -H "Authorization: Bearer $KUBE_TOKEN" \
+                -H "Accept: application/json" \
+                "https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/nodes/$NODE" \
+                | jq ".metadata.annotations") ; then
+                echo "$(date) | Node annotations collected, looking for \"ceph-no-noout\" annotation"
+
+                if [ $(echo "$NODE_ANNOTATIONS" | jq '."ceph-no-noout"') != "null" ] ; then
+                    echo "$(date) | Node annotation \"ceph-no-noout\" exists, not setting Ceph noout flag"
+                else
+                    echo "$(date) | Node annotation \"ceph-no-noout\" doesn't exist, setting Ceph noout flag"
+                    ceph osd set noout
+                fi
+                return 0
+            else
+                echo "$(date) | Kubernetes API server connection error, will retry in 5 seconds..."
+                (( TRIES++ ))
+                /bin/sleep 5
+            fi
+        done
+        return 1
+
+    }
+
+    # checkCephClusterHealth checks `ceph health` command for `HEALTH_OK`
+    checkCephClusterHealth(){
+        echo "$(date) | Running ceph health command"
+        if /usr/bin/ceph health | grep -q "HEALTH_OK"; then
+            echo "$(date) | Ceph cluster health is: OKAY"
+            return 0
+        fi
+        return 1
+    }
+
+    preflightCheck
+
+    echo "$(date) | Running the rook toolbox config initiation script..."
+    /usr/local/bin/toolbox.sh &
+
+    TRIES=0
+    until [ -f /etc/ceph/ceph.conf ]; do
+        [ $TRIES -eq 10 ] && { echo "$(date) | No Ceph config found after 10 tries. Exiting ..."; exit 1; }
+        echo "$(date) | Waiting for Ceph config (try $TRIES from 10) ..."
+        (( TRIES++ ))
+        sleep 3
+    done
+
+    while true; do
+        if checkCephClusterHealth; then
+            if checkForNoout; then
+                if updateNodeRebootAnnotation; then
+                    while true; do
+                        echo "$(date) | Waiting for $NODE to reboot ..."
+                        /bin/sleep 30
+                    done
+                    exit 0
+                else
+                    echo "$(date) | Failed updating annotation for $NODE. Exiting."
+                    exit 1
+                fi
+            else
+                echo "$(date) | Failed setting the Ceph osd noout flag. Exiting."
+                exit 1
+            fi
+        fi
+        echo "$(date) | Ceph cluster Health not HEALTH_OK currently. Checking again in 20 seconds ..."
+        /bin/sleep 20
+    done
diff --git a/cluster/examples/coreos/rbac.yaml b/cluster/examples/coreos/rbac.yaml
new file mode 100644
index 000000000..929f3a557
--- /dev/null
+++ b/cluster/examples/coreos/rbac.yaml
@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: rook-node-annotator
+  namespace: rook
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: rook
+  name: rook-node-annotator
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["patch", "get"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: rook-node-annotator
+  namespace: rook
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: rook-node-annotator
+subjects:
+- kind: ServiceAccount
+  name: rook-node-annotator
+  namespace: rook