From ae2ac846c77a7164e92556e2e2f28b323b9b67d3 Mon Sep 17 00:00:00 2001 From: Karol Chrapek Date: Tue, 21 Aug 2018 13:22:26 +0200 Subject: [PATCH 1/2] Change rook-tools deployment Signed-off-by: Karol Chrapek --- Documentation/toolbox.md | 107 ++++++++++-------- cluster/examples/kubernetes/ceph/toolbox.yaml | 99 +++++++++------- 2 files changed, 114 insertions(+), 92 deletions(-) diff --git a/Documentation/toolbox.md b/Documentation/toolbox.md index 53f7bfeb8..ff271b203 100644 --- a/Documentation/toolbox.md +++ b/Documentation/toolbox.md @@ -10,58 +10,69 @@ The toolbox is based on CentOS, so more tools of your choosing can be easily ins ## Running the Toolbox in Kubernetes -The rook toolbox can run as a pod in a Kubernetes cluster. After you ensure you have a running Kubernetes cluster with rook deployed (see the [Kubernetes](ceph-quickstart.md) instructions), -launch the rook-ceph-tools pod. +The rook toolbox can run as a deployment in a Kubernetes cluster. After you ensure you have a running Kubernetes cluster with rook deployed (see the [Kubernetes](quickstart.md) instructions), +launch the rook-tools pod. Save the tools spec as `toolbox.yaml`: ```yaml -apiVersion: v1 -kind: Pod +apiVersion: apps/v1 +kind: Deployment metadata: - name: rook-ceph-tools - namespace: rook-ceph + name: rook-tools + namespace: rook + labels: + app: rook-tools spec: - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: rook-ceph-tools - image: rook/ceph-toolbox:master - imagePullPolicy: IfNotPresent - env: - - name: ROOK_ADMIN_SECRET - valueFrom: - secretKeyRef: - name: rook-ceph-mon - key: admin-secret - securityContext: - privileged: true - volumeMounts: - - mountPath: /dev - name: dev - - mountPath: /sys/bus - name: sysbus - - mountPath: /lib/modules - name: libmodules - - name: mon-endpoint-volume - mountPath: /etc/rook - # if hostNetwork: false, the "rbd map" command hangs, see https://github.com/rook/rook/issues/2021 - hostNetwork: true - volumes: - - name: dev - hostPath: - path: /dev - - name: sysbus - hostPath: - path: /sys/bus - - name: libmodules - hostPath: - path: /lib/modules - - name: mon-endpoint-volume - configMap: - name: rook-ceph-mon-endpoints - items: - - key: data - path: mon-endpoints + replicas: 1 + selector: + matchLabels: + app: rook-tools + template: + metadata: + labels: + app: rook-tools + spec: + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: rook-tools + image: rook/ceph-toolbox:master + imagePullPolicy: IfNotPresent + env: + - name: ROOK_ADMIN_SECRET + valueFrom: + secretKeyRef: + name: rook-ceph-mon + key: admin-secret + securityContext: + privileged: true + volumeMounts: + - mountPath: /dev + name: dev + - mountPath: /sys/bus + name: sysbus + - mountPath: /lib/modules + name: libmodules + - name: mon-endpoint-volume + mountPath: /etc/rook + # if hostNetwork: false, the "rbd map" command hangs, see https://github.com/rook/rook/issues/2021 + hostNetwork: true + volumes: + - name: dev + hostPath: + path: /dev + - name: sysbus + hostPath: + path: /sys/bus + - name: libmodules + hostPath: + path: /lib/modules + - name: mon-endpoint-volume + configMap: + name: rook-ceph-mon-endpoints + items: + - key: data + path: mon-endpoints ``` Launch the rook-ceph-tools pod: @@ -71,12 +82,12 @@ kubectl create -f toolbox.yaml Wait for the toolbox pod to download its container and get to the `running` state: ```bash -kubectl -n rook-ceph get pod rook-ceph-tools +kubectl -n rook get pod -l "app=rook-tools" ``` Once the rook-ceph-tools pod is running, you can connect to it with: ```bash -kubectl -n rook-ceph exec -it rook-ceph-tools bash +kubectl -n rook exec -it $(kubectl -n rook get pod -l "app=rook-tools" -o jsonpath='{.items[0].metadata.name}') bash ``` All available tools in the toolbox are ready for your troubleshooting needs. Example: diff --git a/cluster/examples/kubernetes/ceph/toolbox.yaml b/cluster/examples/kubernetes/ceph/toolbox.yaml index 3ec39b85d..5d46a01b6 100644 --- a/cluster/examples/kubernetes/ceph/toolbox.yaml +++ b/cluster/examples/kubernetes/ceph/toolbox.yaml @@ -1,46 +1,57 @@ -apiVersion: v1 -kind: Pod +apiVersion: apps/v1 +kind: Deployment metadata: - name: rook-ceph-tools - namespace: rook-ceph + name: rook-tools + namespace: rook + labels: + app: rook-tools spec: - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: rook-ceph-tools - image: rook/ceph-toolbox:master - imagePullPolicy: IfNotPresent - env: - - name: ROOK_ADMIN_SECRET - valueFrom: - secretKeyRef: - name: rook-ceph-mon - key: admin-secret - securityContext: - privileged: true - volumeMounts: - - mountPath: /dev - name: dev - - mountPath: /sys/bus - name: sysbus - - mountPath: /lib/modules - name: libmodules - - name: mon-endpoint-volume - mountPath: /etc/rook - # if hostNetwork: false, the "rbd map" command hangs, see https://github.com/rook/rook/issues/2021 - hostNetwork: true - volumes: - - name: dev - hostPath: - path: /dev - - name: sysbus - hostPath: - path: /sys/bus - - name: libmodules - hostPath: - path: /lib/modules - - name: mon-endpoint-volume - configMap: - name: rook-ceph-mon-endpoints - items: - - key: data - path: mon-endpoints + replicas: 1 + selector: + matchLabels: + app: rook-tools + template: + metadata: + labels: + app: rook-tools + spec: + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: rook-tools + image: rook/ceph-toolbox:master + imagePullPolicy: IfNotPresent + env: + - name: ROOK_ADMIN_SECRET + valueFrom: + secretKeyRef: + name: rook-ceph-mon + key: admin-secret + securityContext: + privileged: true + volumeMounts: + - mountPath: /dev + name: dev + - mountPath: /sys/bus + name: sysbus + - mountPath: /lib/modules + name: libmodules + - name: mon-endpoint-volume + mountPath: /etc/rook + # if hostNetwork: false, the "rbd map" command hangs, see https://github.com/rook/rook/issues/2021 + hostNetwork: true + volumes: + - name: dev + hostPath: + path: /dev + - name: sysbus + hostPath: + path: /sys/bus + - name: libmodules + hostPath: + path: /lib/modules + - name: mon-endpoint-volume + configMap: + name: rook-ceph-mon-endpoints + items: + - key: data + path: mon-endpoints From 227186d1e3369b1951c9f0d8f3616a467cf72475 Mon Sep 17 00:00:00 2001 From: Karol Chrapek Date: Thu, 23 Aug 2018 18:35:00 +0200 Subject: [PATCH 2/2] Update namespace and tools name for rook-ceph-tools Signed-off-by: Karol Chrapek --- Documentation/common-issues.md | 4 ++-- Documentation/toolbox.md | 22 +++++++++---------- Documentation/upgrade.md | 6 ++--- cluster/examples/kubernetes/ceph/toolbox.yaml | 12 +++++----- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Documentation/common-issues.md b/Documentation/common-issues.md index df02951c3..a31f99431 100644 --- a/Documentation/common-issues.md +++ b/Documentation/common-issues.md @@ -49,7 +49,7 @@ After you verify the basic health of the running pods, next you will want to run ### Tools in the Rook Toolbox The [rook-ceph-tools pod](./toolbox.md) is a one-stop shop for both Ceph tools and other troubleshooting tools. Once the pod is up and running one connect to the pod to execute Ceph commands to evaluate that current state of the cluster. ```bash - kubectl exec -it rook-ceph-tools bash + kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[0].metadata.name}') bash ``` ### Tools in other pods @@ -243,7 +243,7 @@ We want to help you get your storage working and learn from those lessons to pre Create a [rook-ceph-tools pod](./toolbox.md) to investigate the current state of CEPH. Here is an example of what one might see. In this case the `ceph status` command would just hang so a CTRL-C needed to be sent. ```console -$ kubectl -n rook-ceph exec -it rook-ceph-tools bash +$ kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[0].metadata.name}') bash root@rook-ceph-tools:/# ceph status ^CCluster connection interrupted or timed out ``` diff --git a/Documentation/toolbox.md b/Documentation/toolbox.md index ff271b203..9c02d2d45 100644 --- a/Documentation/toolbox.md +++ b/Documentation/toolbox.md @@ -10,8 +10,8 @@ The toolbox is based on CentOS, so more tools of your choosing can be easily ins ## Running the Toolbox in Kubernetes -The rook toolbox can run as a deployment in a Kubernetes cluster. After you ensure you have a running Kubernetes cluster with rook deployed (see the [Kubernetes](quickstart.md) instructions), -launch the rook-tools pod. +The rook toolbox can run as a deployment in a Kubernetes cluster. After you ensure you have a running Kubernetes cluster with rook deployed (see the [Kubernetes](ceph-quickstart.md) instructions), +launch the rook-ceph-tools pod. Save the tools spec as `toolbox.yaml`: @@ -19,23 +19,23 @@ Save the tools spec as `toolbox.yaml`: apiVersion: apps/v1 kind: Deployment metadata: - name: rook-tools - namespace: rook + name: rook-ceph-tools + namespace: rook-ceph labels: - app: rook-tools + app: rook-ceph-tools spec: replicas: 1 selector: matchLabels: - app: rook-tools + app: rook-ceph-tools template: metadata: labels: - app: rook-tools + app: rook-ceph-tools spec: dnsPolicy: ClusterFirstWithHostNet containers: - - name: rook-tools + - name: rook-ceph-tools image: rook/ceph-toolbox:master imagePullPolicy: IfNotPresent env: @@ -82,12 +82,12 @@ kubectl create -f toolbox.yaml Wait for the toolbox pod to download its container and get to the `running` state: ```bash -kubectl -n rook get pod -l "app=rook-tools" +kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" ``` Once the rook-ceph-tools pod is running, you can connect to it with: ```bash -kubectl -n rook exec -it $(kubectl -n rook get pod -l "app=rook-tools" -o jsonpath='{.items[0].metadata.name}') bash +kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[0].metadata.name}') bash ``` All available tools in the toolbox are ready for your troubleshooting needs. Example: @@ -100,7 +100,7 @@ rados df When you are done with the toolbox, remove the pod: ```bash -kubectl -n rook-ceph delete pod rook-ceph-tools +kubectl -n rook-ceph delete deployment rook-ceph-tools ``` ## Troubleshooting without the Toolbox diff --git a/Documentation/upgrade.md b/Documentation/upgrade.md index 3b597654d..50463f4a8 100644 --- a/Documentation/upgrade.md +++ b/Documentation/upgrade.md @@ -82,7 +82,7 @@ If pods aren't running or are restarting due to crashes, you can get more inform The Rook toolbox contains the Ceph tools that can give you status details of the cluster with the `ceph status` command. Let's look at some sample output and review some of the details: ```bash -> kubectl -n rook exec -it rook-ceph-tools -- ceph status +> kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[0].metadata.name}') -- ceph status cluster: id: fe7ae378-dc77-46a1-801b-de05286aa78e health: HEALTH_OK @@ -256,7 +256,7 @@ Instructions for verifying cluster health can be found in the [health verificati After upgrading the operator, the placement groups may show as status unknown. If you see this, go to the section on [upgrading OSDs](#object-storage-daemons-osds). Upgrading the OSDs will resolve this issue. ``` -kubectl -n rook exec -it rook-tools -- ceph status +kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[0].metadata.name}') -- ceph status ... pgs: 100.000% pgs unknown 100 unknown @@ -266,7 +266,7 @@ kubectl -n rook exec -it rook-tools -- ceph status The toolbox pod runs the tools we will use during the upgrade for cluster status. The toolbox is not expected to contain any state, so we will delete the old pod and start the new toolbox. ```bash -kubectl -n rook delete pod rook-tools +kubectl -n rook-ceph delete deployment rook-ceph-tools ``` After verifying the old tools pod has terminated, start the new toolbox. You will need to either create the toolbox using the yaml in the `release-0.8` branch or simply set the version of the container to `rook/ceph-toolbox:v0.8.0` before creating the toolbox. diff --git a/cluster/examples/kubernetes/ceph/toolbox.yaml b/cluster/examples/kubernetes/ceph/toolbox.yaml index 5d46a01b6..8df616546 100644 --- a/cluster/examples/kubernetes/ceph/toolbox.yaml +++ b/cluster/examples/kubernetes/ceph/toolbox.yaml @@ -1,23 +1,23 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: rook-tools - namespace: rook + name: rook-ceph-tools + namespace: rook-ceph labels: - app: rook-tools + app: rook-ceph-tools spec: replicas: 1 selector: matchLabels: - app: rook-tools + app: rook-ceph-tools template: metadata: labels: - app: rook-tools + app: rook-ceph-tools spec: dnsPolicy: ClusterFirstWithHostNet containers: - - name: rook-tools + - name: rook-ceph-tools image: rook/ceph-toolbox:master imagePullPolicy: IfNotPresent env: