From adcb56ee70657154ba67abd44ae8ca83c561832c Mon Sep 17 00:00:00 2001 From: parth-gr Date: Mon, 1 Jun 2026 20:11:43 +0530 Subject: [PATCH] core: add a install guide for tnf add a install guide for tnf 1) Add a drbd install script 2) add a drbd instal doc assisted-by: anthropic-ai/claude-code Signed-off-by: parth-gr (cherry picked from commit 8171c3ed3b1068f130bf13598e84a4eea72c23e1) --- .github/workflows/codespell.yaml | 2 +- .../Advanced/two-node-cluster-drbd.md | 219 +++++ deploy/examples/cluster-tnf.yaml | 51 ++ deploy/examples/drbd-setup-experimental.sh | 803 ++++++++++++++++++ deploy/examples/pool-tnf.yaml | 10 + 5 files changed, 1084 insertions(+), 1 deletion(-) create mode 100644 Documentation/Storage-Configuration/Advanced/two-node-cluster-drbd.md create mode 100644 deploy/examples/cluster-tnf.yaml create mode 100755 deploy/examples/drbd-setup-experimental.sh create mode 100644 deploy/examples/pool-tnf.yaml diff --git a/.github/workflows/codespell.yaml b/.github/workflows/codespell.yaml index 8af5904c4..f3f07fbac 100644 --- a/.github/workflows/codespell.yaml +++ b/.github/workflows/codespell.yaml @@ -46,7 +46,7 @@ jobs: # parm: modinfo parameter # assigment: inherited from K8s TopologySpreadConstraints dependency # ro, RO: means read-only - ignore_words_list: aks,keyserver,atleast,ser,ist,ba,iam,te,parm,assigment,ro,RO,addin,NotIn,Parth + ignore_words_list: aks,keyserver,atleast,ser,ist,ba,iam,te,parm,assigment,ro,RO,addin,NotIn,Parth,UpToDate check_filenames: true check_hidden: true misspell: diff --git a/Documentation/Storage-Configuration/Advanced/two-node-cluster-drbd.md b/Documentation/Storage-Configuration/Advanced/two-node-cluster-drbd.md new file mode 100644 index 000000000..51e9341be --- /dev/null +++ b/Documentation/Storage-Configuration/Advanced/two-node-cluster-drbd.md @@ -0,0 +1,219 @@ +--- +title: Two-Node Cluster +--- + +# Two-Node Cluster (experimental) + +This guide provides installation instructions for setting up a two-node Kubernetes cluster with Rook using DRBD (Distributed Replicated Block Device) for storage replication for one of the Ceph mons. + +For architecture details and design rationale, see the [Two-Node Fencing Design Document](https://github.com/rook/rook/blob/master/design/two-node-fencing.md). + +## Prerequisites + +Before starting the installation: + +- **2-node Kubernetes cluster** with fencing configured (e.g., OpenShift with DualReplica topology) +- **Raw block devices** on each node (SSD-class, same size, no filesystem) + +## Installation Steps + +### Step 1: Install DRBD Kernel Modules + +DRBD kernel modules must be installed and loaded on both nodes before running the setup script. + +Follow the DRBD installation documentation for your distribution and install `drbd & drbd_transport_tcp`: + +- **[DRBD 9.0 Installation Guide](https://linbit.com/drbd-user-guide/drbd-guide-9_0-en/#ch-install-packages)** - Official installation guide +- **[DRBD User's Guide](https://linbit.com/drbd-user-guide/)** - Complete documentation +- **[DRBD GitHub Repository](https://github.com/LINBIT/drbd)** - Source code and build instructions + +#### Verify Installation + +On both nodes, verify DRBD modules are loaded: + +```bash +# Check loaded modules +lsmod | grep drbd + +# Expected output: +# drbd_transport_tcp 16384 0 +# drbd 614400 1 drbd_transport_tcp + +# Check DRBD version +drbdadm --version +``` + +### Step 2: Identify Block Devices + +Use the setup script to list available block devices on both nodes: + +```bash +./deploy/examples/drbd-setup-experimental.sh -l +``` + +Example output: + +```text +=== Block devices (node0=node-0, node1=node-1) === +Use the PATH column (e.g. -d /dev/sdb or -d0 / -d1 per-node paths). + +--- node-0 --- +NAME PATH SIZE ROTA TYPE FSTYPE +sdb /dev/sdb 10G 0 disk <-- Use this + +--- node-1 --- +NAME PATH SIZE ROTA TYPE FSTYPE +sdb /dev/sdb 10G 0 disk <-- Use this + +Same path on both nodes: -d +Different paths (same size): -d0 -d1 +``` + +**Requirements for backing devices**: + +- Must be SSD/NVMe (ROTA=0, non-rotational) +- Can be a partition of the disk +- Same size on both nodes +- No existing filesystem (FSTYPE empty) +- Writable (not read-only) +- Volume or partition is not larger than 10Gi + +### Step 3: Run DRBD Setup Script + +The setup script configures DRBD, performs initial sync, and creates the auto-start DaemonSet. + +#### Same Device Path on Both Nodes + +```bash +./deploy/examples/drbd-setup-experimental.sh -d /dev/sdb +``` + +#### Different Device Paths Per Node + +```bash +./deploy/examples/drbd-setup-experimental.sh -d0 /dev/sdb -d1 /dev/sdc +``` + +**Available options**: + +`./deploy/examples/drbd-setup-experimental.sh -h` + +!!! note + Network connectivity on DRBD replication port (default: 7794) between nodes, we can customise this in the script. + +### Step 4: Verify DRBD Setup + +Check auto-start DaemonSet: + +```bash +kubectl get daemonset drbd-autostart -n rook-ceph +kubectl get pods -n rook-ceph -l app=drbd-autostart +``` + +Check for the drbd-configure configmap + +```bash +kubectl get cm -n rook-ceph drbd-configure -o yaml +``` + +### Step 5: Deploy Rook CephCluster + +Create the CephCluster CR with floating monitor configuration: + +Apply the cluster: + +```bash +kubectl apply -f deploy/examples/cluster-tnf.yaml +``` + +### Step 6: Verify Ceph Cluster + +Check monitor pods: + +```bash +kubectl get pods -n rook-ceph -l app=rook-ceph-mon +``` + +Expected: 3 monitor pods (mon-a, mon-b, mon-c) + +Check Ceph status: + +```bash +kubectl exec -n rook-ceph deploy/rook-ceph-tools -- ceph status +``` + +Expected output: + +```text + cluster: + id: + health: HEALTH_OK + + services: + mon: 3 daemons, quorum a,b,c (age ...) + ... +``` + +## Configuration Summary + +The setup creates: + +- **DRBD resource** (`r0`) with Protocol C synchronous replication +- **DRBD device** (`/dev/drbd0`) with XFS filesystem +- **Auto-start DaemonSet** (`drbd-autostart`) in `rook-ceph` namespace +- **Configuration ConfigMap** (`drbd-configure`) in `rook-ceph` namespace +- **Two pinned monitors** (one per node) with local storage +- **One floating monitor** (`mon-c`) backed by DRBD + +## Two-Node Cluster Recommendations + +Configure pools with **replica-2**, [pool-tnf.yaml](https://github.com/rook/rook/blob/master/deploy/examples/pool-tnf.yaml) + +## Troubleshooting + +### DRBD Modules Not Loaded + +**Check**: + +```bash +lsmod | grep drbd +``` + +**Fix**: Install DRBD kernel modules (see Step 1) + +### DRBD Sync Not Progressing + +**Check network connectivity**: + +```bash +nc -zv 7794 +``` + +**Check DRBD status**: + +```bash +drbdadm status r0 +``` + +### Floating Monitor Not Starting + +**Check DRBD is Secondary on both nodes**: + +```bash +drbdadm role r0 +# Expected: Secondary/Secondary +``` + +**Check ConfigMap**: + +```bash +kubectl get configmap drbd-configure -n rook-ceph -o yaml +``` + +### Re-run Setup Script + +The script is idempotent and safe to re-run: + +```bash +./deploy/examples/drbd-setup-experimental.sh -d /dev/sdb +``` diff --git a/deploy/examples/cluster-tnf.yaml b/deploy/examples/cluster-tnf.yaml new file mode 100644 index 000000000..0044aeb3e --- /dev/null +++ b/deploy/examples/cluster-tnf.yaml @@ -0,0 +1,51 @@ +################################################################################################################# +# Define the settings for the rook-ceph cluster with DRBD floating monitor for testing. +# This is designed for two-node minikube clusters with DRBD installed. +# All nodes are used for Ceph storage, and a floating monitor is configured to use DRBD. +# +# For testing, a two-node minikube cluster can be created: +# minikube start --profile=rook --nodes=2 --driver=kvm2 --cpus=4 --memory=8192 --disk-size=40g +# +# Then install DRBD with: +# tests/scripts/install-drbd-minikube.sh +# +# And run the test with: +# tests/scripts/test-drbd-floating-mon.sh +################################################################################################################# + +apiVersion: ceph.rook.io/v1 +kind: CephCluster +metadata: + name: rook-ceph + namespace: rook-ceph # namespace:cluster +spec: + dataDirHostPath: /var/lib/rook + cephVersion: + image: quay.io/ceph/ceph:v20.2.1 + allowUnsupported: false + # The path on the host where configuration files will be persisted + # IMPORTANT: Disable host networking for floating monitor + network: + hostNetwork: false + mon: + # Number of monitors - must be 3 for two-node with floating mon + count: 3 + # Floating monitor configuration + floatingMon: + # Name of the floating monitor (typically 'c') + name: c + # ConfigMap containing DRBD configuration + configmapName: drbd-configure + priorityClassNames: + mon: system-node-critical + osd: system-node-critical + mgr: system-cluster-critical + storage: + useAllNodes: true + useAllDevices: false + cephConfig: + global: + # Set OSD replica size to 2 for two-node cluster + osd_pool_default_size: "2" + mon_warn_on_pool_no_redundancy: "false" + mon_max_pg_per_osd: "500" diff --git a/deploy/examples/drbd-setup-experimental.sh b/deploy/examples/drbd-setup-experimental.sh new file mode 100755 index 000000000..771b738b4 --- /dev/null +++ b/deploy/examples/drbd-setup-experimental.sh @@ -0,0 +1,803 @@ +#!/bin/bash +# +# Copyright 2026 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DRBD Setup Script for Two-Node Cluster, Safe to re-run( idempotent ). +# The script can skip successful steps & run only the required steps. +# +# CLI overrides env for: --drbd-conf-path, --drbd-dir-path, --drbd-resource, --drbd-device, --drbd-port +# +# Prerequisites: +# - Nodes can pull ${DRBD_IMAGE}; ${DRBD_PORT}/tcp open between nodes. +# +set -euo pipefail + +die() { + echo "Error: $*" >&2 + echo "Please try re-running the script." >&2 + exit 1 +} +msg() { echo "DRBD: $*"; } + +# Wall-clock wait helpers: call _wait_begin immediately before a polling loop; on success call _wait_succeeded "message". +_wait_begin() { _WAIT_T0=$(date +%s); } + +_wait_succeeded() { + local d=$(( $(date +%s) - _WAIT_T0 )) + if (( d < 60 )); then + msg "$1 (in ${d}s)" + elif (( d < 3600 )); then + msg "$1 (in $((d / 60)) min $((d % 60))s)" + else + msg "$1 (in $((d / 60)) min)" + fi +} + +DRBD_IMAGE="${DRBD_IMAGE:-quay.io/linbit/drbd-utils:latest}" # ODF DRBD image (drbdadm + sources) +DEBUG_IMAGE="quay.io/podman/stable" + +DRBD_CONF_PATH="${DRBD_CONF_PATH:-/etc/drbd.conf}" # Main file: include of ${DRBD_DIR_PATH}/*.res only +DRBD_DIR_PATH="${DRBD_DIR_PATH:-/etc/drbd.d}" # Per-resource .res files (actual DRBD definition) +DRBD_RESOURCE="${DRBD_RESOURCE:-r0}" # DRBD resource name (e.g. r0) +DRBD_DEVICE="${DRBD_DEVICE:-/dev/drbd0}" # DRBD block device path on nodes (e.g. /dev/drbd0) +DRBD_PORT="${DRBD_PORT:-7794}" # DRBD replication TCP port (e.g. 7794) + +AUTOSTART_DAEMONSET_NAME="${AUTOSTART_DAEMONSET_NAME:-drbd-autostart}" # DRBD auto-start DaemonSet name +AUTOSTART_DAEMONSET_NS="${AUTOSTART_DAEMONSET_NS:-rook-ceph}" # DRBD auto-start DaemonSet namespace +OUTPUT_CM_NS="${OUTPUT_CM_NS:-rook-ceph}" # Namespace for setup summary ConfigMap +OUTPUT_CM_NAME="${OUTPUT_CM_NAME:-drbd-configure}" # Name for the setup summary ConfigMap + +# Approximate wait ceilings in this script: KMM operator ~5m (60×5s); DRBD modules ~10m (60×10s); +# initial sync ~30m (60×30s); autostart DaemonSet ~5m (60×5s). + +# User input: backing paths (e.g. /dev/sdb). -d = same on both nodes; else -d0 / -d1 per node. +BACKING_PATH="" +BACKING_PATH_NODE0="" +BACKING_PATH_NODE1="" +DISK_RESOLVED_NODE0="" +DISK_RESOLVED_NODE1="" + +LIST_DEVICES_ONLY=0 + +# Node info (populated by detect_nodes) +NODE_0="" +NODE_1="" +NODE_0_IP="" +NODE_1_IP="" + +#--- Functions ---# + +usage() { + cat < + $0 -d0 -d1 + $0 -l + +Backing paths are raw block device paths (e.g. /dev/sdb). Use the PATH column from -l. +Disks must be SSD-class (ROTA 0) and same size on both nodes. + + -d PATH + One path used on both nodes. Choose this when each machine has the replica disk at the + same device name (both nodes use e.g. /dev/sdb for the DRBD lower layer). + +Use -d0/-d1 when the two nodes use different paths; do not combine -d with -d0/-d1. + + -d0 PATH Path on node 0 only (first node name after sorting all cluster nodes). + -d1 PATH Path on node 1 only (second node). + +Discovery: + -l List block devices on each node (NAME, PATH, SIZE, ROTA, TYPE, FSTYPE). + +DRBD options: + --drbd-conf-path PATH Host path to drbd.conf (default ${DRBD_CONF_PATH}) + --drbd-dir-path PATH Host dir for resource snippets (default ${DRBD_DIR_PATH}) + --drbd-resource NAME Logical DRBD resource name in config (default ${DRBD_RESOURCE}) + --drbd-device PATH DRBD upper device node path, same on both nodes (default ${DRBD_DEVICE}) + --drbd-port N TCP port for DRBD replication (default ${DRBD_PORT}) + +General: + -h Show this help and exit + +Environment: + Defaults are documented on each assignment near the top of this script. + OUTPUT_CM_NS / OUTPUT_CM_NAME — namespace and name of the summary ConfigMap (floating mon). + +USAGE +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + -d0) + if [[ -z "${2:-}" ]]; then + die "-d0 requires a path (e.g. /dev/sdb)" + fi + BACKING_PATH_NODE0="$2" + shift 2 + ;; + -d1) + if [[ -z "${2:-}" ]]; then + die "-d1 requires a path (e.g. /dev/sdb)" + fi + BACKING_PATH_NODE1="$2" + shift 2 + ;; + -d) + if [[ -z "${2:-}" ]]; then + die "-d requires a path (e.g. /dev/sdb)" + fi + BACKING_PATH="$2" + shift 2 + ;; + -l) + LIST_DEVICES_ONLY=1 + shift + ;; + --drbd-conf-path) + if [[ -z "${2:-}" ]]; then + die "--drbd-conf-path requires an absolute path to drbd.conf" + fi + DRBD_CONF_PATH="$2" + shift 2 + ;; + --drbd-dir-path) + if [[ -z "${2:-}" ]]; then + die "--drbd-dir-path requires an absolute directory path (e.g. /etc/drbd.d)" + fi + DRBD_DIR_PATH="$2" + shift 2 + ;; + --drbd-resource) + if [[ -z "${2:-}" ]]; then + die "--drbd-resource requires a name" + fi + DRBD_RESOURCE="$2" + shift 2 + ;; + --drbd-device) + if [[ -z "${2:-}" ]]; then + die "--drbd-device requires a path (e.g. /dev/drbd0)" + fi + DRBD_DEVICE="$2" + shift 2 + ;; + --drbd-port) + if [[ -z "${2:-}" ]]; then + die "--drbd-port requires a TCP port number" + fi + DRBD_PORT="$2" + shift 2 + ;; + -h) + usage + exit 0 + ;; + *) + die "Unknown option: $1 (use -h)" + ;; + esac + done + + if [[ "$LIST_DEVICES_ONLY" -eq 1 ]]; then + return 0 + fi + + if [[ -n "$BACKING_PATH" && ( -n "$BACKING_PATH_NODE0" || -n "$BACKING_PATH_NODE1" ) ]]; then + die "Use either -d or both -d0 and -d1 (node0/node1 paths), not both" + fi + if [[ -n "$BACKING_PATH_NODE0" || -n "$BACKING_PATH_NODE1" ]]; then + if [[ -z "$BACKING_PATH_NODE0" || -z "$BACKING_PATH_NODE1" ]]; then + die "Both -d0 and -d1 are required when using per-node paths" + fi + fi + if [[ -z "$BACKING_PATH" && -z "$BACKING_PATH_NODE0" ]]; then + die "Specify backing path(s): -d, or -d0 and -d1, or -l to list devices (see -h)" + fi +} + +check_prerequisites() { + local node_count + node_count=$(kubectl get nodes --no-headers 2>/dev/null | wc -l | tr -d ' ') + if [[ "$node_count" -ne 2 ]]; then + die "expected 2 nodes for TNF, found $node_count" + fi +} + +# Resolve the two cluster node names (sorted ascending) and each node's InternalIP for DRBD endpoints. +detect_nodes() { + local nodes_sorted + + nodes_sorted=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | sort) + NODE_0=$(printf '%s\n' "$nodes_sorted" | head -n 1) + NODE_1=$(printf '%s\n' "$nodes_sorted" | head -n 2 | tail -n 1) + if [[ -z "$NODE_0" || -z "$NODE_1" ]]; then + die "could not resolve two node names" + fi + + NODE_0_IP=$(kubectl get node "$NODE_0" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') + NODE_1_IP=$(kubectl get node "$NODE_1" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') + if [[ -z "$NODE_0_IP" || -z "$NODE_1_IP" ]]; then + die "could not read InternalIP (NODE_0=$NODE_0 NODE_1=$NODE_1)" + fi +} + +# list block devices on both nodes with lsblk +list_devices() { + echo "=== Block devices (node0=$NODE_0, node1=$NODE_1) ===" + echo "Use the PATH column (e.g. -d /dev/sdb or -d0 / -d1 per-node paths)." + echo "" + for n in "$NODE_0" "$NODE_1"; do + echo "--- $n ---" + if ! kubectl --request-timeout=120s debug -q "node/$n" -- chroot /host lsblk -o NAME,PATH,SIZE,ROTA,TYPE,FSTYPE; then + echo " Could not list block devices on $n (kubectl debug failed). Check cluster access, then re-run: $0 -l" >&2 + fi + echo "" + done + echo "Same path on both nodes: -d " + echo "Different paths (same size): -d0 -d1 " +} + +# Map user device path -> stable disk by-id symlink for DRBD config on that node. +# Multiple by-id names can resolve to the same canonical device; sort|head -1 picks one deterministically. +resolve_disk_path_on_node() { + local node="$1" device_path="$2" + kubectl debug -q "node/$node" -- chroot /host env "DRBD_BLOCK_DEV=${device_path}" bash -c ' +if ! CANON=$(readlink -f "$DRBD_BLOCK_DEV" 2>/dev/null); then + CANON="$DRBD_BLOCK_DEV" +fi +for id in /dev/disk/by-id/*; do + if [[ ! -e "$id" ]]; then + continue + fi + if [[ "$(readlink -f "$id" 2>/dev/null)" == "$CANON" ]]; then echo "$id"; fi +done | sort -u | head -n 1 +' 2>/dev/null | tail -n 1 +} + +print_config() { + echo "" + msg "Configuration" + local _lw=18 + printf ' %-*s %s\n' "$_lw" "Nodes:" "$NODE_0 ($NODE_0_IP), $NODE_1 ($NODE_1_IP)" + if [[ -n "$BACKING_PATH" ]]; then + printf ' %-*s %s (same path on both nodes)\n' "$_lw" "Backing device:" "$BACKING_PATH" + else + printf ' %-*s %s\n' "$_lw" "Backing devices:" "per-node paths" + printf ' %-*s %s: %s\n' "$_lw" "" "$NODE_0" "$BACKING_PATH_NODE0" + printf ' %-*s %s: %s\n' "$_lw" "" "$NODE_1" "$BACKING_PATH_NODE1" + fi + printf ' %-*s %s\n' "$_lw" "DRBD Config path:" "$DRBD_CONF_PATH" + printf ' %-*s %s\n' "$_lw" "DRBD Dir path:" "$DRBD_DIR_PATH" + printf ' %-*s %s\n' "$_lw" "DRBD Resource:" "$DRBD_RESOURCE" + printf ' %-*s %s\n' "$_lw" "DRBD Device:" "$DRBD_DEVICE" + printf ' %-*s %s\n' "$_lw" "DRBD Port:" "$DRBD_PORT" + echo "" +} + + +_lsblk_one_line() { + local node="$1" device_path="$2" + kubectl debug -q "node/$node" -- chroot /host lsblk -ndo SIZE,RO,ROTA "$device_path" 2>/dev/null | tr -s ' ' | head -1 +} + +# validate the backing device paths and resolve the disk by-id symlink for DRBD config on that node. +validate_and_resolve_disks() { + local p0 p1 row0 row1 size0 ro0 rota0 size1 ro1 rota1 + if [[ -n "$BACKING_PATH" ]]; then + p0="$BACKING_PATH" + p1="$BACKING_PATH" + else + p0="$BACKING_PATH_NODE0" + p1="$BACKING_PATH_NODE1" + fi + + msg "Checking backing device paths..." + row0=$(_lsblk_one_line "$NODE_0" "$p0") + row1=$(_lsblk_one_line "$NODE_1" "$p1") + if [[ -z "$row0" ]]; then + die "device path $p0 not found on $NODE_0" + fi + if [[ -z "$row1" ]]; then + die "device path $p1 not found on $NODE_1" + fi + + read -r size0 ro0 rota0 <<<"$row0" + read -r size1 ro1 rota1 <<<"$row1" + if [[ "$ro0" != "0" ]]; then + die "device path $p0 on $NODE_0 is read-only" + fi + if [[ "$ro1" != "0" ]]; then + die "device path $p1 on $NODE_1 is read-only" + fi + if [[ "$rota0" != "0" ]]; then + die "device path $p0 on $NODE_0 must be non-rotational (SSD/NVMe; lsblk ROTA 0), not rotational HDD (ROTA=${rota0:-?})" + fi + if [[ "$rota1" != "0" ]]; then + die "device path $p1 on $NODE_1 must be non-rotational (SSD/NVMe; lsblk ROTA 0), not rotational HDD (ROTA=${rota1:-?})" + fi + if [[ "$size0" != "$size1" ]]; then + die "backing device path size mismatch: $NODE_0 $size0 vs $NODE_1 $size1" + fi + + echo " $NODE_0: $p0 $size0" + echo " $NODE_1: $p1 $size1" + msg "Backing device paths OK." + + msg "Resolving device paths to /dev/disk/by-id for DRBD config" + DISK_RESOLVED_NODE0=$(resolve_disk_path_on_node "$NODE_0" "$p0") + DISK_RESOLVED_NODE1=$(resolve_disk_path_on_node "$NODE_1" "$p1") + if [[ -z "$DISK_RESOLVED_NODE0" ]]; then + die "no /dev/disk/by-id symlink for device path $p0 on $NODE_0" + fi + if [[ -z "$DISK_RESOLVED_NODE1" ]]; then + die "no /dev/disk/by-id symlink for device path $p1 on $NODE_1" + fi + echo " $NODE_0: $p0 -> $DISK_RESOLVED_NODE0" + echo " $NODE_1: $p1 -> $DISK_RESOLVED_NODE1" +} + + +# check if the DRBD kernel modules are loaded on the node +node_has_drbd_kmods() { + local node="$1" + local out + if ! out=$(kubectl debug -q "node/$node" --image="${DEBUG_IMAGE}" -- chroot /host cat /proc/modules 2>/dev/null); then + return 1 + fi + if ! echo "$out" | grep -qE '^drbd[[:space:]]'; then + return 1 + fi + if ! echo "$out" | grep -qE '^drbd_transport_tcp[[:space:]]'; then + return 1 + fi + return 0 +} + +# wait for the DRBD kernel modules to load on both nodes +wait_for_modules() { + if node_has_drbd_kmods "$NODE_0" && node_has_drbd_kmods "$NODE_1"; then + msg "DRBD kernel modules are already loaded on both nodes." + return 0 + fi + + # Success: /proc/modules on each node contains drbd and drbd_transport_tcp lines (see node_has_drbd_kmods). + msg "Waiting for DRBD kernel modules to load on both nodes (up to 10 min)..." + _wait_begin + local i + for i in $(seq 1 60); do + if node_has_drbd_kmods "$NODE_0" && node_has_drbd_kmods "$NODE_1"; then + _wait_succeeded "DRBD kernel modules are loaded on both nodes" + return 0 + fi + if [[ "$i" -eq 60 ]]; then + die "DRBD modules failed to load after 10 minutes. Check: kubectl get module,pods -n rook-ceph; kubectl debug -q node/${NODE_0} --image='${DEBUG_IMAGE}' -- chroot /host cat /proc/modules | grep -E '^drbd|drbd_transport'" + fi + sleep 10 + done +} + +# Run drbdadm on a node via podman using the DRBD image; mounts host drbd.conf and drbd.d. +drbdctl() { + local node="$1" + shift + if ! kubectl debug -q "node/$node" --image="${DEBUG_IMAGE}" -- chroot /host \ + podman run --rm --privileged \ + -v /dev:/dev \ + -v "${DRBD_CONF_PATH}:${DRBD_CONF_PATH}" \ + -v "${DRBD_DIR_PATH}:${DRBD_DIR_PATH}" \ + --hostname "$node" \ + --net host \ + "${DRBD_IMAGE}" \ + drbdadm -c "${DRBD_CONF_PATH}" "$@"; then + echo "DRBD command failed on node $node: drbdadm $*" >&2 + return 1 + fi +} + +# True when the node has a role (Primary/Secondary) for the DRBD resource. +drbd_node_has_role() { + local node="$1" status_out + if ! status_out=$(drbdctl "$node" status "${DRBD_RESOURCE}" 2>&1); then + return 1 + fi + echo "$status_out" | grep -qiE 'role:[[:space:]]*(Primary|Secondary)' +} + +# True when both nodes show a role (Primary/Secondary) for the DRBD resource. +drbd_resource_up_on_both_nodes() { + drbd_node_has_role "$NODE_0" && drbd_node_has_role "$NODE_1" +} + +# configure the DRBD resource on both nodes +configure_drbd() { + if drbd_resource_up_on_both_nodes; then + msg "DRBD resource is already up on both nodes" + return 0 + fi + + msg "Configuring DRBD resource \"${DRBD_RESOURCE}\" on ${NODE_0} and ${NODE_1}." + local DRBD_RES_BODY DRBD_RES_B64 DRBD_MAIN_B64 + DRBD_RES_BODY="global { usage-count no; } +common { + net { protocol C; after-sb-0pri discard-zero-changes; after-sb-1pri discard-secondary; } + disk { on-io-error pass_on; } + options { on-no-data-accessible suspend-io; } +} +resource ${DRBD_RESOURCE} { + on ${NODE_0} { + device ${DRBD_DEVICE}; + disk ${DISK_RESOLVED_NODE0}; + address ${NODE_0_IP}:${DRBD_PORT}; + node-id 0; + meta-disk internal; + } + on ${NODE_1} { + device ${DRBD_DEVICE}; + disk ${DISK_RESOLVED_NODE1}; + address ${NODE_1_IP}:${DRBD_PORT}; + node-id 1; + meta-disk internal; + } +}" + + DRBD_RES_B64=$(printf '%s' "$DRBD_RES_BODY" | base64 | tr -d '\n') + DRBD_MAIN_B64=$(printf '%s' "include \"${DRBD_DIR_PATH}/*.res\";" | base64 | tr -d '\n') + + local node res_path conf_dir + res_path="${DRBD_DIR_PATH}/${DRBD_RESOURCE}.res" + conf_dir="$(dirname "${DRBD_CONF_PATH}")" + for node in "$NODE_0" "$NODE_1"; do + msg "Node ${node}: writing DRBD config files to the host..." + if ! kubectl debug -q "node/$node" --image="${DEBUG_IMAGE}" -- chroot /host bash -c " +mkdir -p '${conf_dir}' '${DRBD_DIR_PATH}' /var/lib/drbd +echo '${DRBD_RES_B64}' | base64 -d > '${res_path}' +echo '${DRBD_MAIN_B64}' | base64 -d > '${DRBD_CONF_PATH}' +"; then + die "failed to write DRBD config on $node" + fi + + if drbd_node_has_role "$node"; then + msg "Node ${node}: resource already has a role on this host; running drbdadm adjust..." + if ! drbdctl "$node" adjust "${DRBD_RESOURCE}"; then + die "drbdadm adjust failed on $node" + fi + else + msg "Node ${node}: creating DRBD metadata then drbdadm up..." + if ! drbdctl "$node" create-md "${DRBD_RESOURCE}" --force; then + die "drbdadm create-md failed on $node" + fi + if ! drbdctl "$node" up "${DRBD_RESOURCE}"; then + die "drbdadm up failed on $node" + fi + fi + done + msg "DRBD resource is configured and the replication link is up." +} + +# check if the DRBD resource is fully replicated on both nodes +drbd_resource_fully_replicated() { + local n status_out + for n in "$NODE_0" "$NODE_1"; do + if ! status_out=$(drbdctl "$n" status "${DRBD_RESOURCE}" 2>&1); then + return 1 + fi + if ! echo "$status_out" | grep -q "disk:UpToDate"; then + return 1 + fi + if ! echo "$status_out" | grep -q "peer-disk:UpToDate"; then + return 1 + fi + done + return 0 +} + +# Check status of replication each 30s and wait for it to complete. +sync_drbd() { + # Transient Primary on first sorted node for sync; then demote to Secondary on both nodes. + local PRIMARY_NODE="$NODE_0" + DRBD_PROMOTED_MASTER0_THIS_RUN=0 + + if drbd_resource_fully_replicated; then + msg "DRBD data is already fully replicated (UpToDate on both nodes); skipping primary/sync wait." + return 0 + fi + + msg "Promoting $PRIMARY_NODE to Primary to run initial replication..." + if ! drbdctl "$PRIMARY_NODE" primary --force "$DRBD_RESOURCE"; then + die "drbdadm primary failed on $PRIMARY_NODE" + fi + DRBD_PROMOTED_MASTER0_THIS_RUN=1 + + # Poll drbdadm status on the transient primary until peer-disk:UpToDate (full sync). Example + # fragment while syncing: lines with disk:/peer-disk: and possibly done:12.34% for progress. + #sleep 600 + # msg "Waiting for full DRBD sync (up to 30 min; progress every 30s when available)..." + # _wait_begin + # local i STATUS PROGRESS + # for i in $(seq 1 60); do + # STATUS=$(drbdctl "$PRIMARY_NODE" status "$DRBD_RESOURCE" 2>/dev/null) + # if echo "$STATUS" | grep -q "peer-disk:UpToDate"; then + # _wait_succeeded "Initial replication finished; both nodes report UpToDate" + # return 0 + # fi + # PROGRESS=$(echo "$STATUS" | grep -o 'done:[0-9.]*' | head -1 | cut -d: -f2) + # if [[ -n "$PROGRESS" ]]; then + # msg "Replication progress: ${PROGRESS}%" + # fi + # if [[ "$i" -eq 60 ]]; then + # die "DRBD sync timed out after 30m. Status: $STATUS" + # fi + # sleep 30 + # done +} + +# create the filesystem over the DRBD device +create_filesystem_over_drbd() { + local PRIMARY_NODE="$NODE_0" + local fstype + if ! fstype=$(kubectl debug -q "node/$PRIMARY_NODE" --image="${DEBUG_IMAGE}" -- chroot /host blkid -s TYPE -o value "${DRBD_DEVICE}" 2>/dev/null | tr -d ' \n'); then + fstype="" + fi + if [[ "$fstype" == "xfs" ]]; then + msg "${DRBD_DEVICE} already has XFS; skipping mkfs (re-run safe)." + return 0 + fi + + msg "Formatting ${DRBD_DEVICE} with XFS (mkfs.xfs -f; overwrites any existing signature)..." + kubectl debug -q "node/$PRIMARY_NODE" --image="${DEBUG_IMAGE}" -- chroot /host sudo mkfs.xfs -f "${DRBD_DEVICE}" + msg "XFS created on ${DRBD_DEVICE}." +} + +# Demote the transient primary used for initial sync back to Secondary. +make_both_node_secondary() { + if [[ "${DRBD_PROMOTED_MASTER0_THIS_RUN:-0}" -ne 1 ]]; then + return 0 + fi + + local PRIMARY_NODE="$NODE_0" + local i ROLE + + ROLE=$(drbdctl "$PRIMARY_NODE" role "${DRBD_RESOURCE}" 2>/dev/null | cut -d/ -f1) + if [[ "$ROLE" == "Secondary" ]]; then + return 0 + fi + + msg "Demoting $PRIMARY_NODE to Secondary." + if ! drbdctl "$PRIMARY_NODE" secondary "$DRBD_RESOURCE"; then + die "drbdadm secondary failed on $PRIMARY_NODE" + fi + + msg "Waiting for $PRIMARY_NODE to report Secondary role (up to 40s)..." + _wait_begin + for i in $(seq 1 20); do + ROLE=$(drbdctl "$PRIMARY_NODE" role "${DRBD_RESOURCE}" 2>/dev/null | cut -d/ -f1) + if [[ "$ROLE" == "Secondary" ]]; then + _wait_succeeded "$PRIMARY_NODE is now Secondary" + return 0 + fi + sleep 2 + done + die "Node $PRIMARY_NODE did not become Secondary" +} + +# setup the DRBD auto-start DaemonSet to keep the DRBD resource up on both nodes +setup_drbd_autostart() { + if kubectl get daemonset "${AUTOSTART_DAEMONSET_NAME}" -n "${AUTOSTART_DAEMONSET_NS}" &>/dev/null; then + msg "DRBD auto-start DaemonSet already exists." + return 0 + fi + + msg "Creating DRBD auto-start DaemonSet in namespace ${AUTOSTART_DAEMONSET_NS}..." + kubectl create namespace "${AUTOSTART_DAEMONSET_NS}" --dry-run=client -o yaml | kubectl apply -f - >/dev/null 2>&1 + kubectl create serviceaccount drbd-autostart -n "${AUTOSTART_DAEMONSET_NS}" --dry-run=client -o yaml | kubectl apply -f - >/dev/null 2>&1 + # kubectl adm policy add-scc-to-user privileged -z drbd-autostart -n "${AUTOSTART_DAEMONSET_NS}" >/dev/null + + kubectl apply -f - >/dev/null </dev/null; then + echo "DRBD resource ${DRBD_RESOURCE} is already up" + else + echo "Starting DRBD resource ${DRBD_RESOURCE}..." + if ! drbdadm -c "${DRBD_CONF_PATH}" up ${DRBD_RESOURCE}; then + echo "Warning: drbdadm up failed, will retry" + fi + fi + if ! drbdadm -c "${DRBD_CONF_PATH}" status ${DRBD_RESOURCE}; then + : + fi + sleep 60 + done +EOF + + kubectl apply -f - >/dev/null </dev/null); then + READY_COUNT=0 + fi + if [[ -z "$READY_COUNT" ]]; then + READY_COUNT=0 + fi + READY_COUNT=$((0 + READY_COUNT)) + if [[ "$READY_COUNT" -eq 2 ]]; then + _wait_succeeded "DRBD auto-start DaemonSet is running on both nodes" + return 0 + fi + if [[ "$i" -eq 60 ]]; then + die "DaemonSet not ready (kubectl get ds,pods -n ${AUTOSTART_DAEMONSET_NS})" + fi + sleep 5 + done +} + +# create the success ConfigMap to save the setup summary for further consumption. +create_success_configmap() { + msg "Saving setup summary to ConfigMap ${OUTPUT_CM_NS}/${OUTPUT_CM_NAME}" + if ! kubectl create namespace "${OUTPUT_CM_NS}" --dry-run=client -o yaml | kubectl apply -f - >/dev/null 2>&1; then + : + fi + + local bd0 bd1 + if [[ -n "$BACKING_PATH" ]]; then + bd0="$BACKING_PATH" + bd1="$BACKING_PATH" + else + bd0="$BACKING_PATH_NODE0" + bd1="$BACKING_PATH_NODE1" + fi + + kubectl apply -f - >/dev/null < DRBD setup completed successfully <--" + echo "" + echo "Post-install DRBD status on ${NODE_0} (repeat with ${NODE_1}):" + echo " kubectl debug -q node/${NODE_0} --image='${DEBUG_IMAGE}' -- chroot /host podman run --rm --privileged \\" + echo " -v /dev:/dev -v ${DRBD_CONF_PATH}:${DRBD_CONF_PATH} -v ${DRBD_DIR_PATH}:${DRBD_DIR_PATH} \\" + echo " --hostname ${NODE_0} --net host ${DRBD_IMAGE} drbdadm -c ${DRBD_CONF_PATH} status ${DRBD_RESOURCE}" + echo "" +} + +main() { + parse_args "$@" + check_prerequisites # check if the prerequisites are met + detect_nodes # detect the nodes in the cluster + + if [[ "$LIST_DEVICES_ONLY" -eq 1 ]]; then + list_devices # list the block devices on the nodes + exit 0 + fi + + print_config # print the configuration + #validate_and_resolve_disks # validate the disks and resolve the disk by-id symlink for DRBD config on that node + configure_drbd # configure the DRBD resource on both nodes + sync_drbd # sync the DRBD resource on both nodes + create_filesystem_over_drbd # create the filesystem over the DRBD device + make_both_node_secondary # make both nodes secondary + setup_drbd_autostart # setup the DRBD auto-start DaemonSet to keep the DRBD resource up on both nodes + create_success_configmap # create the success ConfigMap to save the setup summary for further consumption + print_success # print the success message +} + +main "$@" diff --git a/deploy/examples/pool-tnf.yaml b/deploy/examples/pool-tnf.yaml new file mode 100644 index 000000000..138a2e558 --- /dev/null +++ b/deploy/examples/pool-tnf.yaml @@ -0,0 +1,10 @@ +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: replicapool + namespace: rook-ceph +spec: + failureDomain: host + replicated: + size: 2 + requireSafeReplicaSize: false