From f16c8dc060c1c5bda225150d5dfa51fd7bb47bcd Mon Sep 17 00:00:00 2001 From: Ashish Ranjan Date: Fri, 6 Sep 2019 13:40:22 +0530 Subject: [PATCH] fix(osd prepare): osd prepare job getting into panic state Osd pods getting into panic state due to index out of range error. logs of osd prepare job: ``` [aranjan@localhost:~]$ oc logs -f rook-ceph-osd-prepare-set1-0-data-6f2nh-zh4zl 2019-09-06 07:48:11.448085 I | cephcmd: desired devices to configure osds: [{Name:/mnt/set1-0-data-6f2nh OSDsPerDevice:1 MetadataDevice: DatabaseSizeMB:0 DeviceClass: IsFilter:false}] 2019-09-06 07:48:11.449841 I | rookcmd: starting Rook v1.0.0-623.gc18d575c.dirty with arguments '/rook/rook ceph osd provision' 2019-09-06 07:48:11.449866 I | rookcmd: flag values: --cluster-id=308f232a-d07a-11e9-96ba-0adfe2bb9bfc, --data-device-filter=, --data-devices=/mnt/set1-0-data-6f2nh, --data-directories=, --encrypted-device=false, --force-format=false, -- help=false, --location=, --log-flush-frequency=5s, --log-level=INFO, --metadata-device=, --node-name=set1-0-data-6f2nh, --operator-image=, --osd-database-size=0, --osd-journal-size=5120, --osd-store=, --osd-wal-size=576, --osds-per-devic e=1, --pvc-backed-osd=true, --service-account= 2019-09-06 07:48:11.449875 I | op-mon: parsing mon endpoints: a=172.30.150.154:6789 2019-09-06 07:48:11.470058 I | cephconfig: writing config file /var/lib/rook/rook-ceph/rook-ceph.config 2019-09-06 07:48:11.470797 I | cephconfig: generated admin config in /var/lib/rook/rook-ceph 2019-09-06 07:48:11.470989 I | cephosd: discovering hardware 2019-09-06 07:48:11.471030 I | exec: Running command: lsblk /mnt/set1-0-data-6f2nh --bytes --nodeps --pairs --output SIZE,ROTA,RO,TYPE,PKNAME 2019-09-06 07:48:11.473887 I | exec: Running command: sgdisk --print /mnt/set1-0-data-6f2nh 2019-09-06 07:48:11.476048 I | cephosd: creating and starting the osds 2019-09-06 07:48:11.476070 I | exec: Running command: lsblk /mnt/set1-0-data-6f2nh --bytes --pairs --output NAME,SIZE,TYPE,PKNAME 2019-09-06 07:48:11.478030 I | sys: Output: NAME="xvdbg" SIZE="10737418240" TYPE="disk" PKNAME="" 2019-09-06 07:48:11.478056 I | cephosd: /mnt/set1-0-data-6f2nh found in the desired devices 2019-09-06 07:48:11.488231 I | cephosd: configuring osd devices: {"Entries":{"/mnt/set1-0-data-6f2nh":{"Data":-1,"Metadata":null,"Config":{"Name":"/mnt/set1-0-data-6f2nh","OSDsPerDevice":1,"MetadataDevice":"","DatabaseSizeMB":0,"DeviceCl ass":"","IsFilter":false},"LegacyPartitionsFound":false}}} 2019-09-06 07:48:11.492191 I | cephosd: device /mnt/set1-0-data-6f2nh to be configured by ceph-volume 2019-09-06 07:48:11.492213 I | cephosd: 0/0 pre-ceph-volume osd devices succeeded on this node 2019-09-06 07:48:11.492399 I | exec: Running command: ceph auth get-or-create-key client.bootstrap-osd mon allow profile bootstrap-osd --connect-timeout=15 --cluster=rook-ceph --conf=/var/lib/rook/rook-ceph/rook-ceph.config --keyring=/va r/lib/rook/rook-ceph/client.admin.keyring --format json --out-file /tmp/973674470 2019-09-06 07:48:12.020858 I | cephosd: Successfully updated lvm config file 2019-09-06 07:48:12.020894 I | cephosd: configuring new device /mnt/set1-0-data-6f2nh 2019-09-06 07:48:12.020908 I | exec: Running command: stdbuf -oL ceph-volume lvm prepare --data /mnt/set1-0-data-6f2nh 2019-09-06 07:48:55.172366 I | cephosd: panic: runtime error: index out of range goroutine 1 [running]: github.com/rook/rook/pkg/daemon/ceph/osd.getLVPath(0x0, 0x0, 0x1e3c60d, 0x2) /home/aranjan/Project/go/src/github.com/rook/rook/pkg/daemon/ceph/osd/volume.go:128 +0x2fd github.com/rook/rook/pkg/daemon/ceph/osd.(*OsdAgent).initializeBlockPVC(0xc000829c30, 0xc00072e1e0, 0xc000736e70, 0x1e4927d, 0xd, 0x0, 0x0) /home/aranjan/Project/go/src/github.com/rook/rook/pkg/daemon/ceph/osd/volume.go:108 +0x50b github.com/rook/rook/pkg/daemon/ceph/osd.(*OsdAgent).configureCVDevices(0xc000829c30, 0xc00072e1e0, 0xc000736e70, 0x38, 0xc000829310, 0x2, 0x2, 0x0) /home/aranjan/Project/go/src/github.com/rook/rook/pkg/daemon/ceph/osd/volume.go:70 +0x33a github.com/rook/rook/pkg/daemon/ceph/osd.(*OsdAgent).configureDevices(0xc000829c30, 0xc00072e1e0, 0xc0000bc370, 0x1c, 0xc0008296e8, 0x1, 0x1, 0xd) /home/aranjan/Project/go/src/github.com/rook/rook/pkg/daemon/ceph/osd/agent.go:213 +0x97d github.com/rook/rook/pkg/daemon/ceph/osd.Provision(0xc00072e1e0, 0xc000829c30, 0xc0000560d0, 0x24) /home/aranjan/Project/go/src/github.com/rook/rook/pkg/daemon/ceph/osd/daemon.go:166 +0xeb2 github.com/rook/rook/cmd/rook/ceph.prepareOSD(0x359be60, 0x35cd220, 0x0, 0x0, 0x0, 0x0) /home/aranjan/Project/go/src/github.com/rook/rook/cmd/rook/ceph/osd.go:252 +0x838 github.com/rook/rook/vendor/github.com/spf13/cobra.(*Command).execute(0x359be60, 0x35cd220, 0x0, 0x0, 0x359be60, 0x35cd220) /home/aranjan/Project/go/src/github.com/rook/rook/vendor/github.com/spf13/cobra/command.go:762 +0x465 github.com/rook/rook/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0x359e200, 0xc0006eff30, 0xb, 0xb) /home/aranjan/Project/go/src/github.com/rook/rook/vendor/github.com/spf13/cobra/command.go:852 +0x2ec github.com/rook/rook/vendor/github.com/spf13/cobra.(*Command).Execute(...) /home/aranjan/Project/go/src/github.com/rook/rook/vendor/github.com/spf13/cobra/command.go:800 main.main() /home/aranjan/Project/go/src/github.com/rook/rook/cmd/rook/main.go:35 +0x12e ``` Signed-off-by: Ashish Ranjan --- pkg/daemon/ceph/osd/volume.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index 4c3212122..808a279de 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -124,7 +124,8 @@ func getLVPath(op string) string { tmp = sys.Grep(op, "Logical volume") lvtmp := strings.Split(tmp, "\"") - if len(vgtmp) > 0 && len(lvtmp) > 0 { + + if len(vgtmp) >= 2 && len(lvtmp) >= 2 { if sys.Grep(vgtmp[1], "ceph") != "" && sys.Grep(lvtmp[1], "osd-block") != "" { return fmt.Sprintf("/dev/%s/%s", vgtmp[1], lvtmp[1]) }