From 76ed166e86de3da532b137a9cd051d69f9ec92b3 Mon Sep 17 00:00:00 2001 From: crombus Date: Mon, 18 Jan 2021 18:08:42 +0530 Subject: [PATCH] ceph: add ceph archive cmd in osd removal Add ceph archive cmd in osd removal for Silencing the warning when osd is removed. Signed-off-by: crombus --- pkg/daemon/ceph/client/crash.go | 90 ++++++++++++++++++++++++++++ pkg/daemon/ceph/client/crash_test.go | 52 ++++++++++++++++ pkg/daemon/ceph/osd/remove.go | 26 ++++++++ 3 files changed, 168 insertions(+) create mode 100644 pkg/daemon/ceph/client/crash.go create mode 100644 pkg/daemon/ceph/client/crash_test.go diff --git a/pkg/daemon/ceph/client/crash.go b/pkg/daemon/ceph/client/crash.go new file mode 100644 index 000000000..8f8d478a9 --- /dev/null +++ b/pkg/daemon/ceph/client/crash.go @@ -0,0 +1,90 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "encoding/json" + + "github.com/pkg/errors" + "github.com/rook/rook/pkg/clusterd" +) + +// CrashList is go representation of the "ceph crash ls" command output +type CrashList struct { + ID string `json:"crash_id"` + Entity string `json:"entity_name"` + Timestamp string `json:"timestamp"` + ProcessName string `json:"process_name,omitempty"` + CephVersion string `json:"ceph_version,omitempty"` + UtsnameHostname string `json:"utsname_hostname,omitempty"` + UtsnameSysname string `json:"utsname_sysname,omitempty"` + UtsnameRelease string `json:"utsname_release,omitempty"` + UtsnameVersion string `json:"utsname_version,omitempty"` + UtsnameMachine string `json:"utsname_machine,omitempty"` + OsName string `json:"os_name,omitempty"` + OsID string `json:"os_id,omitempty"` + OsVersionID string `json:"os_version_id,omitempty"` + OsVersion string `json:"os_version,omitempty"` + AssertCondition string `json:"assert_condition,omitempty"` + AssertFunc string `json:"assert_func,omitempty"` + AssertLine int `json:"assert_line,omitempty"` + AssertFile string `json:"assert_file,omitempty"` + AssertThreadName string `json:"assert_thread_name,omitempty"` + AssertMsg string `json:"assert_msg,omitempty"` + IoError bool `json:"io_error,omitempty"` + IoErrorDevname string `json:"io_error_devname,omitempty"` + IoErrorPath string `json:"io_error_path,omitempty"` + IoErrorCode int `json:"io_error_code,omitempty"` + IoErrorOptype int `json:"io_error_optype,omitempty"` + IoErrorOffset int `json:"io_error_offset,omitempty"` + IoErrorLength int `json:"iio_error_length,omitempty"` + Backtrace []string `json:"backtrace,omitempty"` +} + +// GetCrashList gets the list of Crashes. +func GetCrashList(context *clusterd.Context, clusterInfo *ClusterInfo) ([]CrashList, error) { + crashargs := []string{"crash", "ls"} + output, err := NewCephCommand(context, clusterInfo, crashargs).Run() + if err != nil { + return nil, errors.Wrap(err, "failed to list ceph crash") + } + var crash []CrashList + err = json.Unmarshal(output, &crash) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal crash ls response. %s", string(output)) + } + return crash, err +} + +// ArchiveCrash archives the crash with respective crashID +func ArchiveCrash(context *clusterd.Context, clusterInfo *ClusterInfo, crashID string) error { + crashsilenceargs := []string{"crash", "archive", crashID} + _, err := NewCephCommand(context, clusterInfo, crashsilenceargs).Run() + if err != nil { + return errors.Wrapf(err, "failed to archive crash %q", crashID) + } + return nil +} + +// GetCrash gets the crash list +func GetCrash(context *clusterd.Context, clusterInfo *ClusterInfo) ([]CrashList, error) { + crash, err := GetCrashList(context, clusterInfo) + if err != nil { + return nil, errors.Wrap(err, "failed to list ceph crash") + } + return crash, nil +} diff --git a/pkg/daemon/ceph/client/crash_test.go b/pkg/daemon/ceph/client/crash_test.go new file mode 100644 index 000000000..8e0e17c33 --- /dev/null +++ b/pkg/daemon/ceph/client/crash_test.go @@ -0,0 +1,52 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "testing" + + "github.com/pkg/errors" + "github.com/rook/rook/pkg/clusterd" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" +) + +var ( + fakecrash = `[ + { + "crash_id": "2020-11-09_13:58:08.230130Z_ca918f58-c078-444d-a91a-bd972c14c155", + "timestamp": "2020-11-09 13:58:08.230130Z", + "process_name": "ceph-osd", + "entity_name": "osd.0" + } + ]` +) + +func TestCephCrash(t *testing.T) { + executor := &exectest.MockExecutor{} + context := &clusterd.Context{Executor: executor} + executor.MockExecuteCommandWithOutputFile = func(command, outputFile string, args ...string) (string, error) { + logger.Infof("ExecuteCommandWithOutputFile: %s %v", command, args) + if args[0] == "crash" && args[1] == "ls" { + return fakecrash, nil + } + return "", errors.Errorf("unexpected ceph command %q", args) + } + crash, err := GetCrashList(context, AdminClusterInfo("mycluster")) + assert.NoError(t, err) + assert.Equal(t, 1, len(crash)) +} diff --git a/pkg/daemon/ceph/osd/remove.go b/pkg/daemon/ceph/osd/remove.go index a989561d4..7d436afe8 100644 --- a/pkg/daemon/ceph/osd/remove.go +++ b/pkg/daemon/ceph/osd/remove.go @@ -137,6 +137,32 @@ func removeOSD(clusterdContext *clusterd.Context, clusterInfo *client.ClusterInf if err != nil { logger.Errorf("failed to remove CRUSH host %q. %v", hostName, err) } + // call archiveCrash to silence crash warning in ceph health if any + archiveCrash(clusterdContext, clusterInfo, osdID) logger.Infof("completed removal of OSD %d", osdID) } + +func archiveCrash(clusterdContext *clusterd.Context, clusterInfo *client.ClusterInfo, osdID int) { + // The ceph health warning should be silenced by archiving the crash + crash, err := client.GetCrash(clusterdContext, clusterInfo) + if err != nil { + logger.Errorf("failed to list ceph crash. %v", err) + return + } + if crash != nil { + logger.Info("no ceph crash to silence") + return + } + var crashID string + for _, c := range crash { + if c.Entity == fmt.Sprintf("osd.%d", osdID) { + crashID = c.ID + break + } + } + err = client.ArchiveCrash(clusterdContext, clusterInfo, crashID) + if err != nil { + logger.Errorf("failed to archive the crash %q. %v", crashID, err) + } +}