From 6e171c32185d985d90f1cb06237e2e05282be5f2 Mon Sep 17 00:00:00 2001 From: xiaobaowen Date: Sun, 26 Mar 2023 12:04:05 -0400 Subject: [PATCH] mgr: disable the prometheus module by crds Signed-off-by: xiaobaowen (cherry picked from commit e7254c2a1b7af0d0500d86c633433631adb27afc) --- .../charts/rook-ceph/templates/resources.yaml | 8 ++ deploy/examples/crds.yaml | 8 ++ pkg/apis/ceph.rook.io/v1/types.go | 10 ++ pkg/operator/ceph/cluster/mgr/dashboard.go | 13 +-- pkg/operator/ceph/cluster/mgr/mgr.go | 58 ++++++++++- pkg/operator/ceph/cluster/mgr/mgr_test.go | 97 +++++++++++++++++++ 6 files changed, 178 insertions(+), 16 deletions(-) diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 2b0ceb967..0dd90f80d 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -1867,6 +1867,14 @@ spec: maximum: 65535 minimum: 0 type: integer + interval: + description: Interval determines prometheus scrape interval + type: string + port: + description: Port is the prometheus server port + maximum: 65535 + minimum: 0 + type: integer type: object network: description: Network related configuration diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 213dde29b..ea2eea9ab 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -1865,6 +1865,14 @@ spec: maximum: 65535 minimum: 0 type: integer + interval: + description: Interval determines prometheus scrape interval + type: string + port: + description: Port is the prometheus server port + maximum: 65535 + minimum: 0 + type: integer type: object network: description: Network related configuration diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 633f3f54c..72ec1675a 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -337,6 +337,16 @@ type MonitoringSpec struct { // +kubebuilder:validation:Maximum=65535 // +optional ExternalMgrPrometheusPort uint16 `json:"externalMgrPrometheusPort,omitempty"` + + // Port is the prometheus server port + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=65535 + // +optional + Port int `json:"port,omitempty"` + + // Interval determines prometheus scrape interval + // +optional + Interval *metav1.Duration `json:"interval,omitempty"` } // ClusterStatus represents the status of a Ceph cluster diff --git a/pkg/operator/ceph/cluster/mgr/dashboard.go b/pkg/operator/ceph/cluster/mgr/dashboard.go index 7abf6c25a..4d8441c43 100644 --- a/pkg/operator/ceph/cluster/mgr/dashboard.go +++ b/pkg/operator/ceph/cluster/mgr/dashboard.go @@ -104,7 +104,7 @@ func (c *Cluster) configureDashboardModules() error { } if hasChanged { logger.Info("dashboard config has changed. restarting the dashboard module") - return c.restartDashboard() + return c.restartMgrModule(dashboardModuleName) } return nil } @@ -304,14 +304,3 @@ func decodeSecret(secret *v1.Secret) (string, error) { } return string(password), nil } - -func (c *Cluster) restartDashboard() error { - logger.Info("restarting the mgr module") - if err := client.MgrDisableModule(c.context, c.clusterInfo, dashboardModuleName); err != nil { - return errors.Wrapf(err, "failed to disable mgr module %q.", dashboardModuleName) - } - if err := client.MgrEnableModule(c.context, c.clusterInfo, dashboardModuleName, true); err != nil { - return errors.Wrapf(err, "failed to enable mgr module %q.", dashboardModuleName) - } - return nil -} diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go index efd6d2a60..5b0d67ee9 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr.go +++ b/pkg/operator/ceph/cluster/mgr/mgr.go @@ -20,6 +20,7 @@ package mgr import ( "fmt" "path" + "strconv" "github.com/banzaicloud/k8s-objectmatcher/patch" "github.com/coreos/pkg/capnslog" @@ -318,7 +319,7 @@ func (c *Cluster) updateServiceSelectors(activeDaemon string) error { func (c *Cluster) configureModules(daemonIDs []string) { // Configure the modules asynchronously so we can complete all the configuration much sooner. - startModuleConfiguration("prometheus", c.enablePrometheusModule) + startModuleConfiguration("prometheus", c.configurePrometheusModule) startModuleConfiguration("dashboard", c.configureDashboardModules) // It is a bit confusing but modules that are in the "always_on_modules" list @@ -339,9 +340,58 @@ func startModuleConfiguration(description string, configureModules func() error) } // Ceph docs about the prometheus module: http://docs.ceph.com/docs/master/mgr/prometheus/ -func (c *Cluster) enablePrometheusModule() error { - if err := cephclient.MgrEnableModule(c.context, c.clusterInfo, PrometheusModuleName, true); err != nil { - return errors.Wrap(err, "failed to enable mgr prometheus module") +func (c *Cluster) configurePrometheusModule() error { + if c.spec.Monitoring.Enabled { + if err := cephclient.MgrEnableModule(c.context, c.clusterInfo, PrometheusModuleName, true); err != nil { + return errors.Wrap(err, "failed to enable mgr prometheus module") + } + } else { + if err := cephclient.MgrDisableModule(c.context, c.clusterInfo, PrometheusModuleName); err != nil { + logger.Errorf("failed to disable mgr prometheus module. %v", err) + } + return nil + } + + var ( + err error + portHasChanged bool + intervalHasChanged bool + daemonID = "mgr" + ) + monStore := config.GetMonStore(c.context, c.clusterInfo) + // port + if c.spec.Monitoring.Port != 0 { + port := strconv.Itoa(c.spec.Monitoring.Port) + portHasChanged, err = monStore.SetIfChanged(daemonID, "mgr/prometheus/server_port", port) + if err != nil { + return err + } + logger.Infof("prometheus config will change, port: %s", port) + } + // scrape interval + if c.spec.Monitoring.Interval != nil { + interval := c.spec.Monitoring.Interval.Duration.Seconds() + intervalHasChanged, err = monStore.SetIfChanged(daemonID, "mgr/prometheus/scrape_interval", fmt.Sprintf("%v", interval)) + if err != nil { + return err + } + logger.Infof("prometheus config will change, interval: %v", interval) + } + + if portHasChanged || intervalHasChanged { + logger.Info("prometheus config has changed. restarting the prometheus module") + return c.restartMgrModule(PrometheusModuleName) + } + return nil +} + +func (c *Cluster) restartMgrModule(name string) error { + logger.Infof("restarting the mgr module: %s", name) + if err := cephclient.MgrDisableModule(c.context, c.clusterInfo, name); err != nil { + return errors.Wrapf(err, "failed to disable mgr module %q.", name) + } + if err := cephclient.MgrEnableModule(c.context, c.clusterInfo, name, true); err != nil { + return errors.Wrapf(err, "failed to enable mgr module %q.", name) } return nil } diff --git a/pkg/operator/ceph/cluster/mgr/mgr_test.go b/pkg/operator/ceph/cluster/mgr/mgr_test.go index ad050cf41..051c44575 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr_test.go +++ b/pkg/operator/ceph/cluster/mgr/mgr_test.go @@ -458,3 +458,100 @@ func TestCluster_enableBalancerModule(t *testing.T) { assert.NoError(t, err) }) } + +func TestCluster_configurePrometheusModule(t *testing.T) { + modulesEnabled := 0 + modulesDisabled := 0 + configSettings := map[string]string{} + lastModuleConfigured := "" + modulesGetOutput := "" + executor := &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { + logger.Infof("Command: %s %v", command, args) + if command == "ceph" && len(args) > 3 { + if args[0] == "mgr" && args[1] == "module" { + if args[2] == "enable" { + modulesEnabled++ + } + if args[2] == "disable" { + modulesDisabled++ + } + lastModuleConfigured = args[3] + } + } + return "", nil //return "{\"key\":\"mysecurekey\"}", nil + }, + MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) { + if args[0] == "config" && args[1] == "set" && args[2] == "mgr" { + configSettings[args[3]] = args[4] + } + if args[0] == "config" && args[1] == "get" && args[2] == "mgr" { + return modulesGetOutput, nil + } + return "", nil + }, + } + + c := &Cluster{ + context: &clusterd.Context{Executor: executor, Clientset: testop.New(t, 3)}, + clusterInfo: cephclient.AdminTestClusterInfo("mycluster"), + spec: cephv1.ClusterSpec{ + Monitoring: cephv1.MonitoringSpec{ + Enabled: false, + }, + }, + } + // Disable prometheus module + err := c.configurePrometheusModule() + assert.NoError(t, err) + assert.Equal(t, 0, modulesEnabled) + assert.Equal(t, 1, modulesDisabled) + assert.Equal(t, PrometheusModuleName, lastModuleConfigured) + + // Enable prometheus module, no changed + modulesDisabled = 0 + c.spec.Monitoring.Enabled = true + err = c.configurePrometheusModule() + assert.NoError(t, err) + assert.Equal(t, 1, modulesEnabled) + assert.Equal(t, 0, modulesDisabled) + + // Enable prometheus module, port changed + modulesEnabled = 0 + c.spec.Monitoring.Port = 30001 + err = c.configurePrometheusModule() + assert.NoError(t, err) + assert.Equal(t, 2, modulesEnabled) + assert.Equal(t, 1, modulesDisabled) + assert.Equal(t, "30001", configSettings["mgr/prometheus/server_port"]) + assert.Equal(t, 1, len(configSettings)) + + modulesEnabled = 0 + modulesDisabled = 0 + c.spec.Monitoring.Interval = &metav1.Duration{ + Duration: 30 * time.Second, + } + modulesGetOutput = "30" + configSettings = make(map[string]string) + err = c.configurePrometheusModule() + assert.NoError(t, err) + assert.Equal(t, 2, modulesEnabled) + assert.Equal(t, 1, modulesDisabled) + assert.Equal(t, "30001", configSettings["mgr/prometheus/server_port"]) + assert.Equal(t, 1, len(configSettings)) + + // Enable prometheus module, port and interval changed + modulesEnabled = 0 + modulesDisabled = 0 + c.spec.Monitoring.Port = 30002 + c.spec.Monitoring.Interval = &metav1.Duration{ + Duration: 60 * time.Second, + } + err = c.configurePrometheusModule() + assert.NoError(t, err) + assert.Equal(t, 2, modulesEnabled) + assert.Equal(t, 1, modulesDisabled) + assert.Equal(t, "30002", configSettings["mgr/prometheus/server_port"]) + assert.Equal(t, "60", configSettings["mgr/prometheus/scrape_interval"]) + +}