mgr: disable the prometheus module by crds

Signed-off-by: xiaobaowen <xiaobaowen@deeproute.ai>
(cherry picked from commit e7254c2a1b)

deploy/charts/rook-ceph/templates/resources.yaml

@@ -1867,6 +1867,14 @@ spec:
                       maximum: 65535
                       minimum: 0
                       type: integer
+                    interval:
+                      description: Interval determines prometheus scrape interval
+                      type: string
+                    port:
+                      description: Port is the prometheus server port
+                      maximum: 65535
+                      minimum: 0
+                      type: integer
                   type: object
                 network:
                   description: Network related configuration

deploy/examples/crds.yaml

@@ -1865,6 +1865,14 @@ spec:
                       maximum: 65535
                       minimum: 0
                       type: integer
+                    interval:
+                      description: Interval determines prometheus scrape interval
+                      type: string
+                    port:
+                      description: Port is the prometheus server port
+                      maximum: 65535
+                      minimum: 0
+                      type: integer
                   type: object
                 network:
                   description: Network related configuration

pkg/apis/ceph.rook.io/v1/types.go

@@ -337,6 +337,16 @@ type MonitoringSpec struct {
 	// +kubebuilder:validation:Maximum=65535
 	// +optional
 	ExternalMgrPrometheusPort uint16 `json:"externalMgrPrometheusPort,omitempty"`
+
+	// Port is the prometheus server port
+	// +kubebuilder:validation:Minimum=0
+	// +kubebuilder:validation:Maximum=65535
+	// +optional
+	Port int `json:"port,omitempty"`
+
+	// Interval determines prometheus scrape interval
+	// +optional
+	Interval *metav1.Duration `json:"interval,omitempty"`
 }
 
 // ClusterStatus represents the status of a Ceph cluster

pkg/operator/ceph/cluster/mgr/dashboard.go

@@ -104,7 +104,7 @@ func (c *Cluster) configureDashboardModules() error {
 	}
 	if hasChanged {
 		logger.Info("dashboard config has changed. restarting the dashboard module")
-		return c.restartDashboard()
+		return c.restartMgrModule(dashboardModuleName)
 	}
 	return nil
 }
@@ -304,14 +304,3 @@ func decodeSecret(secret *v1.Secret) (string, error) {
 	}
 	return string(password), nil
 }
-
-func (c *Cluster) restartDashboard() error {
-	logger.Info("restarting the mgr module")
-	if err := client.MgrDisableModule(c.context, c.clusterInfo, dashboardModuleName); err != nil {
-		return errors.Wrapf(err, "failed to disable mgr module %q.", dashboardModuleName)
-	}
-	if err := client.MgrEnableModule(c.context, c.clusterInfo, dashboardModuleName, true); err != nil {
-		return errors.Wrapf(err, "failed to enable mgr module %q.", dashboardModuleName)
-	}
-	return nil
-}

pkg/operator/ceph/cluster/mgr/mgr.go

@@ -20,6 +20,7 @@ package mgr
 import (
 	"fmt"
 	"path"
+	"strconv"
 
 	"github.com/banzaicloud/k8s-objectmatcher/patch"
 	"github.com/coreos/pkg/capnslog"
@@ -318,7 +319,7 @@ func (c *Cluster) updateServiceSelectors(activeDaemon string) error {
 
 func (c *Cluster) configureModules(daemonIDs []string) {
 	// Configure the modules asynchronously so we can complete all the configuration much sooner.
-	startModuleConfiguration("prometheus", c.enablePrometheusModule)
+	startModuleConfiguration("prometheus", c.configurePrometheusModule)
 	startModuleConfiguration("dashboard", c.configureDashboardModules)
 
 	// It is a bit confusing but modules that are in the "always_on_modules" list
@@ -339,9 +340,58 @@ func startModuleConfiguration(description string, configureModules func() error)
 }
 
 // Ceph docs about the prometheus module: http://docs.ceph.com/docs/master/mgr/prometheus/
-func (c *Cluster) enablePrometheusModule() error {
-	if err := cephclient.MgrEnableModule(c.context, c.clusterInfo, PrometheusModuleName, true); err != nil {
-		return errors.Wrap(err, "failed to enable mgr prometheus module")
+func (c *Cluster) configurePrometheusModule() error {
+	if c.spec.Monitoring.Enabled {
+		if err := cephclient.MgrEnableModule(c.context, c.clusterInfo, PrometheusModuleName, true); err != nil {
+			return errors.Wrap(err, "failed to enable mgr prometheus module")
+		}
+	} else {
+		if err := cephclient.MgrDisableModule(c.context, c.clusterInfo, PrometheusModuleName); err != nil {
+			logger.Errorf("failed to disable mgr prometheus module. %v", err)
+		}
+		return nil
+	}
+
+	var (
+		err                error
+		portHasChanged     bool
+		intervalHasChanged bool
+		daemonID           = "mgr"
+	)
+	monStore := config.GetMonStore(c.context, c.clusterInfo)
+	// port
+	if c.spec.Monitoring.Port != 0 {
+		port := strconv.Itoa(c.spec.Monitoring.Port)
+		portHasChanged, err = monStore.SetIfChanged(daemonID, "mgr/prometheus/server_port", port)
+		if err != nil {
+			return err
+		}
+		logger.Infof("prometheus config will change, port: %s", port)
+	}
+	// scrape interval
+	if c.spec.Monitoring.Interval != nil {
+		interval := c.spec.Monitoring.Interval.Duration.Seconds()
+		intervalHasChanged, err = monStore.SetIfChanged(daemonID, "mgr/prometheus/scrape_interval", fmt.Sprintf("%v", interval))
+		if err != nil {
+			return err
+		}
+		logger.Infof("prometheus config will change, interval: %v", interval)
+	}
+
+	if portHasChanged || intervalHasChanged {
+		logger.Info("prometheus config has changed. restarting the prometheus module")
+		return c.restartMgrModule(PrometheusModuleName)
+	}
+	return nil
+}
+
+func (c *Cluster) restartMgrModule(name string) error {
+	logger.Infof("restarting the mgr module: %s", name)
+	if err := cephclient.MgrDisableModule(c.context, c.clusterInfo, name); err != nil {
+		return errors.Wrapf(err, "failed to disable mgr module %q.", name)
+	}
+	if err := cephclient.MgrEnableModule(c.context, c.clusterInfo, name, true); err != nil {
+		return errors.Wrapf(err, "failed to enable mgr module %q.", name)
 	}
 	return nil
 }

pkg/operator/ceph/cluster/mgr/mgr_test.go

@@ -458,3 +458,100 @@ func TestCluster_enableBalancerModule(t *testing.T) {
 		assert.NoError(t, err)
 	})
 }
+
+func TestCluster_configurePrometheusModule(t *testing.T) {
+	modulesEnabled := 0
+	modulesDisabled := 0
+	configSettings := map[string]string{}
+	lastModuleConfigured := ""
+	modulesGetOutput := ""
+	executor := &exectest.MockExecutor{
+		MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
+			logger.Infof("Command: %s %v", command, args)
+			if command == "ceph" && len(args) > 3 {
+				if args[0] == "mgr" && args[1] == "module" {
+					if args[2] == "enable" {
+						modulesEnabled++
+					}
+					if args[2] == "disable" {
+						modulesDisabled++
+					}
+					lastModuleConfigured = args[3]
+				}
+			}
+			return "", nil //return "{\"key\":\"mysecurekey\"}", nil
+		},
+		MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) {
+			if args[0] == "config" && args[1] == "set" && args[2] == "mgr" {
+				configSettings[args[3]] = args[4]
+			}
+			if args[0] == "config" && args[1] == "get" && args[2] == "mgr" {
+				return modulesGetOutput, nil
+			}
+			return "", nil
+		},
+	}
+
+	c := &Cluster{
+		context:     &clusterd.Context{Executor: executor, Clientset: testop.New(t, 3)},
+		clusterInfo: cephclient.AdminTestClusterInfo("mycluster"),
+		spec: cephv1.ClusterSpec{
+			Monitoring: cephv1.MonitoringSpec{
+				Enabled: false,
+			},
+		},
+	}
+	// Disable prometheus module
+	err := c.configurePrometheusModule()
+	assert.NoError(t, err)
+	assert.Equal(t, 0, modulesEnabled)
+	assert.Equal(t, 1, modulesDisabled)
+	assert.Equal(t, PrometheusModuleName, lastModuleConfigured)
+
+	// Enable prometheus module, no changed
+	modulesDisabled = 0
+	c.spec.Monitoring.Enabled = true
+	err = c.configurePrometheusModule()
+	assert.NoError(t, err)
+	assert.Equal(t, 1, modulesEnabled)
+	assert.Equal(t, 0, modulesDisabled)
+
+	// Enable prometheus module, port changed
+	modulesEnabled = 0
+	c.spec.Monitoring.Port = 30001
+	err = c.configurePrometheusModule()
+	assert.NoError(t, err)
+	assert.Equal(t, 2, modulesEnabled)
+	assert.Equal(t, 1, modulesDisabled)
+	assert.Equal(t, "30001", configSettings["mgr/prometheus/server_port"])
+	assert.Equal(t, 1, len(configSettings))
+
+	modulesEnabled = 0
+	modulesDisabled = 0
+	c.spec.Monitoring.Interval = &metav1.Duration{
+		Duration: 30 * time.Second,
+	}
+	modulesGetOutput = "30"
+	configSettings = make(map[string]string)
+	err = c.configurePrometheusModule()
+	assert.NoError(t, err)
+	assert.Equal(t, 2, modulesEnabled)
+	assert.Equal(t, 1, modulesDisabled)
+	assert.Equal(t, "30001", configSettings["mgr/prometheus/server_port"])
+	assert.Equal(t, 1, len(configSettings))
+
+	// Enable prometheus module, port and interval changed
+	modulesEnabled = 0
+	modulesDisabled = 0
+	c.spec.Monitoring.Port = 30002
+	c.spec.Monitoring.Interval = &metav1.Duration{
+		Duration: 60 * time.Second,
+	}
+	err = c.configurePrometheusModule()
+	assert.NoError(t, err)
+	assert.Equal(t, 2, modulesEnabled)
+	assert.Equal(t, 1, modulesDisabled)
+	assert.Equal(t, "30002", configSettings["mgr/prometheus/server_port"])
+	assert.Equal(t, "60", configSettings["mgr/prometheus/scrape_interval"])
+
+}