core: add configuration option for metrics bindAddress

Alerting on controller-runtime's workqueue_depth can be useful for
debugging controllers. Also having a prometheus target for a pod gives
another data point that the system is working as expected. It is useful
for uptime alerts.

Make the bind address configurable via the configmap while still retaining the default
behavior that it is disabled.

Resolves: #14538

Signed-off-by: Justin Cichra <jcichra@cloudflare.com>

deploy/charts/rook-ceph/templates/configmap.yaml

@@ -9,6 +9,9 @@ data:
   ROOK_LOG_LEVEL: {{ .Values.logLevel | quote }}
   ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS: {{ .Values.cephCommandsTimeoutSeconds | quote }}
   ROOK_OBC_WATCH_OPERATOR_NAMESPACE: {{ .Values.enableOBCWatchOperatorNamespace | quote }}
+{{- if .Values.operatorMetricsBindAddress }}
+  ROOK_OPERATOR_METRICS_BIND_ADDRESS: {{ .Values.operatorMetricsBindAddress | quote }}
+{{- end }}
 {{- if .Values.obcProvisionerNamePrefix }}
   ROOK_OBC_PROVISIONER_NAME_PREFIX: {{ .Values.obcProvisionerNamePrefix | quote }}
 {{- end }}

deploy/examples/operator.yaml

@@ -25,6 +25,9 @@ data:
   # The logging level for the operator: ERROR | WARNING | INFO | DEBUG
   ROOK_LOG_LEVEL: "INFO"
 
+  # The address for the operator's controller-runtime metrics. 0 is disabled. :8080 serves metrics on port 8080.
+  ROOK_OPERATOR_METRICS_BIND_ADDRESS: "0"
+
   # Allow using loop devices for osds in test clusters.
   ROOK_CEPH_ALLOW_LOOP_DEVICES: "false"
 

pkg/operator/ceph/cr_manager.go

@@ -44,6 +44,7 @@ import (
 	"github.com/rook/rook/pkg/operator/ceph/object/zonegroup"
 	"github.com/rook/rook/pkg/operator/ceph/pool"
 	"github.com/rook/rook/pkg/operator/ceph/pool/radosnamespace"
+	"github.com/rook/rook/pkg/operator/k8sutil"
 	"k8s.io/apimachinery/pkg/runtime"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
@@ -142,14 +143,18 @@ func (o *Operator) startCRDManager(context context.Context, mgrErrorCh chan erro
 		}
 	}
 
+	metricsBindAddress, err := k8sutil.GetOperatorSetting(context, o.context.Clientset, opcontroller.OperatorSettingConfigMapName, "ROOK_OPERATOR_METRICS_BIND_ADDRESS", "0")
+	if err != nil {
+		mgrErrorCh <- errors.Wrap(err, "failed to get configmap value `ROOK_OPERATOR_METRICS_BIND_ADDRESS`.")
+		return
+	}
 	skipNameValidation := true
 	// Set up a manager
 	mgrOpts := manager.Options{
 		LeaderElection: false,
 		Metrics: metricsserver.Options{
-			// BindAddress is the bind address for controller runtime metrics server default is 8080. Since we don't use the
-			// controller runtime metrics server, we need to set the bind address 0 so that port 8080 is available.
-			BindAddress: "0",
+			// BindAddress is the bind address for controller runtime metrics server. Defaulted to "0" which is off.
+			BindAddress: metricsBindAddress,
 		},
 		Scheme: scheme,
 		Controller: config.Controller{