Merge pull request #12444 from rook/mergify/bp/release-1.11/pr-12430

exporter: Ignore failed deletion of service monitor (backport #12430)

.commitlintrc.json

@@ -15,6 +15,7 @@
         "core",
         "csi",
         "docs",
+        "exporter",
         "external",
         "file",
         "helm",

pkg/operator/ceph/cluster/mgr/mgr.go

@@ -515,7 +515,7 @@ func (c *Cluster) EnableServiceMonitor() error {
 
 	applyMonitoringLabels(c, serviceMonitor)
 
-	if _, err = k8sutil.CreateOrUpdateServiceMonitor(c.clusterInfo.Context, serviceMonitor); err != nil {
+	if _, err = k8sutil.CreateOrUpdateServiceMonitor(c.context, c.clusterInfo.Context, serviceMonitor); err != nil {
 		return errors.Wrap(err, "service monitor could not be enabled")
 	}
 	return nil

pkg/operator/ceph/cluster/nodedaemon/exporter.go

@@ -24,6 +24,7 @@ import (
 	"github.com/pkg/errors"
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/operator/ceph/config"
 	"github.com/rook/rook/pkg/operator/ceph/config/keyring"
 	"github.com/rook/rook/pkg/operator/ceph/controller"
@@ -219,7 +220,7 @@ func MakeCephExporterMetricsService(cephCluster cephv1.CephCluster, servicePortM
 }
 
 // EnableCephExporterServiceMonitor add a servicemonitor that allows prometheus to scrape from the monitoring endpoint of the exporter
-func EnableCephExporterServiceMonitor(cephCluster cephv1.CephCluster, scheme *runtime.Scheme, opManagerContext context.Context) error {
+func EnableCephExporterServiceMonitor(context *clusterd.Context, cephCluster cephv1.CephCluster, scheme *runtime.Scheme, opManagerContext context.Context) error {
 	serviceMonitor := k8sutil.GetServiceMonitor(cephExporterAppName, cephCluster.Namespace)
 	cephv1.GetCephExporterLabels(cephCluster.Spec.Labels).OverwriteApplyToObjectMeta(&serviceMonitor.ObjectMeta)
 
@@ -230,7 +231,7 @@ func EnableCephExporterServiceMonitor(cephCluster cephv1.CephCluster, scheme *ru
 	serviceMonitor.Spec.Selector.MatchLabels = controller.AppLabels(cephExporterAppName, cephCluster.Namespace)
 	applyCephExporterLabels(cephCluster, serviceMonitor)
 
-	if _, err = k8sutil.CreateOrUpdateServiceMonitor(opManagerContext, serviceMonitor); err != nil {
+	if _, err = k8sutil.CreateOrUpdateServiceMonitor(context, opManagerContext, serviceMonitor); err != nil {
 		return errors.Wrap(err, "service monitor could not be enabled")
 	}
 	return nil

pkg/operator/ceph/cluster/nodedaemon/reconcile.go

@@ -86,9 +86,9 @@ func (r *ReconcileNode) Reconcile(context context.Context, request reconcile.Req
 }
 
 func (r *ReconcileNode) cleanupExporterResources(clusterNamespace string, ns string, nodeName string) (reconcile.Result, error) {
-	err := k8sutil.DeleteServiceMonitor(r.opManagerContext, ns, cephExporterAppName)
+	err := k8sutil.DeleteServiceMonitor(r.context, r.opManagerContext, ns, cephExporterAppName)
 	if err != nil {
-		logger.Debugf("failed to delete service monitor for ceph exporter in namespace %q on node %q", ns, nodeName)
+		logger.Debugf("failed to delete service monitor for ceph exporter in namespace %q on node %q. %v", ns, nodeName, err)
 	}
 	err = k8sutil.DeleteService(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace, cephExporterAppName)
 	if err != nil {
@@ -271,7 +271,7 @@ func (r *ReconcileNode) createOrUpdateNodeDaemons(node corev1.Node, tolerations
 				}
 
 				if cephCluster.Spec.Monitoring.Enabled {
-					if err := EnableCephExporterServiceMonitor(cephCluster, r.scheme, r.opManagerContext); err != nil {
+					if err := EnableCephExporterServiceMonitor(r.context, cephCluster, r.scheme, r.opManagerContext); err != nil {
 						return errors.Wrap(err, "failed to enable service monitor")
 					}
 					logger.Debug("service monitor for ceph exporter was enabled successfully")

pkg/operator/k8sutil/prometheus.go

@@ -21,20 +21,17 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/pkg/errors"
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	monitoringclient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
+	"github.com/rook/rook/pkg/clusterd"
 	kerror "k8s.io/apimachinery/pkg/api/errors"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/tools/clientcmd"
 )
 
-func getMonitoringClient() (*monitoringclient.Clientset, error) {
-	cfg, err := clientcmd.BuildConfigFromFlags("", "")
-	if err != nil {
-		return nil, fmt.Errorf("failed to build config. %v", err)
-	}
-	client, err := monitoringclient.NewForConfig(cfg)
+func getMonitoringClient(context *clusterd.Context) (*monitoringclient.Clientset, error) {
+	client, err := monitoringclient.NewForConfig(context.KubeConfig)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get monitoring client. %v", err)
 	}
@@ -75,11 +72,11 @@ func GetServiceMonitor(name string, namespace string) *monitoringv1.ServiceMonit
 }
 
 // CreateOrUpdateServiceMonitor creates serviceMonitor object or an error
-func CreateOrUpdateServiceMonitor(ctx context.Context, serviceMonitorDefinition *monitoringv1.ServiceMonitor) (*monitoringv1.ServiceMonitor, error) {
+func CreateOrUpdateServiceMonitor(context *clusterd.Context, ctx context.Context, serviceMonitorDefinition *monitoringv1.ServiceMonitor) (*monitoringv1.ServiceMonitor, error) {
 	name := serviceMonitorDefinition.GetName()
 	namespace := serviceMonitorDefinition.GetNamespace()
 	logger.Debugf("creating servicemonitor %s", name)
-	client, err := getMonitoringClient()
+	client, err := getMonitoringClient(context)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get monitoring client. %v", err)
 	}
@@ -104,16 +101,23 @@ func CreateOrUpdateServiceMonitor(ctx context.Context, serviceMonitorDefinition
 }
 
 // DeleteServiceMonitor deletes a ServiceMonitor and returns the error if any
-func DeleteServiceMonitor(ctx context.Context, ns string, name string) error {
-	client, err := getMonitoringClient()
+func DeleteServiceMonitor(context *clusterd.Context, ctx context.Context, ns string, name string) error {
+	client, err := getMonitoringClient(context)
 	if err != nil {
 		return fmt.Errorf("failed to get monitoring client. %v", err)
 	}
+	_, err = client.MonitoringV1().ServiceMonitors(ns).Get(ctx, name, metav1.GetOptions{})
+	if err != nil {
+		// Either the service monitor does not exist or there are no privileges to detect it
+		// so we ignore any errors
+		return nil
+	}
 	err = client.MonitoringV1().ServiceMonitors(ns).Delete(ctx, name, metav1.DeleteOptions{})
 	if err != nil {
 		if kerror.IsNotFound(err) {
 			return nil
 		}
+		return errors.Wrapf(err, "failed to delete service monitor %q", name)
 	}
-	return err
+	return nil
 }