From 93adc5dd6b7944dcaf0acd091c36623bf4869c8d Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Fri, 14 Nov 2025 16:03:53 -0700 Subject: [PATCH] core: include namespace more consistently in logging When writing to the operator log, where possible the namespace of the resource on which the operator is working will be logged as a standard prefix to more easily troubleshoot what resource is being reconciled Signed-off-by: Travis Nielsen --- pkg/operator/ceph/cluster/cephstatus.go | 49 ++--- pkg/operator/ceph/cluster/cephx.go | 33 +-- pkg/operator/ceph/cluster/cleanup.go | 17 +- pkg/operator/ceph/cluster/cluster.go | 65 +++--- pkg/operator/ceph/cluster/cluster_external.go | 29 +-- pkg/operator/ceph/cluster/controller.go | 33 +-- pkg/operator/ceph/cluster/mgr/config.go | 7 +- pkg/operator/ceph/cluster/mgr/dashboard.go | 31 +-- pkg/operator/ceph/cluster/mgr/drain.go | 7 +- pkg/operator/ceph/cluster/mgr/mgr.go | 73 +++---- pkg/operator/ceph/cluster/mgr/spec.go | 3 +- pkg/operator/ceph/cluster/mon/drain.go | 9 +- pkg/operator/ceph/cluster/mon/health.go | 204 +++++++++--------- pkg/operator/ceph/cluster/mon/mon.go | 178 +++++++-------- pkg/operator/ceph/cluster/mon/predicate.go | 15 +- .../ceph/cluster/mon/predicate_test.go | 2 +- pkg/operator/ceph/cluster/mon/service.go | 9 +- pkg/operator/ceph/cluster/mon/spec.go | 12 +- pkg/operator/ceph/cluster/monitoring.go | 9 +- .../ceph/cluster/nodedaemon/exporter.go | 9 +- .../ceph/cluster/nodedaemon/keyring.go | 13 +- .../ceph/cluster/nodedaemon/pruner.go | 13 +- .../ceph/cluster/nodedaemon/reconcile.go | 35 +-- pkg/operator/ceph/cluster/osd/create.go | 39 ++-- pkg/operator/ceph/cluster/osd/deviceSet.go | 23 +- pkg/operator/ceph/cluster/osd/health.go | 23 +- pkg/operator/ceph/cluster/osd/key_rotation.go | 11 +- pkg/operator/ceph/cluster/osd/migrate.go | 11 +- pkg/operator/ceph/cluster/osd/osd.go | 99 ++++----- .../ceph/cluster/osd/provision_spec.go | 9 +- pkg/operator/ceph/cluster/osd/spec.go | 21 +- pkg/operator/ceph/cluster/osd/status.go | 27 +-- pkg/operator/ceph/cluster/osd/update.go | 33 +-- pkg/operator/ceph/cluster/osd/volumes.go | 3 +- pkg/operator/ceph/cluster/predicate.go | 15 +- pkg/operator/ceph/cluster/rbd/config.go | 7 +- pkg/operator/ceph/cluster/rbd/controller.go | 21 +- pkg/operator/ceph/cluster/rbd/mirror.go | 11 +- .../ceph/cluster/telemetry/telemetry.go | 5 +- pkg/operator/ceph/cluster/utils.go | 15 +- pkg/operator/ceph/cluster/version.go | 43 ++-- pkg/operator/ceph/cluster/version_test.go | 15 +- pkg/util/log/log.go | 63 ++++++ 43 files changed, 727 insertions(+), 622 deletions(-) create mode 100644 pkg/util/log/log.go diff --git a/pkg/operator/ceph/cluster/cephstatus.go b/pkg/operator/ceph/cluster/cephstatus.go index c7d8314f4..fda5df88c 100644 --- a/pkg/operator/ceph/cluster/cephstatus.go +++ b/pkg/operator/ceph/cluster/cephstatus.go @@ -34,6 +34,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/reporting" cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -75,7 +76,7 @@ func newCephStatusChecker(context *clusterd.Context, clusterInfo *cephclient.Clu checkInterval = &checkIntervalCRSetting.Duration } if checkInterval != nil { - logger.Infof("ceph status check interval is %s", checkInterval.String()) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "ceph status check interval is %s", checkInterval.String()) c.interval = checkInterval } @@ -87,7 +88,7 @@ func (c *cephStatusChecker) checkCephStatus(monitoringRoutines *sync.Map, daemon // check the status immediately before starting the loop v, ok := monitoringRoutines.Load(daemon) if !ok { - logger.Infof("ceph cluster %q has been deleted. stopping ceph status check", c.clusterInfo.Namespace) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "ceph cluster %q has been deleted. stopping ceph status check", c.clusterInfo.Namespace) return } c.checkStatus(v.(*opcontroller.ClusterHealth).InternalCtx) @@ -97,13 +98,13 @@ func (c *cephStatusChecker) checkCephStatus(monitoringRoutines *sync.Map, daemon // we will get an invalid pointer error and the go routine will panic v, ok := monitoringRoutines.Load(daemon) if !ok { - logger.Infof("ceph cluster %q has been deleted. stopping ceph status check", c.clusterInfo.Namespace) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "ceph cluster %q has been deleted. stopping ceph status check", c.clusterInfo.Namespace) return } health := v.(*opcontroller.ClusterHealth) select { case <-health.InternalCtx.Done(): - logger.Infof("stopping monitoring of ceph status") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "stopping monitoring of ceph status") monitoringRoutines.Delete(daemon) return @@ -118,27 +119,27 @@ func (c *cephStatusChecker) checkStatus(ctx context.Context) { var status cephclient.CephStatus var err error - logger.Debugf("checking health of cluster") + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "checking health of cluster") // Check ceph's status status, err = cephclient.StatusWithUser(c.context, c.clusterInfo) if err != nil { if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { - logger.Info("skipping ceph status since operator is still initializing") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "skipping ceph status since operator is still initializing") return } - logger.Errorf("failed to get ceph status. %v", err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get ceph status. %v", err) status := cephStatusOnError(err.Error()) c.updateCephStatus(status) return } - logger.Debugf("cluster status: %+v", status) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "cluster status: %+v", status) c.updateCephStatus(&status) if status.Health.Status != "HEALTH_OK" { - logger.Debug("checking for stuck pods on not ready nodes") + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "checking for stuck pods on not ready nodes") if err := c.forceDeleteStuckRookPodsOnNotReadyNodes(ctx); err != nil { - logger.Errorf("failed to delete pod on not ready nodes. %v", err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to delete pod on not ready nodes. %v", err) } } @@ -148,16 +149,16 @@ func (c *cephStatusChecker) checkStatus(ctx context.Context) { func (c *cephStatusChecker) configureHealthSettings(status cephclient.CephStatus) { // loop through the health codes and log what we find for healthCode, check := range status.Health.Checks { - logger.Debugf("Health: %q, code: %q, message: %q", check.Severity, healthCode, check.Summary.Message) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "Health: %q, code: %q, message: %q", check.Severity, healthCode, check.Summary.Message) } // disable the insecure global id if there are no old clients if _, ok := status.Health.Checks["AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED"]; ok { if _, ok := status.Health.Checks["AUTH_INSECURE_GLOBAL_ID_RECLAIM"]; !ok { - logger.Info("Disabling the insecure global ID as no legacy clients are currently connected. If you still require the insecure connections, see the CVE to suppress the health warning and re-enable the insecure connections. https://docs.ceph.com/en/latest/security/CVE-2021-20288/") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Disabling the insecure global ID as no legacy clients are currently connected. If you still require the insecure connections, see the CVE to suppress the health warning and re-enable the insecure connections. https://docs.ceph.com/en/latest/security/CVE-2021-20288/") config.DisableInsecureGlobalID(c.context, c.clusterInfo) } else { - logger.Warning("insecure clients are connected to the cluster, to resolve the AUTH_INSECURE_GLOBAL_ID_RECLAIM health warning please refer to the upgrade guide to ensure all Ceph daemons are updated.") + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "insecure clients are connected to the cluster, to resolve the AUTH_INSECURE_GLOBAL_ID_RECLAIM health warning please refer to the upgrade guide to ensure all Ceph daemons are updated.") } } } @@ -168,10 +169,10 @@ func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus) { cephCluster, err := c.context.RookClientset.CephV1().CephClusters(clusterName.Namespace).Get(c.clusterInfo.Context, clusterName.Name, metav1.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { - logger.Debug("CephCluster resource not found. Ignoring since object must be deleted.") + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "CephCluster resource not found. Ignoring since object must be deleted.") return } - logger.Errorf("failed to retrieve ceph cluster %q in namespace %q to update status to %+v", clusterName.Name, clusterName.Namespace, status) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to retrieve ceph cluster %q in namespace %q to update status to %+v", clusterName.Name, clusterName.Namespace, status) return } @@ -181,16 +182,16 @@ func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus) { // versions store the ceph version of all the ceph daemons and overall cluster version versions, err := cephclient.GetAllCephDaemonVersions(c.context, c.clusterInfo) if err != nil { - logger.Errorf("failed to get ceph daemons versions. %v", err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get ceph daemons versions. %v", err) } else { // Update status with Ceph versions cephCluster.Status.CephStatus.Versions = versions } // Update condition - logger.Debugf("updating ceph cluster %q status to %+v", clusterName.Namespace, status) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "updating ceph cluster %q status to %+v", clusterName.Namespace, status) if err := reporting.UpdateStatus(c.context.Client, cephCluster); err != nil { - logger.Errorf("failed to update cluster status to %+v. %v", status, err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to update cluster status to %+v. %v", status, err) } } @@ -237,15 +238,15 @@ func formatTime(t time.Time) string { } func (c *ClusterController) updateClusterCephVersion(cluster *cluster, cephVersion cephver.CephVersion) { - logger.Infof("cluster %q: version %q detected for image %q", cluster.namespacedName.Namespace, cephVersion.String(), cluster.Spec.CephVersion.Image) + log.NamespacedInfo(cluster.Namespace, logger, "cluster %q: version %q detected for image %q", cluster.namespacedName.Namespace, cephVersion.String(), cluster.Spec.CephVersion.Image) cephCluster, err := c.context.RookClientset.CephV1().CephClusters(cluster.namespacedName.Namespace).Get(c.OpManagerCtx, cluster.namespacedName.Name, metav1.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { - logger.Debug("CephCluster resource not found. Ignoring since object must be deleted.") + log.NamespacedDebug(cluster.Namespace, logger, "CephCluster resource not found. Ignoring since object must be deleted.") return } - logger.Errorf("failed to retrieve ceph cluster %q to update ceph version to %+v. %v", cluster.namespacedName.Name, cephVersion, err) + log.NamespacedError(cluster.Namespace, logger, "failed to retrieve ceph cluster %q to update ceph version to %+v. %v", cluster.namespacedName.Name, cephVersion, err) return } @@ -257,7 +258,7 @@ func (c *ClusterController) updateClusterCephVersion(cluster *cluster, cephVersi // do not overwrite the ceph status that is updated in a separate goroutine cephCluster.Status.CephVersion = cephClusterVersion if err := reporting.UpdateStatus(c.client, cephCluster); err != nil { - logger.Errorf("failed to update cluster %q version. %v", cluster.namespacedName.Name, err) + log.NamespacedError(cluster.Namespace, logger, "failed to update cluster %q version. %v", cluster.namespacedName.Name, err) return } } @@ -289,11 +290,11 @@ func (c *cephStatusChecker) forceDeleteStuckRookPodsOnNotReadyNodes(ctx context. for _, node := range nodes { pods, err := c.getRookPodsOnNode(node.Name) if err != nil { - logger.Errorf("failed to get pods on NotReady node %q. %v", node.Name, err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get pods on NotReady node %q. %v", node.Name, err) } for _, pod := range pods { if err := k8sutil.ForceDeletePodIfStuck(ctx, c.context, pod); err != nil { - logger.Warningf("skipping forced delete of stuck pod %q. %v", pod.Name, err) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "skipping forced delete of stuck pod %q. %v", pod.Name, err) } } } diff --git a/pkg/operator/ceph/cluster/cephx.go b/pkg/operator/ceph/cluster/cephx.go index 6b6de386c..7e3e47f56 100644 --- a/pkg/operator/ceph/cluster/cephx.go +++ b/pkg/operator/ceph/cluster/cephx.go @@ -33,6 +33,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/reporting" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/util/retry" ) @@ -73,9 +74,9 @@ func claimAdminRotationLock(namespace string) error { adminRotationMutex.Lock() defer adminRotationMutex.Unlock() if _, ok := adminRotationInProgress[namespace]; ok { - return fmt.Errorf("failed to claim admin rotation lock for cluster in namespace %q", namespace) + return fmt.Errorf("failed to claim admin rotation lock") } - logger.Debugf("claimed admin rotation lock for cluster in namespace %q", namespace) + log.NamespacedDebug(namespace, logger, "claimed admin rotation lock") adminRotationInProgress[namespace] = struct{}{} return nil } @@ -118,7 +119,7 @@ func rotateAdminCephxKey( ) error { if clusterInfo.CephCred.Username != cephclient.AdminUsername { // this shouldn't happen during normal runtime - this could indicate external mode cluster - logger.Infof("cannot rotate admin cephx key with non-rotatable username %q for cluster in namespace %q", clusterInfo.CephCred.Username, clusterInfo.Namespace) + log.NamespacedInfo(clusterInfo.Namespace, logger, "cannot rotate admin cephx key with non-rotatable username %q", clusterInfo.CephCred.Username) return nil } @@ -129,7 +130,7 @@ func rotateAdminCephxKey( return errors.Wrap(err, "failed to determine if admin cephx key should be rotated") } if !shouldRotate { - logger.Debugf("not rotating admin cephx key for cluster in namespace %q", clusterInfo.Namespace) + log.NamespacedDebug(clusterInfo.Namespace, logger, "not rotating admin cephx key") // no rotation, but still update cephx status - critical for greenfield Uninitialized clusters err := updateCephClusterAdminCephxStatus(clusterdCtx, clusterInfo, false) if err != nil { @@ -143,7 +144,7 @@ func rotateAdminCephxKey( } defer releaseAdminRotationLock(clusterInfo.Namespace) - logger.Infof("beginning admin cephx key rotation for cluster in namespace %q", clusterInfo.Namespace) + log.NamespacedInfo(clusterInfo.Namespace, logger, "beginning admin cephx key rotation") // As an optimization, use a (shallow) copy of clusterInfo with a context that won't be canceled // during normal runtime. This disallows the rotation process from being interrupted partway @@ -194,7 +195,7 @@ func recoverPriorAdminCephxKeyRotation( ) error { if clusterInfo == nil { // new cluster hasn't been deployed yet. there should be no way for recovery to be needed - logger.Debugf("no admin cephx key recovery possible for cluster with empty clusterInfo in namespace %q", clusterNamespace) + log.NamespacedDebug(clusterNamespace, logger, "no admin cephx key recovery possible for cluster with empty clusterInfo") return nil } @@ -203,7 +204,7 @@ func recoverPriorAdminCephxKeyRotation( rotatorKeyring, err := s.GetKeyringFromSecret(adminRotatorSecretPrefix) if err != nil { if kerrors.IsNotFound(err) { - logger.Debugf("no admin cephx key recovery needed for cluster in namespace %q", clusterInfo.Namespace) + log.NamespacedDebug(clusterNamespace, logger, "no admin cephx key recovery needed") return nil } return errors.Wrap(err, "failed to get admin rotator cephx key secret to check if admin cephx key rotation recovery is needed") @@ -214,7 +215,7 @@ func recoverPriorAdminCephxKeyRotation( } defer releaseAdminRotationLock(clusterNamespace) - logger.Infof("recovering from interrupted admin cephx key rotation for cluster in namespace %q", clusterInfo.Namespace) + log.NamespacedInfo(clusterNamespace, logger, "recovering from interrupted admin cephx key rotation") // As an optimization, use a (shallow) copy of clusterInfo with a context that won't be canceled // during normal runtime. This disallows the recovery process from being interrupted partway @@ -239,7 +240,7 @@ func recoverPriorAdminCephxKeyRotation( authList, err := cephclient.AuthList(clusterdCtx, clusterInfoCopy) if err != nil { - logger.Debugf("primary admin user failed to list ceph auth - this is an expected condition when recovering from admin cephx key rotation") + log.NamespacedDebug(clusterNamespace, logger, "primary admin user failed to list ceph auth - this is an expected condition when recovering from admin cephx key rotation") } else if !entityExistsInAuthList(adminRotatorUsername, authList) { // err == nil // corner case: if the primary client.admin works AND client.admin-rotator is not present in // the `auth ls` output, it means rotation succeeded and was interrupted during cleanup. @@ -271,7 +272,7 @@ func rotateAdminCephxKeyUsingRotator( ) error { // write admin rotator keyring file to tempdir so it can be used for rotation rotatorKeyringTmpfile := filepath.Join(tmpDir, adminRotatorUsername+".keyring") - logger.Infof("temporarily storing admin rotator keyring at %q for cluster in namespace %q", rotatorKeyringTmpfile, clusterInfo.Namespace) + log.NamespacedInfo(clusterInfo.Namespace, logger, "temporarily storing admin rotator keyring at %q", rotatorKeyringTmpfile) err := cephclient.WriteKeyring(rotatorKeyringTmpfile, rotatorKeyring) if err != nil { return errors.Wrapf(err, "failed to write admin rotator keyring to %q", rotatorKeyringTmpfile) @@ -287,7 +288,7 @@ func rotateAdminCephxKeyUsingRotator( } // as `client.admin-rotator`: run `ceph auth rotate client.admin` - logger.Infof("admin cephx key will be rotated for cluster in namespace %q. rook will restart afterwards. some reconciles and health checks may fail in between - this is normal", clusterInfo.Namespace) + log.NamespacedInfo(clusterInfo.Namespace, logger, "admin cephx key will be rotated. rook will restart afterwards. some reconciles and health checks may fail in between - this is normal") newAdminKey, err := cephclient.AuthRotate(clusterdCtx, rotatorInfo, cephclient.AdminUsername) if err != nil { return errors.Wrapf(err, "failed to rotate admin key using admin rotator client") @@ -296,7 +297,7 @@ func rotateAdminCephxKeyUsingRotator( // write new admin keyring to tempdir so it can be verified before it's stored permanently newAdminKeyringTmpfile := filepath.Join(tmpDir, cephclient.AdminUsername+".keyring") - logger.Infof("temporarily storing rotated admin cephx keyring at %q for cluster in namespace %q", newAdminKeyringTmpfile, clusterInfo.Namespace) + log.NamespacedInfo(clusterInfo.Namespace, logger, "temporarily storing rotated admin cephx keyring at %q", newAdminKeyringTmpfile) err = cephclient.WriteKeyring(newAdminKeyringTmpfile, newAdminKeyring) if err != nil { return errors.Wrapf(err, "failed to write admin rotator keyring to %q", newAdminKeyringTmpfile) @@ -345,7 +346,7 @@ func finalizeAdminKeyRotation( // clean up tmpDir now that it's no longer useful for recovery if err := os.RemoveAll(tmpDir); err != nil { // most likely, operator pod got rescheduled to a different node where dir isn't present - logger.Infof("non-critical failure removing admin cephx key rotation temp dir %q. %v", tmpDir, err) + log.NamespacedInfo(clusterInfo.Namespace, logger, "non-critical failure removing admin cephx key rotation temp dir %q. %v", tmpDir, err) } // delete the `rook-ceph-admin-rotator-keyring` secret (store.Delete) @@ -361,7 +362,7 @@ func finalizeAdminKeyRotation( if err != nil { // this should be rare, and failure here doesn't need to block finalization of admin key // rotation. status will be missing ceph version, but cluster can continue to reconcile - logger.Errorf("non-critical failure to determine missing ceph version of cluster in namespace %q for admin cephx key status", clusterInfo.Namespace) + log.NamespacedError(clusterInfo.Namespace, logger, "non-critical failure to determine missing ceph version of cluster for admin cephx key status") } else { clusterInfo.CephVersion = ver } @@ -375,7 +376,7 @@ func finalizeAdminKeyRotation( // restart Rook to force health checkers to restart with latest clusterInfo // do this even during rotation recovery since the health checkers could still be running - logger.Infof("restarting rook operator after successful admin cephx key rotation for cluster in namespace %q", clusterInfo.Namespace) + log.NamespacedInfo(clusterInfo.Namespace, logger, "restarting rook operator after successful admin cephx key rotation") reloadManagerFunc() return errSuccessfulAdminKeyRotation @@ -390,7 +391,7 @@ func updateCephClusterAdminCephxStatus(clusterdCtx *clusterd.Context, clusterInf } cephx := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, clusterInfo.CephVersion, cluster.Status.Cephx.Admin) cluster.Status.Cephx.Admin = cephx - logger.Debugf("updating admin key cephx status to %+v", cephx) + log.NamespacedDebug(clusterInfo.Namespace, logger, "updating admin key cephx status to %+v", cephx) if err := reporting.UpdateStatus(clusterdCtx.Client, cluster); err != nil { return errors.Wrap(err, "failed to update admin key cephx status") } diff --git a/pkg/operator/ceph/cluster/cleanup.go b/pkg/operator/ceph/cluster/cleanup.go index c04449722..8716ef2cf 100644 --- a/pkg/operator/ceph/cluster/cleanup.go +++ b/pkg/operator/ceph/cluster/cleanup.go @@ -34,6 +34,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/file/mirror" "github.com/rook/rook/pkg/operator/ceph/object" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" batch "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -59,10 +60,10 @@ var ( ) func (c *ClusterController) startClusterCleanUp(context context.Context, cluster *cephv1.CephCluster, cephHosts []string, monSecret, clusterFSID string) { - logger.Infof("starting clean up for cluster %q", cluster.Name) + log.NamespacedInfo(cluster.Namespace, logger, "starting clean up for cluster %q", cluster.Name) err := c.waitForCephDaemonCleanUp(context, cluster, time.Duration(clusterCleanUpPolicyRetryInterval)*time.Second) if err != nil { - logger.Errorf("failed to wait till ceph daemons are destroyed. %v", err) + log.NamespacedError(cluster.Namespace, logger, "failed to wait till ceph daemons are destroyed. %v", err) return } @@ -71,7 +72,7 @@ func (c *ClusterController) startClusterCleanUp(context context.Context, cluster func (c *ClusterController) startCleanUpJobs(cluster *cephv1.CephCluster, cephHosts []string, monSecret, clusterFSID string) { for _, hostName := range cephHosts { - logger.Infof("starting clean up job on node %q", hostName) + log.NamespacedInfo(cluster.Namespace, logger, "starting clean up job on node %q", hostName) jobName := k8sutil.TruncateNodeNameForJob("cluster-cleanup-job-%s", hostName) podSpec := c.cleanUpJobTemplateSpec(cluster, monSecret, clusterFSID) podSpec.Spec.NodeSelector = map[string]string{k8sutil.LabelHostname(): hostName} @@ -93,7 +94,7 @@ func (c *ClusterController) startCleanUpJobs(cluster *cephv1.CephCluster, cephHo cephv1.GetCleanupLabels(cluster.Spec.Labels).ApplyToObjectMeta(&job.ObjectMeta) if err := k8sutil.RunReplaceableJob(c.OpManagerCtx, c.context.Clientset, job, true); err != nil { - logger.Errorf("failed to run cluster clean up job on node %q. %v", hostName, err) + log.NamespacedError(cluster.Namespace, logger, "failed to run cluster clean up job on node %q. %v", hostName, err) } } } @@ -205,7 +206,7 @@ func getCleanupPlacement(c cephv1.ClusterSpec) cephv1.Placement { } func (c *ClusterController) waitForCephDaemonCleanUp(context context.Context, cluster *cephv1.CephCluster, retryInterval time.Duration) error { - logger.Infof("waiting for all the ceph daemons to be cleaned up in the cluster %q", cluster.Namespace) + log.NamespacedInfo(cluster.Namespace, logger, "waiting for all the ceph daemons to be cleaned up in the cluster %q", cluster.Namespace) for { select { case <-time.After(retryInterval): @@ -215,11 +216,11 @@ func (c *ClusterController) waitForCephDaemonCleanUp(context context.Context, cl } if len(cephHosts) == 0 { - logger.Info("all ceph daemons are cleaned up") + log.NamespacedInfo(cluster.Namespace, logger, "all ceph daemons are cleaned up") return nil } - logger.Debugf("waiting for ceph daemons in cluster %q to be cleaned up. Retrying in %q", + log.NamespacedDebug(cluster.Namespace, logger, "waiting for ceph daemons in cluster %q to be cleaned up. Retrying in %q", cluster.Namespace, retryInterval.String()) case <-context.Done(): return errors.Errorf("cancelling the host cleanup job. %s", context.Err()) @@ -250,7 +251,7 @@ func (c *ClusterController) getCephHosts(namespace string) ([]string, error) { fmt.Fprintf(&b, "%s: %d. ", app, len(podList.Items)) } - logger.Infof("existing ceph daemons in the namespace %q. %s", namespace, b.String()) + log.NamespacedInfo(namespace, logger, "existing ceph daemons in the namespace %q. %s", namespace, b.String()) for nodeName := range nodeNameList { podHostName, err := k8sutil.GetNodeHostName(c.OpManagerCtx, c.context.Clientset, nodeName) diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index fcd3105c8..70cd53054 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -46,6 +46,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/reporting" cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" rookversion "github.com/rook/rook/pkg/version" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -104,7 +105,7 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep // Execute actions before the monitors are up and running, if needed during upgrades. // These actions would be skipped in a new cluster. - logger.Debug("monitors are about to reconcile, executing pre actions") + log.NamespacedDebug(c.Namespace, logger, "monitors are about to reconcile, executing pre actions") err = c.preMonStartupActions(cephVersion) if err != nil { return errors.Wrap(err, "failed to execute actions before reconciling the ceph monitors") @@ -132,7 +133,7 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep } // Execute actions after the monitors are up and running - logger.Debug("monitors are up and running, executing post actions") + log.NamespacedDebug(c.Namespace, logger, "monitors are up and running, executing post actions") err = c.postMonStartupActions() if err != nil { return errors.Wrap(err, "failed to execute post actions after all the ceph monitors started") @@ -147,7 +148,7 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep } // Execute actions after the managers are up and running - logger.Debug("managers are up and running, executing post actions") + log.NamespacedDebug(c.Namespace, logger, "managers are up and running, executing post actions") err = c.postMgrStartupActions() if err != nil { return errors.Wrap(err, "failed to execute post actions after all the ceph managers started") @@ -168,7 +169,7 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep } } - logger.Infof("done reconciling ceph cluster in namespace %q", c.Namespace) + log.NamespacedInfo(c.Namespace, logger, "done reconciling ceph cluster") // We should be done updating by now if c.isUpgrade { @@ -186,7 +187,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error { cleanDataDirHostPath := path.Clean(cluster.Spec.DataDirHostPath) for _, b := range disallowedHostDirectories { if cleanDataDirHostPath == b { - logger.Errorf("dataDirHostPath (given: %q) must not be used, conflicts with %q internal path", cluster.Spec.DataDirHostPath, b) + log.NamespacedError(cluster.Namespace, logger, "dataDirHostPath (given: %q) must not be used, conflicts with %q internal path", cluster.Spec.DataDirHostPath, b) return nil } } @@ -202,7 +203,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error { clusterInfo, _, _, err := controller.LoadClusterInfo(c.context, c.OpManagerCtx, cluster.Namespace, cluster.Spec) if err != nil { if errors.Is(err, controller.ClusterInfoNoClusterNoSecret) { - logger.Info("clusterInfo not yet found, must be a new cluster.") + log.NamespacedInfo(cluster.Namespace, logger, "clusterInfo not yet found, must be a new cluster.") // ClusterInfoNoClusterNoSecret should return nil clusterInfo, but this is a // mandatory condition for recoverPriorAdminCephxKeyRotation() below, so ensure it clusterInfo = nil @@ -216,7 +217,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error { if cluster.mons.ClusterInfo == nil { // ClusterInfo stored in cluster.mons can be lost from the object stored in the // clusterMap during admin key rotation corner cases. rehydrate it if needed - logger.Debugf("applying missing ClusterInfo to tracked mons info for cluster in namespace %q", cluster.Namespace) + log.NamespacedDebug(cluster.Namespace, logger, "applying missing ClusterInfo to tracked mons info") cluster.mons.ClusterInfo = clusterInfo } } @@ -301,7 +302,7 @@ func (c *ClusterController) configureLocalCephCluster(cluster *cluster) error { // Validate the cluster Specs func preClusterStartValidation(cluster *cluster) error { if cluster.Spec.Mon.Count == 0 { - logger.Warningf("mon count should be at least 1, will use default value of %d", mon.DefaultMonCount) + log.NamespacedWarning(cluster.Namespace, logger, "mon count should be at least 1, will use default value of %d", mon.DefaultMonCount) cluster.Spec.Mon.Count = mon.DefaultMonCount } if !cluster.Spec.Mon.AllowMultiplePerNode { @@ -328,7 +329,7 @@ func preClusterStartValidation(cluster *cluster) error { } } - logger.Debug("cluster spec successfully validated") + log.NamespacedDebug(cluster.Namespace, logger, "cluster spec successfully validated") return nil } @@ -391,7 +392,7 @@ func (c *cluster) replaceDefaultReplicationRule(newRoot string) error { // CRUSH root // in both cases, we cannot do anything about it either way and // we’ll assume that the user knows what they’re doing. - logger.Warning("replicated_rule is in use, not replaced") + log.NamespacedWarning(c.Namespace, logger, "replicated_rule is in use, not replaced") return nil } // the error does not refer to EBUSY -> return as error @@ -432,7 +433,7 @@ func (c *cluster) removeDefaultCrushRoot() error { // // in all cases, we cannot do anything about it either way and // we’ll assume that the user knows what they’re doing. - logger.Debug("default is not empty or is still in use, not removed") + log.NamespacedDebug(c.Namespace, logger, "default is not empty or is still in use, not removed") return nil } } @@ -446,19 +447,19 @@ func (c *cluster) removeDefaultCrushRoot() error { // Those objects may interfere with the normal operation of the cluster. // Note that errors which indicate that the objects are in use are ignored and the objects will continue to exist in that case. func (c *cluster) replaceDefaultCrushMap(newRoot string) (err error) { - logger.Info("creating new CRUSH root if it does not exist") + log.NamespacedInfo(c.Namespace, logger, "creating new CRUSH root if it does not exist") err = c.createCrushRoot(newRoot) if err != nil { return errors.Wrap(err, "failed to create CRUSH root") } - logger.Info("replacing default replicated_rule CRUSH rule for use of non-default CRUSH root") + log.NamespacedInfo(c.Namespace, logger, "replacing default replicated_rule CRUSH rule for use of non-default CRUSH root") err = c.replaceDefaultReplicationRule(newRoot) if err != nil { return errors.Wrap(err, "failed to replace default rule") } - logger.Info("replacing default CRUSH node if applicable") + log.NamespacedInfo(c.Namespace, logger, "replacing default CRUSH node if applicable") err = c.removeDefaultCrushRoot() if err != nil { return errors.Wrap(err, "failed to remove default CRUSH root") @@ -589,12 +590,12 @@ func (c *cluster) configureStorageSettings() error { func (c *cluster) setClusterFullRatio(ratioCommand string, desiredRatio *float64, actualRatio float64) error { if !shouldUpdateFloatSetting(desiredRatio, actualRatio) { if desiredRatio != nil { - logger.Infof("desired value %s=%.2f is already set", ratioCommand, *desiredRatio) + log.NamespacedInfo(c.Namespace, logger, "desired value %s=%.2f is already set", ratioCommand, *desiredRatio) } return nil } desiredStringVal := fmt.Sprintf("%.2f", *desiredRatio) - logger.Infof("updating %s from %.2f to %s", ratioCommand, actualRatio, desiredStringVal) + log.NamespacedInfo(c.Namespace, logger, "updating %s from %.2f to %s", ratioCommand, actualRatio, desiredStringVal) args := []string{"osd", ratioCommand, desiredStringVal} cephCmd := client.NewCephCommand(c.context, c.ClusterInfo, args) output, err := cephCmd.Run() @@ -648,7 +649,7 @@ func (c *cluster) reportTelemetry() { } func reportClusterTelemetry(c *cluster) { - logger.Info("reporting cluster telemetry") + log.NamespacedInfo(c.Namespace, logger, "reporting cluster telemetry") // Identify this as a rook cluster for Ceph telemetry by setting the Rook version. telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.RookVersionKey, rookversion.Version) @@ -656,7 +657,7 @@ func reportClusterTelemetry(c *cluster) { // Report the K8s version serverVersion, err := c.context.Clientset.Discovery().ServerVersion() if err != nil { - logger.Warningf("failed to report the K8s server version. %v", err) + log.NamespacedWarning(c.Namespace, logger, "failed to report the K8s server version. %v", err) } else { telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.K8sVersionKey, serverVersion.String()) } @@ -699,13 +700,13 @@ func reportClusterTelemetry(c *cluster) { } func reportNodeTelemetry(c *cluster) { - logger.Info("reporting node telemetry") + log.NamespacedInfo(c.Namespace, logger, "reporting node telemetry") operatorNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar) // Report the K8sNodeCount nodelist, err := c.context.Clientset.CoreV1().Nodes().List(c.ClusterInfo.Context, metav1.ListOptions{}) if err != nil { - logger.Warningf("failed to report the K8s node count. %v", err) + log.NamespacedWarning(c.Namespace, logger, "failed to report the K8s node count. %v", err) } else { telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.K8sNodeCount, strconv.Itoa(len(nodelist.Items))) } @@ -717,7 +718,7 @@ func reportNodeTelemetry(c *cluster) { listoption := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, nodedaemon.CrashCollectorAppName)} cephNodeList, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(c.ClusterInfo.Context, listoption) if err != nil { - logger.Warningf("failed to report the ceph node count. %v", err) + log.NamespacedWarning(c.Namespace, logger, "failed to report the ceph node count. %v", err) } else { telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.CephNodeCount, strconv.Itoa(len(cephNodeList.Items))) } @@ -726,7 +727,7 @@ func reportNodeTelemetry(c *cluster) { listoption := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, csi.CsiRBDPlugin)} cephRbdNodelist, err := c.context.Clientset.CoreV1().Pods(operatorNamespace).List(c.ClusterInfo.Context, listoption) if err != nil { - logger.Warningf("failed to report the ceph rbd node count. %v", err) + log.NamespacedWarning(c.Namespace, logger, "failed to report the ceph rbd node count. %v", err) } else { telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.RBDNodeCount, strconv.Itoa(len(cephRbdNodelist.Items))) } @@ -735,7 +736,7 @@ func reportNodeTelemetry(c *cluster) { listoption = metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, csi.CsiCephFSPlugin)} cephFSNodelist, err := c.context.Clientset.CoreV1().Pods(operatorNamespace).List(c.ClusterInfo.Context, listoption) if err != nil { - logger.Warningf("failed to report the ceph cephfs node count. %v", err) + log.NamespacedWarning(c.Namespace, logger, "failed to report the ceph cephfs node count. %v", err) } else { telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.CephFSNodeCount, strconv.Itoa(len(cephFSNodelist.Items))) } @@ -744,7 +745,7 @@ func reportNodeTelemetry(c *cluster) { listoption = metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, csi.CsiNFSPlugin)} cephNFSNodelist, err := c.context.Clientset.CoreV1().Pods(operatorNamespace).List(c.ClusterInfo.Context, listoption) if err != nil { - logger.Warningf("failed to report the ceph nfs node count. %v", err) + log.NamespacedWarning(c.Namespace, logger, "failed to report the ceph nfs node count. %v", err) } else { telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.NFSNodeCount, strconv.Itoa(len(cephNFSNodelist.Items))) } @@ -764,7 +765,7 @@ func (c *cluster) configureMsgr2() error { encryptionEnabled := c.Spec.NetworkEncryptionEnabled() if encryptionEnabled { - logger.Infof("setting msgr2 encryption mode to %q", encryptionSetting) + log.NamespacedInfo(c.Namespace, logger, "setting msgr2 encryption mode to %q", encryptionSetting) if err := monStore.SetAll("global", encryptionGlobalConfigSettings); err != nil { return err } @@ -797,7 +798,7 @@ func (c *cluster) configureMsgr2() error { globalConfigSettings := map[string]string{ "ms_osd_compress_mode": "force", } - logger.Infof("setting msgr2 compression mode to %q", "force") + log.NamespacedInfo(c.Namespace, logger, "setting msgr2 compression mode to %q", "force") if err := monStore.SetAll("global", globalConfigSettings); err != nil { return err } @@ -819,9 +820,9 @@ func (c *cluster) fetchCephConfigFromSecrets() (map[string]map[string]string, er key, module, selector.LocalObjectReference.Name, err) } - logger.Debugf("setting Ceph config key %q in module %q from secret %q", + log.NamespacedDebug(c.Namespace, logger, "setting Ceph config key %q in module %q from secret %q", key, module, selector.LocalObjectReference.Name) - logger.Tracef("setting Ceph config key %q in module %q to value %q from secret %q", + log.NamespacedTrace(c.Namespace, logger, "setting Ceph config key %q in module %q to value %q from secret %q", key, module, val, selector.LocalObjectReference.Name) result[module][key] = val } @@ -851,7 +852,7 @@ func (c *cluster) fetchSecretValue(selector v1.SecretKeySelector) (string, error func initClusterCephxStatus(c *cluster) error { initErr := c.ClusterInfo.IsInitialized() if initErr == nil { - logger.Debugf("not setting uninitialized cephx status on already initialized CephCluster in namespace %q", c.Namespace) + log.NamespacedDebug(c.Namespace, logger, "not setting uninitialized cephx status on already initialized CephCluster") return nil } if c.ClusterInfo.Context.Err() != nil { @@ -874,7 +875,7 @@ func initClusterCephxStatus(c *cluster) error { } uninitializedStatus := keyring.UninitializedCephxStatus() - logger.Infof("initializing cephx status for CephCluster in namespace %q", c.Namespace) + log.NamespacedInfo(c.Namespace, logger, "initializing cephx status") clusterObj.Status.Cephx = cephv1.ClusterCephxStatus{ Mon: uninitializedStatus, Mgr: uninitializedStatus, @@ -901,9 +902,9 @@ func (c *cluster) deleteBootstrapKeys() { for _, bootstrapkey := range bootstrapKeys { err := client.AuthDelete(c.context, c.ClusterInfo, bootstrapkey) if err != nil { - logger.Debugf("failed to delete bootstrap key %q in the namespace %q. Error: %v", bootstrapkey, c.Namespace, err) + log.NamespacedDebug(c.Namespace, logger, "failed to delete bootstrap key %q. Error: %v", bootstrapkey, err) } else { - logger.Infof("successfully deleted bootstrap key %q in the namespace %q", bootstrapkey, c.Namespace) + log.NamespacedInfo(c.Namespace, logger, "successfully deleted bootstrap key %q", bootstrapkey) } } } diff --git a/pkg/operator/ceph/cluster/cluster_external.go b/pkg/operator/ceph/cluster/cluster_external.go index 5baeac457..6a12d515c 100644 --- a/pkg/operator/ceph/cluster/cluster_external.go +++ b/pkg/operator/ceph/cluster/cluster_external.go @@ -33,6 +33,7 @@ import ( opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/csi" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -65,7 +66,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error if cluster.Spec.CephVersion.Image == "" { err = mon.WriteConnectionConfig(c.context, cluster.ClusterInfo) if err != nil { - logger.Errorf("failed to write config. attempting to continue. %v", err) + log.NamespacedError(cluster.Namespace, logger, "failed to write config. attempting to continue. %v", err) } } @@ -81,13 +82,13 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error // If we don't do this, daemons will never start, waiting forever for this configmap to be present // // Only do this when doing a bit of management... - logger.Infof("creating %q configmap", k8sutil.ConfigOverrideName) + log.NamespacedInfo(cluster.Namespace, logger, "creating %q configmap", k8sutil.ConfigOverrideName) err = populateConfigOverrideConfigMap(c.context, cluster.namespacedName.Namespace, cluster.ClusterInfo.OwnerInfo, cluster.clusterMetadata) if err != nil { return errors.Wrap(err, "failed to populate config override config map") } - logger.Infof("creating %q secret", config.StoreName) + log.NamespacedInfo(cluster.Namespace, logger, "creating %q secret", config.StoreName) err = config.GetStore(c.context, cluster.namespacedName.Namespace, cluster.ClusterInfo.OwnerInfo).CreateOrUpdate(cluster.ClusterInfo) if err != nil { return errors.Wrap(err, "failed to update the global config") @@ -98,7 +99,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error if err := cluster.ClusterInfo.IsInitialized(); err != nil { return errors.Wrap(err, "the cluster identity was not established") } - logger.Info("external cluster identity established") + log.NamespacedInfo(cluster.Namespace, logger, "external cluster identity established") // Create CSI Secrets only if the user has provided the admin key if cluster.ClusterInfo.CephCred.Username == client.AdminUsername { @@ -117,7 +118,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error cluster.Spec.Network.Connections = &cephv1.ConnectionsSpec{} } cluster.Spec.Network.Connections.RequireMsgr2 = true - logger.Debugf("a v2 port was found for a mon endpoint, so msgr2 is required") + log.NamespacedDebug(cluster.Namespace, logger, "a v2 port was found for a mon endpoint, so msgr2 is required") break } } @@ -136,7 +137,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error if err != nil { return errors.Wrap(err, "failed to update csi cluster config") } - logger.Info("successfully updated csi config map") + log.NamespacedInfo(cluster.Namespace, logger, "successfully updated csi config map") // Create Crash Collector Secret if !cluster.Spec.CrashCollector.Disable { @@ -177,7 +178,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error } if csi.EnableCSIOperator() { - logger.Info("create cephConnection and defaultClientProfile for external mode") + log.NamespacedInfo(cluster.Namespace, logger, "create cephConnection and defaultClientProfile for external mode") err = csi.CreateUpdateCephConnection(c.context.Client, cluster.ClusterInfo, *cluster.Spec) if err != nil { return errors.Wrap(err, "failed to create/update cephConnection") @@ -202,7 +203,7 @@ func purgeExternalCluster(clientset kubernetes.Interface, namespace string) { for _, cm := range cmsToDelete { err := clientset.CoreV1().ConfigMaps(namespace).Delete(ctx, cm, metav1.DeleteOptions{}) if err != nil && !kerrors.IsNotFound(err) { - logger.Errorf("failed to delete config map %q. %v", cm, err) + log.NamespacedError(namespace, logger, "failed to delete config map %q. %v", cm, err) } } @@ -219,7 +220,7 @@ func purgeExternalCluster(clientset kubernetes.Interface, namespace string) { for _, secret := range secretsToDelete { err := clientset.CoreV1().Secrets(namespace).Delete(ctx, secret, metav1.DeleteOptions{}) if err != nil && !kerrors.IsNotFound(err) { - logger.Errorf("failed to delete secret %q. %v", secret, err) + log.NamespacedError(namespace, logger, "failed to delete secret %q. %v", secret, err) } } } @@ -255,12 +256,12 @@ func (c *ClusterController) configureExternalClusterMonitoring(context *clusterd if err != nil { return err } - logger.Info("creating mgr external monitoring service") + log.NamespacedInfo(cluster.Namespace, logger, "creating mgr external monitoring service") _, err = k8sutil.CreateOrUpdateService(cluster.ClusterInfo.Context, context.Clientset, cluster.Namespace, service) if err != nil && !kerrors.IsAlreadyExists(err) { return errors.Wrap(err, "failed to create or update mgr service") } - logger.Info("mgr external metrics service created") + log.NamespacedInfo(cluster.Namespace, logger, "mgr external metrics service created") // Configure external metrics endpoint err = opcontroller.ConfigureExternalMetricsEndpoint(context, cluster.Spec.Monitoring, cluster.ClusterInfo, cluster.ownerInfo) @@ -269,13 +270,13 @@ func (c *ClusterController) configureExternalClusterMonitoring(context *clusterd } // Deploy external ServiceMonitor - logger.Info("creating external service monitor") + log.NamespacedInfo(cluster.Namespace, logger, "creating external service monitor") // servicemonitor takes some metadata from the service for easy mapping err = manager.EnableServiceMonitor() if err != nil { - logger.Errorf("failed to enable external service monitor. %v", err) + log.NamespacedError(cluster.Namespace, logger, "failed to enable external service monitor. %v", err) } else { - logger.Info("external service monitor created") + log.NamespacedInfo(cluster.Namespace, logger, "external service monitor created") } return nil } diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index cfc3685ca..bdc9e5da1 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -39,6 +39,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/csi" "github.com/rook/rook/pkg/operator/ceph/reporting" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -284,7 +285,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R err := r.client.Get(r.opManagerContext, request.NamespacedName, cephCluster) if err != nil { if kerrors.IsNotFound(err) { - logger.Debug("cephCluster resource not found. Ignoring since object must be deleted.") + log.NamespacedDebug(request.Namespace, logger, "cephCluster resource not found. Ignoring since object must be deleted.") return reconcile.Result{}, *cephCluster, nil } // Error reading the object - requeue the request. @@ -297,7 +298,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R return reconcile.Result{}, *cephCluster, errors.Wrap(err, "failed to add finalizer") } if generationUpdated { - logger.Infof("reconciling the ceph cluster %q after adding finalizer", cephCluster.Name) + log.NamespacedInfo(request.Namespace, logger, "reconciling the ceph cluster %q after adding finalizer", cephCluster.Name) return reconcile.Result{}, *cephCluster, nil } @@ -317,7 +318,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R // started). So we would get messages from the other go routine printing a message // repeatedly that the context is cancelled. if errors.Is(r.opManagerContext.Err(), context.Canceled) { - logger.Info("context cancelled, exiting reconcile") + log.NamespacedInfo(request.Namespace, logger, "context cancelled, exiting reconcile") return reconcile.Result{}, *cephCluster, nil } @@ -357,7 +358,7 @@ func (r *ReconcileCephCluster) reconcileDelete(cephCluster *cephv1.CephCluster) if cephCluster.Spec.CleanupPolicy.HasDataDirCleanPolicy() && !cephCluster.Spec.External.Enable { monSecret, clusterFSID, err := r.clusterController.getCleanUpDetails(&cephCluster.Spec, cephCluster.Namespace) if err != nil { - logger.Warningf("failed to get mon secret. skip cluster cleanup. remove finalizer. %v", err) + log.NamespacedWarning(cephCluster.Namespace, logger, "failed to get mon secret. skip cluster cleanup. remove finalizer. %v", err) doCleanup = false } @@ -400,7 +401,7 @@ func NewClusterController(context *clusterd.Context, rookImage string) *ClusterC func (c *ClusterController) reconcileCephCluster(clusterObj *cephv1.CephCluster, ownerInfo *k8sutil.OwnerInfo) error { if clusterObj.Spec.CleanupPolicy.HasDataDirCleanPolicy() { - logger.Infof("skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set", clusterObj.Name, clusterObj.Namespace) + log.NamespacedInfo(clusterObj.Namespace, logger, "skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set", clusterObj.Name, clusterObj.Namespace) return nil } var clustr *cluster @@ -425,7 +426,7 @@ func (c *ClusterController) reconcileCephCluster(clusterObj *cephv1.CephCluster, clustr.Spec = &clusterObj.Spec c.clusterMap.Store(clustr.Namespace, clustr) - logger.Infof("reconciling ceph cluster in namespace %q", clustr.namespacedName.Namespace) + log.NamedInfo(clustr.namespacedName, logger, "reconciling ceph cluster") // Start the main ceph cluster orchestration return c.initializeCluster(clustr) @@ -438,14 +439,14 @@ func (c *ClusterController) requestClusterDelete(clusterObj *cephv1.CephCluster) if rawCluster, ok := c.clusterMap.Load(clusterObj.Namespace); ok { existingCluster = rawCluster.(*cluster) if existingCluster.namespacedName.Name != clusterObj.Name { - logger.Errorf("skipping deletion of CephCluster %q. CephCluster CR %q already exists in this namespace. only one cluster cr per namespace is supported.", + log.NamespacedError(nsName, logger, "skipping deletion of CephCluster %q. CephCluster CR %q already exists in this namespace. only one cluster cr per namespace is supported.", nsName, existingCluster.namespacedName.Name) return reconcile.Result{}, nil // do not requeue the delete } } _, err := c.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(c.OpManagerCtx, "networkfences.csiaddons.openshift.io", metav1.GetOptions{}) if err == nil { - logger.Info("removing networkFence if matching cephCluster UID found") + log.NamespacedInfo(clusterObj.Namespace, logger, "removing networkFence if matching cephCluster UID found") networkFenceList := &addonsv1alpha1.NetworkFenceList{} labelSelector := labels.SelectorFromSet(map[string]string{ networkFenceLabel: string(clusterObj.GetUID()), @@ -476,7 +477,7 @@ func (c *ClusterController) requestClusterDelete(clusterObj *cephv1.CephCluster) } } - logger.Infof("cleaning up CephCluster %q", nsName) + log.NamespacedInfo(clusterObj.Namespace, logger, "cleaning up CephCluster %q", nsName) if existingCluster != nil { // We used to stop the bucket controller here but when we get a DELETE event for the CephCluster // we will reload the CRD manager anyway so the bucket controller go routine will be stopped @@ -495,7 +496,7 @@ func (c *ClusterController) requestClusterDelete(clusterObj *cephv1.CephCluster) } if clusterObj.Spec.CleanupPolicy.AllowUninstallWithVolumes { - logger.Info("skipping check for existing PVs as allowUninstallWithVolumes is set to true") + log.NamespacedInfo(clusterObj.Namespace, logger, "skipping check for existing PVs as allowUninstallWithVolumes is set to true") } else { err := c.checkIfVolumesExist(clusterObj) if err != nil { @@ -510,7 +511,7 @@ func (c *ClusterController) requestClusterDelete(clusterObj *cephv1.CephCluster) // Delete keys from KMS err := c.deleteOSDEncryptionKeyFromKMS(clusterObj) if err != nil { - logger.Errorf("failed to delete osd encryption keys for CephCluster %q from kms; deletion will continue. %v", nsName, err) + log.NamespacedError(clusterObj.Namespace, logger, "failed to delete osd encryption keys for CephCluster %q from kms; deletion will continue. %v", nsName, err) } } @@ -534,7 +535,7 @@ func (c *ClusterController) checkIfVolumesExist(cluster *cephv1.CephCluster) err func (c *ClusterController) csiVolumesAllowForDeletion(cluster *cephv1.CephCluster) error { drivers := []string{csi.CephFSDriverName, csi.RBDDriverName} - logger.Infof("checking any PVC created by drivers %q and %q with clusterID %q", csi.CephFSDriverName, csi.RBDDriverName, cluster.Namespace) + log.NamespacedInfo(cluster.Namespace, logger, "checking any PVC created by drivers %q and %q with clusterID %q", csi.CephFSDriverName, csi.RBDDriverName, cluster.Namespace) // check any PV is created in this cluster attachmentsExist, err := c.checkPVPresentInCluster(drivers, cluster.Namespace) if err != nil { @@ -542,7 +543,7 @@ func (c *ClusterController) csiVolumesAllowForDeletion(cluster *cephv1.CephClust } // no PVC created in this cluster if !attachmentsExist { - logger.Infof("no volume attachments for cluster %q", cluster.Namespace) + log.NamespacedInfo(cluster.Namespace, logger, "no volume attachments for cluster %q", cluster.Namespace) return nil } @@ -557,7 +558,7 @@ func (c *ClusterController) checkPVPresentInCluster(drivers []string, clusterID for _, p := range pv.Items { if p.Spec.CSI == nil { - logger.Errorf("Spec.CSI is nil for PV %q", p.Name) + log.NamespacedError(clusterID, logger, "Spec.CSI is nil for PV %q", p.Name) continue } if p.Spec.CSI.VolumeAttributes["clusterID"] == clusterID { @@ -600,7 +601,7 @@ func (r *ReconcileCephCluster) removeFinalizer(client client.Client, name types. err := client.Get(r.opManagerContext, name, obj) if err != nil { if kerrors.IsNotFound(err) { - logger.Debugf("%s resource not found. Ignoring since object must be deleted.", name.Name) + log.NamedDebug(name, logger, "%s resource not found. Ignoring since object must be deleted.", name.Name) return nil } return errors.Wrapf(err, "failed to retrieve %q to remove finalizer", name.Name) @@ -666,7 +667,7 @@ func (c *ClusterController) deleteOSDEncryptionKeyFromKMS(currentCluster *cephv1 // Generate and store the encrypted key in whatever KMS is configured err = kmsConfig.DeleteSecret(osdPVC.Name) if err != nil { - logger.Errorf("failed to delete secret. %v", err) + log.NamespacedError(currentCluster.Namespace, logger, "failed to delete secret. %v", err) continue } } diff --git a/pkg/operator/ceph/cluster/mgr/config.go b/pkg/operator/ceph/cluster/mgr/config.go index bec9a7482..5b53c30af 100644 --- a/pkg/operator/ceph/cluster/mgr/config.go +++ b/pkg/operator/ceph/cluster/mgr/config.go @@ -22,6 +22,7 @@ import ( "github.com/pkg/errors" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" + "github.com/rook/rook/pkg/util/log" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -89,7 +90,7 @@ func (c *Cluster) generateKeyring(m *mgrConfig) (string, error) { } if c.shouldRotateCephxKeys { - logger.Infof("rotating cephx key for mgr daemon %q in the namespace %q", m.ResourceName, c.clusterInfo.Namespace) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "rotating cephx key for mgr daemon %q in the namespace %q", m.ResourceName, c.clusterInfo.Namespace) newKey, err := s.RotateKey(user) if err != nil { return "", errors.Wrapf(err, "failed to rotate cephx key for mgr daemon %q", m.ResourceName) @@ -101,9 +102,9 @@ func (c *Cluster) generateKeyring(m *mgrConfig) (string, error) { err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Delete(c.clusterInfo.Context, m.ResourceName, metav1.DeleteOptions{}) if err != nil { if kerrors.IsNotFound(err) { - logger.Debugf("legacy mgr key %q is already removed", m.ResourceName) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "legacy mgr key %q is already removed", m.ResourceName) } else { - logger.Warningf("legacy mgr key %q could not be removed. %v", m.ResourceName, err) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "legacy mgr key %q could not be removed. %v", m.ResourceName, err) } } diff --git a/pkg/operator/ceph/cluster/mgr/dashboard.go b/pkg/operator/ceph/cluster/mgr/dashboard.go index c7f32d62c..df09a0f9b 100644 --- a/pkg/operator/ceph/cluster/mgr/dashboard.go +++ b/pkg/operator/ceph/cluster/mgr/dashboard.go @@ -32,6 +32,7 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util" "github.com/rook/rook/pkg/util/exec" + "github.com/rook/rook/pkg/util/log" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -91,7 +92,7 @@ func (c *Cluster) configureDashboardModules() error { } } else { if err := client.MgrDisableModule(c.context, c.clusterInfo, dashboardModuleName); err != nil { - logger.Errorf("failed to disable mgr dashboard module. %v", err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to disable mgr dashboard module. %v", err) } return nil } @@ -106,7 +107,7 @@ func (c *Cluster) configureDashboardModules() error { return err } if secureRequiresRestart || configChanged { - logger.Info("dashboard config has changed. restarting the dashboard module") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "dashboard config has changed. restarting the dashboard module") return c.restartMgrModule(dashboardModuleName) } return nil @@ -131,16 +132,16 @@ func (c *Cluster) deleteManagerDaemonConfiguration() bool { for _, key := range mgrKeysToDelete { err := monStore.Delete(mgrDaemonID, key) if err != nil { - logger.Errorf("failed to delete configuration entry %q %q, err: %v", mgrDaemonID, key, err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to delete configuration entry %q %q, err: %v", mgrDaemonID, key, err) success = false } } } if success { - logger.Info("All per-daemon mgr configuration has been deleted successfully.") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "All per-daemon mgr configuration has been deleted successfully.") } else { - logger.Error("At least one delete operation failed while trying to delete per-daemon mgr configuration.") + log.NamespacedError(c.clusterInfo.Namespace, logger, "At least one delete operation failed while trying to delete per-daemon mgr configuration.") } return success @@ -236,10 +237,10 @@ func (c *Cluster) createSelfSignedCert() (bool, error) { args := []string{"config-key", "get", "mgr/dashboard/crt"} output, err := client.NewCephCommand(c.context, c.clusterInfo, args).RunWithTimeout(exec.CephCommandsTimeout) if err == nil && len(output) > 0 { - logger.Info("dashboard is already initialized with a cert") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "dashboard is already initialized with a cert") return true, nil } - logger.Debugf("dashboard cert does not appear to exist. err=%v", err) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "dashboard cert does not appear to exist. err=%v", err) // create a self-signed cert for the https connections args = []string{"dashboard", "create-self-signed-cert"} @@ -248,14 +249,14 @@ func (c *Cluster) createSelfSignedCert() (bool, error) { for i := 0; i < 5; i++ { _, err := client.NewCephCommand(c.context, c.clusterInfo, args).RunWithTimeout(exec.CephCommandsTimeout) if err == context.DeadlineExceeded { - logger.Warning("cert creation timed out. trying again") + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "cert creation timed out. trying again") continue } if err != nil { exitCode, parsed := c.exitCode(err) if parsed { if exitCode == invalidArgErrorCode { - logger.Info("dashboard module is not ready yet. trying again") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "dashboard module is not ready yet. trying again") time.Sleep(dashboardInitWaitTime) continue } @@ -263,16 +264,16 @@ func (c *Cluster) createSelfSignedCert() (bool, error) { return false, errors.Wrap(err, "failed to create self signed cert on mgr") } } - logger.Info("dashboard cert created") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "dashboard cert created") return false, nil } - logger.Info("dashboard cert creation exceeded retries") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "dashboard cert creation exceeded retries") return false, nil } func (c *Cluster) setLoginCredentials(password string) error { // Set the login credentials. Write the command/args to the debug log so we don't write the password by default to the log. - logger.Infof("setting ceph dashboard %q login creds", dashboardUsername) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "setting ceph dashboard %q login creds", dashboardUsername) var args []string // for latest Ceph versions @@ -284,7 +285,7 @@ func (c *Cluster) setLoginCredentials(password string) error { defer func() { if err := os.Remove(file.Name()); err != nil { - logger.Errorf("failed to clean up dashboard password file %q. %v", file.Name(), err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to clean up dashboard password file %q. %v", file.Name(), err) } }() @@ -314,14 +315,14 @@ func (c *Cluster) setLoginCredentials(password string) error { return errors.Wrapf(err, "failed to set password for user %q", dashboardUsername) } - logger.Info("successfully set ceph dashboard creds") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "successfully set ceph dashboard creds") return nil } func (c *Cluster) getOrGenerateDashboardPassword() (string, error) { secret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(c.clusterInfo.Context, dashboardPasswordName, metav1.GetOptions{}) if err == nil { - logger.Info("the dashboard secret was already generated") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "the dashboard secret was already generated") return decodeSecret(secret) } if !kerrors.IsNotFound(err) { diff --git a/pkg/operator/ceph/cluster/mgr/drain.go b/pkg/operator/ceph/cluster/mgr/drain.go index 0f91fcffd..18932d313 100644 --- a/pkg/operator/ceph/cluster/mgr/drain.go +++ b/pkg/operator/ceph/cluster/mgr/drain.go @@ -19,6 +19,7 @@ package mgr import ( "github.com/pkg/errors" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" policyv1 "k8s.io/api/policy/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -63,13 +64,13 @@ func (c *Cluster) deleteMgrPDB() { err := c.context.Client.Get(c.clusterInfo.Context, pdbRequest, mgrPDB) if err != nil { if !kerrors.IsNotFound(err) { - logger.Errorf("failed to get mgr pdb %q. %v", mgrPDBName, err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get mgr pdb %q. %v", mgrPDBName, err) } return } - logger.Debugf("ensuring the mgr pdb %q is deleted", mgrPDBName) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "ensuring the mgr pdb %q is deleted", mgrPDBName) err = c.context.Client.Delete(c.clusterInfo.Context, mgrPDB) if err != nil { - logger.Errorf("failed to delete mgr pdb %q. %v", mgrPDBName, err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to delete mgr pdb %q. %v", mgrPDBName, err) } } diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go index 0ecdf20b7..2e720d9c2 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr.go +++ b/pkg/operator/ceph/cluster/mgr/mgr.go @@ -36,6 +36,7 @@ import ( cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/exec" + "github.com/rook/rook/pkg/util/log" v1 "k8s.io/api/apps/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -115,7 +116,7 @@ func (c *Cluster) Start() error { return errors.Wrap(err, "error checking pod memory") } - logger.Infof("start running mgr") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "start running mgr") daemonIDs := c.getDaemonIDs() var deploymentsToWaitFor []*v1.Deployment @@ -130,7 +131,7 @@ func (c *Cluster) Start() error { } if c.shouldRotateCephxKeys { - logger.Infof("cephx keys for mgr daemons in the namespace %q will be rotated", c.clusterInfo.Namespace) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "cephx keys for mgr daemons in the namespace %q will be rotated", c.clusterInfo.Namespace) } for _, daemonID := range daemonIDs { @@ -167,7 +168,7 @@ func (c *Cluster) Start() error { } if mgrsToSkipReconcile.Has(daemonID) { - logger.Warningf("Skipping reconcile of mgr %q labeled with %q", daemonID, cephv1.SkipReconcileLabelKey) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "Skipping reconcile of mgr %q labeled with %q", daemonID, cephv1.SkipReconcileLabelKey) continue } @@ -176,13 +177,13 @@ func (c *Cluster) Start() error { if !kerrors.IsAlreadyExists(err) { return errors.Wrapf(err, "failed to create mgr deployment %s", resourceName) } - logger.Infof("deployment for mgr %s already exists. updating if needed", resourceName) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "deployment for mgr %s already exists. updating if needed", resourceName) if err := updateDeploymentAndWait(c.context, c.clusterInfo, d, config.MgrType, mgrConfig.DaemonID, c.spec.SkipUpgradeChecks, c.spec.ContinueUpgradeAfterChecksEvenIfNotHealthy); err != nil { - logger.Errorf("failed to update mgr deployment %q. %v", resourceName, err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to update mgr deployment %q. %v", resourceName, err) if c.spec.ContinueUpgradeAfterChecksEvenIfNotHealthy { - logger.Infof("continuing reconcile of ceph cluster in namespace %s after error waiting for mgr because continueUpgradeAfterChecksEvenIfNotHealthy is true", c.clusterInfo.Namespace) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "continuing reconcile of ceph cluster in namespace %s after error waiting for mgr because continueUpgradeAfterChecksEvenIfNotHealthy is true", c.clusterInfo.Namespace) } else { - logger.Infof("stopping reconcile of ceph cluster in namespace %s after error waiting for mgr because continueUpgradeAfterChecksEvenIfNotHealthy is false", c.clusterInfo.Namespace) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "stopping reconcile of ceph cluster in namespace %s after error waiting for mgr because continueUpgradeAfterChecksEvenIfNotHealthy is false", c.clusterInfo.Namespace) return errors.Wrapf(err, "failed to update mgr deployment %q", resourceName) } } @@ -237,11 +238,11 @@ func (c *Cluster) removeExtraMgrs(daemonIDs []string) { options := metav1.ListOptions{LabelSelector: "app=" + AppName} mgrDeployments, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(c.clusterInfo.Context, options) if err != nil { - logger.Warningf("failed to check for extra mgrs. %v", err) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to check for extra mgrs. %v", err) return } if len(mgrDeployments.Items) == len(daemonIDs) { - logger.Debugf("expected number %d of mgrs found", len(daemonIDs)) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "expected number %d of mgrs found", len(daemonIDs)) return } @@ -263,9 +264,9 @@ func (c *Cluster) removeExtraMgrs(daemonIDs []string) { if !found { err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(c.clusterInfo.Context, mgrDeployment.Name, metav1.DeleteOptions{}) if err == nil { - logger.Infof("removed extra mgr %q", mgrDeployment.Name) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "removed extra mgr %q", mgrDeployment.Name) } else { - logger.Warningf("failed to remove extra mgr %q. %v", mgrDeployment.Name, err) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to remove extra mgr %q. %v", mgrDeployment.Name, err) } } } @@ -278,7 +279,7 @@ func (c *Cluster) SetMgrRoleLabel(daemonNameToUpdate string, isActive bool) erro LabelSelector: fmt.Sprintf("%s=%s,%s=%s", k8sutil.AppAttr, AppName, controller.DaemonIDLabel, daemonNameToUpdate), }) if err != nil { - logger.Infof("cannot get pod for mgr daemon %s", daemonNameToUpdate) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "cannot get pod for mgr daemon %s", daemonNameToUpdate) return err // force mrg_role update in the next call } @@ -295,7 +296,7 @@ func (c *Cluster) SetMgrRoleLabel(daemonNameToUpdate string, isActive bool) erro labels := pod.GetLabels() currMgrRole, mgrHasLabel := labels[mgrRoleLabelName] if !mgrHasLabel || currMgrRole != newMgrRole { - logger.Infof("updating mgr_role label value of daemon %s to '%s'. New active mgr is %s.", daemonNameToUpdate, newMgrRole, daemonNameToUpdate) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "updating mgr_role label value of daemon %s to '%s'. New active mgr is %s.", daemonNameToUpdate, newMgrRole, daemonNameToUpdate) labels[mgrRoleLabelName] = newMgrRole pod.SetLabels(labels) _, err = c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).Update(c.clusterInfo.Context, &pods.Items[i], metav1.UpdateOptions{}) @@ -346,7 +347,7 @@ func (c *Cluster) reconcileServices() error { if err := c.EnableServiceMonitor(); err != nil { // We don't want to return an error to block the cluster reconcile // since monitoring is an optional service. - logger.Errorf("failed to enable service monitor, prometheus may need to be installed. %v", err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to enable service monitor, prometheus may need to be installed. %v", err) } } @@ -361,7 +362,7 @@ func (c *Cluster) updateServiceSelectors() { selector := metav1.ListOptions{LabelSelector: "app=rook-ceph-mgr"} services, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).List(c.clusterInfo.Context, selector) if err != nil { - logger.Errorf("failed to query mgr services to update labels: %v", err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to query mgr services to update labels: %v", err) return } for i, service := range services.Items { @@ -374,7 +375,7 @@ func (c *Cluster) updateServiceSelectors() { // Check if the service has a DaemonIDLabel (legacy mgr HA implementation) and remove it _, hasDaemonLabel := service.Spec.Selector[controller.DaemonIDLabel] if hasDaemonLabel { - logger.Infof("removing %s selector label on mgr service %q", controller.DaemonIDLabel, service.Name) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "removing %s selector label on mgr service %q", controller.DaemonIDLabel, service.Name) delete(service.Spec.Selector, controller.DaemonIDLabel) updateService = true } @@ -382,16 +383,16 @@ func (c *Cluster) updateServiceSelectors() { // In case the service doesn't have the new mgr_role DaemonIDLabel we add it _, hasMgrRoleLabel := service.Spec.Selector[mgrRoleLabelName] if !hasMgrRoleLabel { - logger.Infof("adding %s selector label on mgr service %q", mgrRoleLabelName, service.Name) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "adding %s selector label on mgr service %q", mgrRoleLabelName, service.Name) service.Spec.Selector[mgrRoleLabelName] = activeMgrStatus updateService = true } if updateService { if _, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Update(c.clusterInfo.Context, &services.Items[i], metav1.UpdateOptions{}); err != nil { - logger.Errorf("failed to update service %q. %v", service.Name, err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to update service %q. %v", service.Name, err) } else { - logger.Infof("service %q successfully updated", service.Name) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "service %q successfully updated", service.Name) } } } @@ -399,22 +400,22 @@ func (c *Cluster) updateServiceSelectors() { func (c *Cluster) configureModules(daemonIDs []string) { // Configure the modules asynchronously so we can complete all the configuration much sooner. - startModuleConfiguration("prometheus", c.configurePrometheusModule) - startModuleConfiguration("dashboard", c.configureDashboardModules) + c.startModuleConfiguration("prometheus", c.configurePrometheusModule) + c.startModuleConfiguration("dashboard", c.configureDashboardModules) // It is a bit confusing but modules that are in the "always_on_modules" list // are "just" enabled, but still they must be configured to work properly - startModuleConfiguration("balancer", c.enableBalancerModule) - startModuleConfiguration("mgr module(s) from the spec", c.configureMgrModules) + c.startModuleConfiguration("balancer", c.enableBalancerModule) + c.startModuleConfiguration("mgr module(s) from the spec", c.configureMgrModules) } -func startModuleConfiguration(description string, configureModules func() error) { +func (c *Cluster) startModuleConfiguration(description string, configureModules func() error) { go func() { err := configureModules() if err != nil { - logger.Errorf("failed modules: %q. %v", description, err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed modules: %q. %v", description, err) } else { - logger.Infof("successful modules: %s", description) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "successful modules: %s", description) } }() } @@ -427,7 +428,7 @@ func (c *Cluster) configurePrometheusModule() error { } } else { if err := cephclient.MgrDisableModule(c.context, c.clusterInfo, PrometheusModuleName); err != nil { - logger.Errorf("failed to disable mgr prometheus module. %v", err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to disable mgr prometheus module. %v", err) } return nil } @@ -446,7 +447,7 @@ func (c *Cluster) configurePrometheusModule() error { if err != nil { return err } - logger.Infof("prometheus config will change, port: %s", port) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "prometheus config will change, port: %s", port) } // scrape interval if c.spec.Monitoring.Interval != nil { @@ -455,18 +456,18 @@ func (c *Cluster) configurePrometheusModule() error { if err != nil { return err } - logger.Infof("prometheus config will change, interval: %v", interval) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "prometheus config will change, interval: %v", interval) } if portHasChanged || intervalHasChanged { - logger.Info("prometheus config has changed. restarting the prometheus module") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "prometheus config has changed. restarting the prometheus module") return c.restartMgrModule(PrometheusModuleName) } return nil } func (c *Cluster) restartMgrModule(name string) error { - logger.Infof("restarting the mgr module: %s", name) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "restarting the mgr module: %s", name) if err := cephclient.MgrDisableModule(c.context, c.clusterInfo, name); err != nil { return errors.Wrapf(err, "failed to disable mgr module %q.", name) } @@ -520,7 +521,7 @@ func (c *Cluster) configureMgrModules() error { // Configure special settings for individual modules that are enabled switch module.Name { case rookModuleName: - startModuleConfiguration("orchestrator modules", c.configureOrchestratorModules) + c.startModuleConfiguration("orchestrator modules", c.configureOrchestratorModules) } } else { @@ -605,10 +606,10 @@ func applyMonitoringLabels(c *Cluster, serviceMonitor *monitoringv1.ServiceMonit serviceMonitor.Spec.Endpoints[0].RelabelConfigs = append( serviceMonitor.Spec.Endpoints[0].RelabelConfigs, relabelConfig) } else { - logger.Info("rook.io/managedBy not specified in monitoring labels") + log.NamespacedInfo(serviceMonitor.Namespace, logger, "rook.io/managedBy not specified in monitoring labels") } } else { - logger.Debug("monitoring labels not specified") + log.NamespacedDebug(serviceMonitor.Namespace, logger, "monitoring labels not specified") } } } @@ -638,11 +639,11 @@ func updateMgrCephxStatus(c *clusterd.Context, clusterInfo *cephclient.ClusterIn } updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, clusterInfo.CephVersion, cluster.Status.Cephx.Mgr) cluster.Status.Cephx.Mgr = updatedStatus - logger.Debugf("updating mgr daemon cephx status to %+v", cluster.Status.Cephx.Mgr) + log.NamespacedDebug(clusterInfo.Namespace, logger, "updating mgr daemon cephx status to %+v", cluster.Status.Cephx.Mgr) if err := reporting.UpdateStatus(c.Client, cluster); err != nil { return errors.Wrap(err, "failed to update cluster cephx status for mgr daemon") } - logger.Info("successfully updated the cephx status for mgr daemon") + log.NamespacedInfo(clusterInfo.Namespace, logger, "successfully updated the cephx status for mgr daemon") return nil }) diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 4f77f53ea..7611657f9 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -29,6 +29,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config/keyring" "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -41,7 +42,7 @@ const ( ) func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) { - logger.Debugf("mgrConfig: %+v", mgrConfig) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "mgrConfig: %+v", mgrConfig) volumes := controller.DaemonVolumes(mgrConfig.DataPathMap, mgrConfig.ResourceName, c.spec.DataDirHostPath) if c.spec.Network.IsMultus() { diff --git a/pkg/operator/ceph/cluster/mon/drain.go b/pkg/operator/ceph/cluster/mon/drain.go index c80461dd1..436b1cc82 100644 --- a/pkg/operator/ceph/cluster/mon/drain.go +++ b/pkg/operator/ceph/cluster/mon/drain.go @@ -19,6 +19,7 @@ package mon import ( "github.com/pkg/errors" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -38,7 +39,7 @@ func (c *Cluster) reconcileMonPDB() error { monCount := c.spec.Mon.Count if monCount <= 2 { - logger.Debug("managePodBudgets is set, but mon-count <= 2. Not creating a disruptionbudget for Mons") + log.NamespacedDebug(c.Namespace, logger, "managePodBudgets is set, but mon-count <= 2. Not creating a disruptionbudget for Mons") return nil } @@ -76,7 +77,7 @@ func (c *Cluster) blockMonDrain(request types.NamespacedName) error { if !c.spec.DisruptionManagement.ManagePodBudgets { return nil } - logger.Info("prevent voluntary mon drain while failing over") + log.NamespacedInfo(c.Namespace, logger, "prevent voluntary mon drain while failing over") // change MaxUnavailable mon PDB to 0 _, err := c.createOrUpdateMonPDB(0) if err != nil { @@ -90,7 +91,7 @@ func (c *Cluster) allowMonDrain(request types.NamespacedName) error { if !c.spec.DisruptionManagement.ManagePodBudgets { return nil } - logger.Info("allow voluntary mon drain after failover") + log.NamespacedInfo(c.Namespace, logger, "allow voluntary mon drain after failover") _, err := c.createOrUpdateMonPDB(c.getMaxUnavailableMonPodCount()) if err != nil { return errors.Wrapf(err, "failed to update MaxUnavailable for mon PDB %q", request.Name) @@ -100,7 +101,7 @@ func (c *Cluster) allowMonDrain(request types.NamespacedName) error { func (c *Cluster) getMaxUnavailableMonPodCount() int32 { if c.spec.Mon.Count >= 5 { - logger.Debug("setting the mon pdb max unavailable count to 2 in case there are 5 or more mons") + log.NamespacedDebug(c.Namespace, logger, "setting the mon pdb max unavailable count to 2 in case there are 5 or more mons") return 2 } diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index defb83734..3e8ae7647 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -31,6 +31,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -71,7 +72,7 @@ func updateMonTimeout(monCluster *Cluster) { if monCRDTimeoutSetting != "" { if monTimeout, err := time.ParseDuration(monCRDTimeoutSetting); err == nil { if monTimeout == timeZero { - logger.Warning("monitor failover is disabled") + log.NamespacedWarning(monCluster.Namespace, logger, "monitor failover is disabled") } MonOutTimeout = monTimeout } @@ -95,7 +96,7 @@ func updateMonInterval(monCluster *Cluster, h *HealthChecker) { checkInterval := monCluster.spec.HealthCheck.DaemonHealth.Monitor.Interval // allow overriding the check interval if checkInterval != nil { - logger.Debugf("ceph mon status in namespace %q check interval %q", monCluster.Namespace, checkInterval.Duration.String()) + log.NamespacedDebug(monCluster.Namespace, logger, "ceph mon status check interval %q", checkInterval.Duration.String()) h.interval = checkInterval.Duration } } @@ -113,6 +114,7 @@ func NewHealthChecker(monCluster *Cluster) *HealthChecker { // Check periodically checks the health of the monitors func (hc *HealthChecker) Check(monitoringRoutines *sync.Map, daemon string) { + namespace := hc.monCluster.Namespace for { // Update Mon Timeout with CR details updateMonTimeout(hc.monCluster) @@ -124,28 +126,28 @@ func (hc *HealthChecker) Check(monitoringRoutines *sync.Map, daemon string) { // we will get an invalid pointer error and the go routine will panic v, ok := monitoringRoutines.Load(daemon) if !ok { - logger.Infof("ceph cluster %q has been deleted. stopping monitoring of mons", hc.monCluster.Namespace) + log.NamespacedInfo(namespace, logger, "ceph cluster has been deleted. stopping monitoring of mons") return } health := v.(*controller.ClusterHealth) select { case <-health.InternalCtx.Done(): - logger.Infof("stopping monitoring of mons in namespace %q (internal context canceled)", hc.monCluster.Namespace) + log.NamespacedInfo(namespace, logger, "stopping monitoring of mons (internal context canceled)") monitoringRoutines.Delete(daemon) return // Since c.ClusterInfo.IsInitialized() below uses a different context, we need to check if the context is done case <-hc.monCluster.ClusterInfo.Context.Done(): - logger.Infof("stopping monitoring of mons in namespace %q (mon cluster info context canceled)", hc.monCluster.Namespace) + log.NamespacedInfo(namespace, logger, "stopping monitoring of mons (mon cluster info context canceled)") monitoringRoutines.Delete(daemon) return case <-time.After(hc.interval): - logger.Debugf("checking health of mons") + log.NamespacedDebug(namespace, logger, "checking health of mons") err := hc.monCluster.checkHealth(health.InternalCtx) if err != nil { - logger.Warningf("failed to check mon health. %v", err) + log.NamespacedWarning(namespace, logger, "failed to check mon health. %v", err) } } } @@ -170,11 +172,11 @@ func (c *Cluster) checkHealth(ctx context.Context) error { return errors.Wrap(err, "failed to check for mons to skip reconcile") } if monsToSkipReconcile.Len() > 0 { - logger.Warningf("skipping mon health check since mons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(monsToSkipReconcile)) + log.NamespacedWarning(c.Namespace, logger, "skipping mon health check since mons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(monsToSkipReconcile)) return nil } - logger.Debugf("Checking health for mons in cluster %q", c.ClusterInfo.Namespace) + log.NamespacedDebug(c.Namespace, logger, "Checking health for mons in cluster %q", c.ClusterInfo.Namespace) // For an external connection we use a special function to get the status if c.spec.External.Enable { @@ -203,7 +205,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error { if err != nil { return errors.Wrap(err, "failed to get mon quorum status") } - logger.Debugf("Mon quorum status: %+v", quorumStatus) + log.NamespacedDebug(c.Namespace, logger, "Mon quorum status: %+v", quorumStatus) // handle external Mons quorumStatus, err = c.reconcileExternalMons(ctx, quorumStatus) @@ -214,7 +216,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error { // Use a local mon count in case the user updates the crd in another goroutine. // We need to complete a health check with a consistent value. desiredMonCount := c.spec.Mon.Count - logger.Debugf("targeting the mon count %d", desiredMonCount) + log.NamespacedDebug(c.Namespace, logger, "targeting the mon count %d", desiredMonCount) // Source of truth of which mons should exist is our *clusterInfo* monsNotFound := map[string]interface{}{} @@ -235,14 +237,14 @@ func (c *Cluster) checkHealth(ctx context.Context) error { // when the mon isn't in the clusterInfo, but is in quorum and there are // enough mons, remove it else remove it on the next run if inQuorum && len(quorumStatus.Quorum) > desiredMonCount { - logger.Warningf("mon %q not in source of truth but in quorum, removing", mon.Name) + log.NamespacedWarning(c.Namespace, logger, "mon %q not in source of truth but in quorum, removing", mon.Name) if err := c.removeMon(mon.Name); err != nil { - logger.Warningf("failed to remove mon %q. %v", mon.Name, err) + log.NamespacedWarning(c.Namespace, logger, "failed to remove mon %q. %v", mon.Name, err) } // only remove one extra mon per health check return nil } - logger.Warningf( + log.NamespacedWarning(c.Namespace, logger, "mon %q not in source of truth and not in quorum, not enough mons to remove now (wanted: %d, current: %d)", mon.Name, desiredMonCount, @@ -254,16 +256,16 @@ func (c *Cluster) checkHealth(ctx context.Context) error { if _, err := c.trackMonInOrOutOfQuorum(mon.Name, true); err != nil { return errors.Wrapf(err, "failed to track out of quorum mon %q", mon.Name) } - logger.Debugf("mon %q found in quorum", mon.Name) + log.NamespacedDebug(c.Namespace, logger, "mon %q found in quorum", mon.Name) // delete the "timeout" for a mon if the pod is in quorum again if _, ok := c.monTimeoutList[mon.Name]; ok { delete(c.monTimeoutList, mon.Name) - logger.Infof("mon %q is back in quorum, removed from mon out timeout list", mon.Name) + log.NamespacedInfo(c.Namespace, logger, "mon %q is back in quorum, removed from mon out timeout list", mon.Name) } continue } - logger.Debugf("mon %q NOT found in quorum. Mon quorum status: %+v", mon.Name, quorumStatus) + log.NamespacedDebug(c.Namespace, logger, "mon %q NOT found in quorum. Mon quorum status: %+v", mon.Name, quorumStatus) allMonsInQuorum = false if _, err := c.trackMonInOrOutOfQuorum(mon.Name, false); err != nil { return errors.Wrapf(err, "failed to track out of quorum mon %q", mon.Name) @@ -271,7 +273,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error { // if the time out is set to 0 this indicate that we don't want to trigger mon failover if MonOutTimeout == timeZero { - logger.Warningf("mon %q NOT found in quorum and health timeout is 0, mon will never fail over", mon.Name) + log.NamespacedWarning(c.Namespace, logger, "mon %q NOT found in quorum and health timeout is 0, mon will never fail over", mon.Name) continue } @@ -290,7 +292,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error { if !rescheduleImmediately { if time.Since(c.monTimeoutList[mon.Name]) <= MonOutTimeout { timeToFailover := int(MonOutTimeout.Seconds() - time.Since(c.monTimeoutList[mon.Name]).Seconds()) - logger.Warningf("mon %q not found in quorum, waiting for timeout (%d seconds left) before failover", mon.Name, timeToFailover) + log.NamespacedWarning(c.Namespace, logger, "mon %q not found in quorum, waiting for timeout (%d seconds left) before failover", mon.Name, timeToFailover) continue } @@ -298,15 +300,15 @@ func (c *Cluster) checkHealth(ctx context.Context) error { monLabelSelector := fmt.Sprintf("%s=%s,%s=%s", k8sutil.AppAttr, AppName, controller.DaemonIDLabel, mon.Name) isScheduled, err := k8sutil.IsPodScheduled(ctx, c.context.Clientset, c.Namespace, monLabelSelector) if err != nil { - logger.Warningf("failed to check if mon %q is assigned to a node, continuing with mon failover. %v", mon.Name, err) + log.NamespacedWarning(c.Namespace, logger, "failed to check if mon %q is assigned to a node, continuing with mon failover. %v", mon.Name, err) } else if !isScheduled && retriesBeforeNodeDrainFailover > 0 { - logger.Warningf("mon %q NOT found in quorum after timeout. Mon pod is not scheduled. Retrying with a timeout of %.2f seconds before failover", mon.Name, MonOutTimeout.Seconds()) + log.NamespacedWarning(c.Namespace, logger, "mon %q NOT found in quorum after timeout. Mon pod is not scheduled. Retrying with a timeout of %.2f seconds before failover", mon.Name, MonOutTimeout.Seconds()) delete(c.monTimeoutList, mon.Name) retriesBeforeNodeDrainFailover = retriesBeforeNodeDrainFailover - 1 return nil } retriesBeforeNodeDrainFailover = 1 - logger.Warningf("mon %q NOT found in quorum and timeout exceeded, mon will be failed over", mon.Name) + log.NamespacedWarning(c.Namespace, logger, "mon %q NOT found in quorum and timeout exceeded, mon will be failed over", mon.Name) } if !c.failMon(len(quorumStatus.MonMap.Mons), desiredMonCount, mon.Name) { @@ -328,7 +330,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error { // after all unhealthy mons have been removed or failed over // handle all mons that haven't been in the Ceph mon map for mon := range monsNotFound { - logger.Warningf("mon %s NOT found in ceph mon map, failover", mon) + log.NamespacedWarning(c.Namespace, logger, "mon %s NOT found in ceph mon map, failover", mon) c.failMon(len(c.ClusterInfo.InternalMonitors), desiredMonCount, mon) // only deal with one "not found in ceph mon map" mon per health check return nil @@ -336,23 +338,23 @@ func (c *Cluster) checkHealth(ctx context.Context) error { // create/start new mons when there are fewer mons than the desired count in the CRD if len(quorumStatus.MonMap.Mons) < desiredMonCount { - logger.Infof("adding mons. currently %d mons are in quorum and the desired count is %d.", len(quorumStatus.MonMap.Mons), desiredMonCount) + log.NamespacedInfo(c.Namespace, logger, "adding mons. currently %d mons are in quorum and the desired count is %d.", len(quorumStatus.MonMap.Mons), desiredMonCount) return c.startMons(desiredMonCount) } // remove extra mons if the desired count has decreased in the CRD and all the mons are currently healthy if allMonsInQuorum && len(quorumStatus.MonMap.Mons) > desiredMonCount { if desiredMonCount < 2 && len(quorumStatus.MonMap.Mons) == 2 { - logger.Warningf("cannot reduce mon quorum size from 2 to 1") + log.NamespacedWarning(c.Namespace, logger, "cannot reduce mon quorum size from 2 to 1") } else { - logger.Infof("removing an extra mon. currently %d are in quorum and only %d are desired", len(quorumStatus.MonMap.Mons), desiredMonCount) + log.NamespacedInfo(c.Namespace, logger, "removing an extra mon. currently %d are in quorum and only %d are desired", len(quorumStatus.MonMap.Mons), desiredMonCount) return c.removeMon(c.determineExtraMonToRemove()) } } if allMonsInQuorum && len(quorumStatus.MonMap.Mons) == desiredMonCount { // remove any pending/not needed mon canary deployment if everything is ok - logger.Debug("mon cluster is healthy, removing any existing canary deployment") + log.NamespacedDebug(c.Namespace, logger, "mon cluster is healthy, removing any existing canary deployment") c.removeCanaryDeployments(monCanaryLabelSelector) // Check whether two healthy mons are on the same node when they should not be. @@ -382,7 +384,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error { // failover any mons present in the mon fail over list for _, mon := range c.ClusterInfo.InternalMonitors { if _, ok := c.monsToFailover[mon.Name]; ok { - logger.Infof("fail over mon %q from the mon fail over list", mon.Name) + log.NamespacedInfo(c.Namespace, logger, "fail over mon %q from the mon fail over list", mon.Name) c.failMon(len(c.ClusterInfo.InternalMonitors), desiredMonCount, mon.Name) delete(c.monsToFailover, mon.Name) return nil @@ -394,25 +396,25 @@ func (c *Cluster) checkHealth(ctx context.Context) error { func (c *Cluster) shouldFailoverMonImmediately(ctx context.Context, monName string) bool { // If the assigned node does not exist, reschedule immediately - logger.Debugf("checking if mon %q is scheduled on a node", monName) + log.NamespacedDebug(c.Namespace, logger, "checking if mon %q is scheduled on a node", monName) assignedNode := "" if mapping, ok := c.mapping.Schedule[monName]; ok { if mapping == nil { // mon schedule information is not available in the rook-ceph-mon-endpoints configmap in case of mons on PVC - logger.Debugf("mon schedule information is not available for mon %q", monName) + log.NamespacedDebug(c.Namespace, logger, "mon schedule information is not available for mon %q", monName) return false } assignedNode = mapping.Hostname } if assignedNode == "" { - logger.Debugf("mon %q is not assigned to a node", monName) + log.NamespacedDebug(c.Namespace, logger, "mon %q is not assigned to a node", monName) return false } - logger.Debugf("mon %q is scheduled on node %q", monName, assignedNode) + log.NamespacedDebug(c.Namespace, logger, "mon %q is scheduled on node %q", monName, assignedNode) nodeExists, err := k8sutil.NodeWithHostnameExists(ctx, c.context.Clientset, assignedNode) if err == nil && !nodeExists { - logger.Warningf("mon %q NOT found in quorum and its assigned node %q is not found, failing over immediately", monName, assignedNode) + log.NamespacedWarning(c.Namespace, logger, "mon %q NOT found in quorum and its assigned node %q is not found, failing over immediately", monName, assignedNode) return true } @@ -423,11 +425,11 @@ func (c *Cluster) shouldFailoverMonImmediately(ctx context.Context, monName stri func (c *Cluster) reconcileExternalMons(ctx context.Context, quorumStatus cephclient.MonStatusResponse) (cephclient.MonStatusResponse, error) { if len(c.spec.Mon.ExternalMonIDs) != 0 { if c.spec.External.Enable { - logger.Warning("ignore external mon IDs in cluster spec: cluster is running in external mode") + log.NamespacedWarning(c.Namespace, logger, "ignore external mon IDs in cluster spec: cluster is running in external mode") return quorumStatus, nil } if c.spec.IsStretchCluster() { - logger.Warning("ignore external mon IDs in cluster spec: cluster is running in stretch mode") + log.NamespacedWarning(c.Namespace, logger, "ignore external mon IDs in cluster spec: cluster is running in stretch mode") return quorumStatus, nil } } @@ -444,19 +446,19 @@ func (c *Cluster) reconcileExternalMons(ctx context.Context, quorumStatus cephcl } // existing external mon was removed from Cluster CRD spec: // remove it from CLusterInfo - logger.Debugf("existing external mon %q was removed from spec: removing it", extID) + log.NamespacedDebug(c.Namespace, logger, "existing external mon %q was removed from spec: removing it", extID) delete(c.ClusterInfo.ExternalMons, extID) extMonsChanged = true } // handle external monitors if configured in cluster CRD: - logger.Debugf("external mon IDs: %v", extMonIDs) + log.NamespacedDebug(c.Namespace, logger, "external mon IDs: %v", extMonIDs) for _, extID := range extMonIDs { monStatus, inQuorum := getMonByID(extID, quorumStatus) if inQuorum { - logger.Debugf("external mon %q in quorum", extID) + log.NamespacedDebug(c.Namespace, logger, "external mon %q in quorum", extID) } else { - logger.Debugf("external mon %q not in quorum %+v, %+v", extID, quorumStatus.Quorum, quorumStatus.MonMap.Mons) + log.NamespacedDebug(c.Namespace, logger, "external mon %q not in quorum %+v, %+v", extID, quorumStatus.Quorum, quorumStatus.MonMap.Mons) } _, inInfo := c.ClusterInfo.ExternalMons[extID] if inQuorum && !inInfo { @@ -464,12 +466,12 @@ func (c *Cluster) reconcileExternalMons(ctx context.Context, quorumStatus cephcl monInfo := c.monStatusToInfo(monStatus) c.ClusterInfo.ExternalMons[extID] = monInfo extMonsChanged = true - logger.Infof("new external mon %q found: %s, adding it", extID, monInfo.Endpoint) + log.NamespacedInfo(c.Namespace, logger, "new external mon %q found: %s, adding it", extID, monInfo.Endpoint) } else if !inQuorum && inInfo { // remove external mon from cluster info if it is out of quorum: delete(c.ClusterInfo.ExternalMons, extID) extMonsChanged = true - logger.Infof("new external mon %q not in quorum: removing it", extID) + log.NamespacedInfo(c.Namespace, logger, "new external mon %q not in quorum: removing it", extID) } } if extMonsChanged { @@ -516,7 +518,7 @@ func (c *Cluster) trackMonInOrOutOfQuorum(monName string, inQuorum bool) (bool, // All mons are in quorum, so make sure no mons are marked out of quorum for monName, mon := range c.ClusterInfo.InternalMonitors { if mon.OutOfQuorum { - logger.Infof("resetting mon %q to be back in quorum", monName) + log.NamespacedInfo(c.Namespace, logger, "resetting mon %q to be back in quorum", monName) mon.OutOfQuorum = false updateNeeded = true } @@ -524,18 +526,18 @@ func (c *Cluster) trackMonInOrOutOfQuorum(monName string, inQuorum bool) (bool, } else { mon, ok := c.ClusterInfo.InternalMonitors[monName] if !ok { - logger.Infof("mon %q not found to keep track of being out of quorum", monName) + log.NamespacedInfo(c.Namespace, logger, "mon %q not found to keep track of being out of quorum", monName) return false, nil } // Mark the mon in quorum if inQuorum && mon.OutOfQuorum { - logger.Infof("marking mon %q back in quorum", monName) + log.NamespacedInfo(c.Namespace, logger, "marking mon %q back in quorum", monName) mon.OutOfQuorum = false updateNeeded = true } // Mark the mon out of quorum if !inQuorum && !mon.OutOfQuorum { - logger.Infof("marking mon %q out of quorum", monName) + log.NamespacedInfo(c.Namespace, logger, "marking mon %q out of quorum", monName) mon.OutOfQuorum = true updateNeeded = true } @@ -570,7 +572,7 @@ func (c *Cluster) determineExtraMonToRemove() string { if stretchMonToRemove != "" { return stretchMonToRemove } - logger.Infof("did not find an extra mon to remove from the stretch cluster") + log.NamespacedInfo(c.Namespace, logger, "did not find an extra mon to remove from the stretch cluster") return "" } @@ -578,13 +580,13 @@ func (c *Cluster) determineExtraMonToRemove() string { arbitraryMon := "" for _, mon := range mons { if mon.NodeName == "" { - logger.Debugf("mon %q is not scheduled to a specific host", mon.DaemonName) + log.NamespacedDebug(c.Namespace, logger, "mon %q is not scheduled to a specific host", mon.DaemonName) arbitraryMon = mon.DaemonName continue } // Check if there are multiple mons on the node if existingMon, ok := nodesWithMons[mon.NodeName]; ok { - logger.Infof("found mons %q and %q on node %s, removing mon %q", existingMon, mon.DaemonName, mon.NodeName, mon.DaemonName) + log.NamespacedInfo(c.Namespace, logger, "found mons %q and %q on node %s, removing mon %q", existingMon, mon.DaemonName, mon.NodeName, mon.DaemonName) return mon.DaemonName } nodesWithMons[mon.NodeName] = mon.DaemonName @@ -593,7 +595,7 @@ func (c *Cluster) determineExtraMonToRemove() string { arbitraryMon = mon.DaemonName } - logger.Infof("removing arbitrary extra mon %q", arbitraryMon) + log.NamespacedInfo(c.Namespace, logger, "removing arbitrary extra mon %q", arbitraryMon) return arbitraryMon } @@ -603,7 +605,7 @@ func (c *Cluster) findExtraMonToRemoveFromStretchCluster(mons []*monConfig) stri monInZones := map[string]string{} for _, m := range mons { if m.Zone == "" { - logger.Warningf("zone not found on mon %q", m.DaemonName) + log.NamespacedWarning(c.Namespace, logger, "zone not found on mon %q", m.DaemonName) continue } zoneCount[m.Zone]++ @@ -620,12 +622,12 @@ func (c *Cluster) findExtraMonToRemoveFromStretchCluster(mons []*monConfig) stri } if zone.Arbiter { if count > 1 { - logger.Infof("removing extra mon %q in arbiter zone %q", monInZones[zone.Name], zone.Name) + log.NamespacedInfo(c.Namespace, logger, "removing extra mon %q in arbiter zone %q", monInZones[zone.Name], zone.Name) return monInZones[zone.Name] } } else { if count > 2 { - logger.Infof("removing extra mon %q in zone %q", monInZones[zone.Name], zone.Name) + log.NamespacedInfo(c.Namespace, logger, "removing extra mon %q in zone %q", monInZones[zone.Name], zone.Name) return monInZones[zone.Name] } } @@ -639,69 +641,69 @@ func (c *Cluster) findExtraMonToRemoveFromStretchCluster(mons []*monConfig) stri func (c *Cluster) failMon(monCount, desiredMonCount int, name string) bool { // make sure the failed mon is marked out of quorum if _, err := c.trackMonInOrOutOfQuorum(name, false); err != nil { - logger.Errorf("failed to track failed mon %q. %v", name, err) + log.NamespacedError(c.Namespace, logger, "failed to track failed mon %q. %v", name, err) } if monCount > desiredMonCount { // no need to create a new mon since we have an extra if err := c.removeMon(name); err != nil { - logger.Errorf("failed to remove mon %q. %v", name, err) + log.NamespacedError(c.Namespace, logger, "failed to remove mon %q. %v", name, err) } return true } // prevent any voluntary mon drain while failing over if err := c.blockMonDrain(types.NamespacedName{Name: monPDBName, Namespace: c.Namespace}); err != nil { - logger.Errorf("failed to block mon drain. %v", err) + log.NamespacedError(c.Namespace, logger, "failed to block mon drain. %v", err) } // bring up a new mon to replace the unhealthy mon if err := c.failoverMon(name); err != nil { - logger.Errorf("failed to failover mon %q. %v", name, err) + log.NamespacedError(c.Namespace, logger, "failed to failover mon %q. %v", name, err) } // allow any voluntary mon drain after failover if err := c.allowMonDrain(types.NamespacedName{Name: monPDBName, Namespace: c.Namespace}); err != nil { - logger.Errorf("failed to allow mon drain. %v", err) + log.NamespacedError(c.Namespace, logger, "failed to allow mon drain. %v", err) } return true } func (c *Cluster) removeOrphanMonResources() { if c.spec.Mon.VolumeClaimTemplate == nil { - logger.Debug("skipping check for orphaned mon pvcs since using the host path") + log.NamespacedDebug(c.Namespace, logger, "skipping check for orphaned mon pvcs since using the host path") return } - logger.Info("checking for orphaned mon resources") + log.NamespacedInfo(c.Namespace, logger, "checking for orphaned mon resources") opts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, AppName)} pvcs, err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).List(c.ClusterInfo.Context, opts) if err != nil { - logger.Infof("failed to check for orphaned mon pvcs. %v", err) + log.NamespacedInfo(c.Namespace, logger, "failed to check for orphaned mon pvcs. %v", err) return } for _, pvc := range pvcs.Items { - logger.Debugf("checking if pvc %q is orphaned", pvc.Name) + log.NamespacedDebug(c.Namespace, logger, "checking if pvc %q is orphaned", pvc.Name) _, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, pvc.Name, metav1.GetOptions{}) if err == nil { - logger.Debugf("skipping pvc removal since the mon daemon %q still requires it", pvc.Name) + log.NamespacedDebug(c.Namespace, logger, "skipping pvc removal since the mon daemon %q still requires it", pvc.Name) continue } if !kerrors.IsNotFound(err) { - logger.Infof("skipping pvc removal since the mon daemon %q might still require it. %v", pvc.Name, err) + log.NamespacedInfo(c.Namespace, logger, "skipping pvc removal since the mon daemon %q might still require it. %v", pvc.Name, err) continue } - logger.Infof("removing pvc %q since it is no longer needed for the mon daemon", pvc.Name) + log.NamespacedInfo(c.Namespace, logger, "removing pvc %q since it is no longer needed for the mon daemon", pvc.Name) var gracePeriod int64 // delete immediately propagation := metav1.DeletePropagationForeground options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation} err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(c.ClusterInfo.Context, pvc.Name, *options) if err != nil { - logger.Warningf("failed to delete orphaned monitor pvc %q. %v", pvc.Name, err) + log.NamespacedWarning(c.Namespace, logger, "failed to delete orphaned monitor pvc %q. %v", pvc.Name, err) } } } @@ -722,7 +724,7 @@ func (c *Cluster) updateMonDeploymentReplica(name string, enabled bool) error { d.Spec.Replicas = &desiredReplicas // update the deployment - logger.Infof("scaling the mon %q deployment to replica %d", name, desiredReplicas) + log.NamespacedInfo(c.Namespace, logger, "scaling the mon %q deployment to replica %d", name, desiredReplicas) _, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Update(c.ClusterInfo.Context, d, metav1.UpdateOptions{}) if err != nil { return errors.Wrapf(err, "failed to update mon %q replicas from %d to %d", name, originalReplicas, desiredReplicas) @@ -731,7 +733,7 @@ func (c *Cluster) updateMonDeploymentReplica(name string, enabled bool) error { } func (c *Cluster) failoverMon(name string) error { - logger.Infof("Failing over monitor %q", name) + log.NamespacedInfo(c.Namespace, logger, "Failing over monitor %q", name) // remove the failed mon from a local list of the existing mons for finding a stretch zone existingMons := c.clusterInfoToMonConfigWithExclude(name) @@ -743,13 +745,13 @@ func (c *Cluster) failoverMon(name string) error { // Start a new monitor m := c.newMonConfig(c.maxMonID+1, zone) - logger.Infof("starting new mon: %+v", m) + log.NamespacedInfo(c.Namespace, logger, "starting new mon: %+v", m) // Scale down the failed mon to allow a new one to start if c.stopMonDuringFailover(name) { if err := c.updateMonDeploymentReplica(name, false); err != nil { // attempt to continue with the failover even if the bad mon could not be stopped - logger.Warningf("failed to stop mon %q for failover. %v", name, err) + log.NamespacedWarning(c.Namespace, logger, "failed to stop mon %q for failover. %v", name, err) } } @@ -761,21 +763,21 @@ func (c *Cluster) failoverMon(name string) error { // do nothing if the new mon was started successfully, the deployment will anyway be deleted return } - logger.Warningf("failover of mon %q unsuccessful, cleaning up replacement mon %q", name, m.DaemonName) + log.NamespacedWarning(c.Namespace, logger, "failover of mon %q unsuccessful, cleaning up replacement mon %q", name, m.DaemonName) if err := c.updateMonDeploymentReplica(name, true); err != nil { // attempt to continue even if the bad mon could not be restarted - logger.Warningf("failed to restart failed mon %q after new mon wouldn't start. %v", name, err) + log.NamespacedWarning(c.Namespace, logger, "failed to restart failed mon %q after new mon wouldn't start. %v", name, err) } if err := c.removeMonWithOptionalQuorum(m.DaemonName, newMonMightBeInQuorum); err != nil { - logger.Infof("failed to remove mon %q from quorum. %v", m.DaemonName, err) + log.NamespacedInfo(c.Namespace, logger, "failed to remove mon %q from quorum. %v", m.DaemonName, err) } // Make sure the maxMonID is reverted to its previous value // The maxMonId is committed to a configmap immediately after the mon deployment // is started, even though c.maxMonID is not incremented until the mon failover is successful - logger.Infof("reverting maxMonId to %d", c.maxMonID) + log.NamespacedInfo(c.Namespace, logger, "reverting maxMonId to %d", c.maxMonID) if err := c.commitMaxMonIDRequireIncrementing(c.maxMonID, false); err != nil { - logger.Errorf("failed to revert maxMonId after starting mon %q", m.DaemonName) + log.NamespacedError(c.Namespace, logger, "failed to revert maxMonId after starting mon %q", m.DaemonName) } }() @@ -803,7 +805,7 @@ func (c *Cluster) failoverMon(name string) error { if err != nil { return errors.Wrapf(err, "failed to export service %q", monService.Name) } - logger.Infof("mon %q exported IP is %s", m.DaemonName, exportedIP) + log.NamespacedInfo(c.Namespace, logger, "mon %q exported IP is %s", m.DaemonName, exportedIP) m.PublicIP = exportedIP } else { m.PublicIP = monService.Spec.ClusterIP @@ -846,7 +848,7 @@ func (c *Cluster) stopMonDuringFailover(name string) bool { if mon, ok := c.monsToFailover[name]; ok && !mon.UseHostNetwork { return true } - logger.Infof("skipping stopping mon %q during failover, since with host networking a new mon cannot be started on the same node", name) + log.NamespacedInfo(c.Namespace, logger, "skipping stopping mon %q during failover, since with host networking a new mon cannot be started on the same node", name) return false } @@ -858,15 +860,15 @@ func (c *Cluster) removeMon(daemonName string) error { // make a best effort to remove the mon and all its resources func (c *Cluster) removeMonWithOptionalQuorum(daemonName string, shouldRemoveFromQuorum bool) error { if daemonName == "" { - logger.Info("did not identify a mon to remove") + log.NamespacedInfo(c.Namespace, logger, "did not identify a mon to remove") return nil } - logger.Infof("ensuring removal of unhealthy monitor %s", daemonName) + log.NamespacedInfo(c.Namespace, logger, "ensuring removal of unhealthy monitor %s", daemonName) // Remove the bad monitor from quorum if shouldRemoveFromQuorum { if err := c.removeMonitorFromQuorum(daemonName); err != nil { - logger.Errorf("failed to remove mon %q from quorum. %v", daemonName, err) + log.NamespacedError(c.Namespace, logger, "failed to remove mon %q from quorum. %v", daemonName, err) } } delete(c.ClusterInfo.InternalMonitors, daemonName) @@ -899,39 +901,39 @@ func (c *Cluster) removeMonResources(daemonName string) { options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation} if err := c.context.Clientset.CoreV1().Services(c.Namespace).Delete(c.ClusterInfo.Context, resourceName, *options); err != nil { if kerrors.IsNotFound(err) { - logger.Infof("dead mon service %s was already gone", resourceName) + log.NamespacedInfo(c.Namespace, logger, "dead mon service %s was already gone", resourceName) } else { - logger.Errorf("failed to remove dead mon service %q. %v", resourceName, err) + log.NamespacedError(c.Namespace, logger, "failed to remove dead mon service %q. %v", resourceName, err) } } // Remove the PVC backing the mon if it existed if err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(c.ClusterInfo.Context, resourceName, metav1.DeleteOptions{}); err != nil { if kerrors.IsNotFound(err) { - logger.Infof("mon pvc did not exist %q", resourceName) + log.NamespacedInfo(c.Namespace, logger, "mon pvc did not exist %q", resourceName) } else { - logger.Errorf("failed to remove dead mon pvc %q. %v", resourceName, err) + log.NamespacedError(c.Namespace, logger, "failed to remove dead mon pvc %q. %v", resourceName, err) } } // Remove the mon pod if it is still there if err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Delete(c.ClusterInfo.Context, resourceName, *options); err != nil { if kerrors.IsNotFound(err) { - logger.Infof("dead mon %s was already gone", resourceName) + log.NamespacedInfo(c.Namespace, logger, "dead mon %s was already gone", resourceName) } else { - logger.Errorf("failed to remove dead mon deployment %q. %v", resourceName, err) + log.NamespacedError(c.Namespace, logger, "failed to remove dead mon deployment %q. %v", resourceName, err) } } } func (c *Cluster) removeMonitorFromQuorum(name string) error { - logger.Debugf("removing monitor %s", name) + log.NamespacedDebug(c.Namespace, logger, "removing monitor %s", name) args := []string{"mon", "remove", name} if _, err := cephclient.NewCephCommand(c.context, c.ClusterInfo, args).Run(); err != nil { return errors.Wrapf(err, "mon %s remove failed", name) } - logger.Infof("removed monitor %s", name) + log.NamespacedInfo(c.Namespace, logger, "removed monitor %s", name) return nil } @@ -969,11 +971,11 @@ func (c *Cluster) addOrRemoveExternalMonitor(status cephclient.MonStatusResponse oldClusterInfoMonitors[mon.Name] = mon delete(c.ClusterInfo.InternalMonitors, monName) } - logger.Debugf("ClusterInfo is now Empty, refilling it from status.MonMap.Mons") + log.NamespacedDebug(c.Namespace, logger, "ClusterInfo is now Empty, refilling it from status.MonMap.Mons") monCount := len(status.MonMap.Mons) if monCount == 1 { - logger.Warning("external cluster mon count is 1, consider adding new monitors.") + log.NamespacedWarning(c.Namespace, logger, "external cluster mon count is 1, consider adding new monitors.") } // Iterate over the mons first and compare it with ClusterInfo @@ -984,25 +986,25 @@ func (c *Cluster) addOrRemoveExternalMonitor(status cephclient.MonStatusResponse // If the mon is part of the quorum if inQuorum { info := c.monStatusToInfo(mon) - logger.Infof("new external mon %q found: %s, adding it", mon.Name, info.Endpoint) + log.NamespacedInfo(c.Namespace, logger, "new external mon %q found: %s, adding it", mon.Name, info.Endpoint) c.ClusterInfo.InternalMonitors[mon.Name] = info } else { - logger.Debugf("mon %q is not in quorum and not in ClusterInfo", mon.Name) + log.NamespacedDebug(c.Namespace, logger, "mon %q is not in quorum and not in ClusterInfo", mon.Name) } changed = true } else { // mon is in ClusterInfo - logger.Debugf("mon %q is in ClusterInfo, let's test if it's in quorum", mon.Name) + log.NamespacedDebug(c.Namespace, logger, "mon %q is in ClusterInfo, let's test if it's in quorum", mon.Name) if !inQuorum { // this mon was in clusterInfo but is not part of the quorum anymore // thus don't add it again to ClusterInfo - logger.Infof("monitor %q is not part of the external cluster monitor quorum, removing it", mon.Name) + log.NamespacedInfo(c.Namespace, logger, "monitor %q is not part of the external cluster monitor quorum, removing it", mon.Name) changed = true } else { // this mon was in clusterInfo and is still in the quorum // add it again c.ClusterInfo.InternalMonitors[mon.Name] = oldClusterInfoMonitors[mon.Name] - logger.Debugf("everything is fine mon %q in the clusterInfo and its quorum status is %v", mon.Name, inQuorum) + log.NamespacedDebug(c.Namespace, logger, "everything is fine mon %q in the clusterInfo and its quorum status is %v", mon.Name, inQuorum) } } } @@ -1019,7 +1021,7 @@ func (c *Cluster) addOrRemoveExternalMonitor(status cephclient.MonStatusResponse } } - logger.Debugf("ClusterInfo.Monitors is %+v", c.ClusterInfo.InternalMonitors) + log.NamespacedDebug(c.Namespace, logger, "ClusterInfo.Monitors is %+v", c.ClusterInfo.InternalMonitors) return changed, nil } @@ -1041,11 +1043,11 @@ func (c *Cluster) monStatusToInfo(mon cephclient.MonMapEntry) *cephclient.MonInf func (c *Cluster) evictMonIfMultipleOnSameNode() error { if c.spec.Mon.AllowMultiplePerNode { - logger.Debug("skipping check for multiple mons on same node since multiple mons are allowed") + log.NamespacedDebug(c.Namespace, logger, "skipping check for multiple mons on same node since multiple mons are allowed") return nil } - logger.Info("checking if multiple mons are on the same node") + log.NamespacedInfo(c.Namespace, logger, "checking if multiple mons are on the same node") // Get all the mon pods label := fmt.Sprintf("app=%s", AppName) @@ -1056,13 +1058,13 @@ func (c *Cluster) evictMonIfMultipleOnSameNode() error { nodesToMons := map[string]string{} for _, pod := range pods.Items { - logger.Debugf("analyzing mon pod %q on node %q", pod.Name, pod.Spec.NodeName) + log.NamespacedDebug(c.Namespace, logger, "analyzing mon pod %q on node %q", pod.Name, pod.Spec.NodeName) if _, ok := pod.Labels["mon_canary"]; ok { - logger.Debugf("skipping mon canary pod %q", pod.Name) + log.NamespacedDebug(c.Namespace, logger, "skipping mon canary pod %q", pod.Name) continue } if pod.Spec.NodeName == "" { - logger.Warningf("mon %q is not assigned to a node", pod.Name) + log.NamespacedWarning(c.Namespace, logger, "mon %q is not assigned to a node", pod.Name) continue } monName := pod.Labels["mon"] @@ -1073,7 +1075,7 @@ func (c *Cluster) evictMonIfMultipleOnSameNode() error { continue } - logger.Warningf("Both mons %q and %q are on node %q. Evicting mon %q", monName, previousMonName, pod.Spec.NodeName, monName) + log.NamespacedWarning(c.Namespace, logger, "Both mons %q and %q are on node %q. Evicting mon %q", monName, previousMonName, pod.Spec.NodeName, monName) return c.failoverMon(monName) } diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index 061457952..00dffcc74 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -44,6 +44,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/reporting" cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" @@ -217,21 +218,21 @@ func (c *Cluster) Start(clusterInfo *cephclient.ClusterInfo, rookImage string, c return nil, errors.Wrap(err, "failed to check pod memory") } - logger.Infof("start running mons") + log.NamespacedInfo(c.Namespace, logger, "start running mons") - logger.Debugf("establishing ceph cluster info") + log.NamespacedDebug(c.Namespace, logger, "establishing ceph cluster info") if err := c.initClusterInfo(cephVersion, c.ClusterInfo.NamespacedName().Name); err != nil { return nil, errors.Wrap(err, "failed to initialize ceph cluster info") } - logger.Infof("targeting the mon count %d", c.spec.Mon.Count) + log.NamespacedInfo(c.Namespace, logger, "targeting the mon count %d", c.spec.Mon.Count) monsToSkipReconcile, err := controller.GetDaemonsToSkipReconcile(c.ClusterInfo.Context, c.context, c.Namespace, config.MonType, AppName) if err != nil { return nil, errors.Wrap(err, "failed to check for mons to skip reconcile") } if monsToSkipReconcile.Len() > 0 { - logger.Warningf("skipping mon reconcile since mons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(monsToSkipReconcile)) + log.NamespacedWarning(c.Namespace, logger, "skipping mon reconcile since mons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(monsToSkipReconcile)) return c.ClusterInfo, nil } @@ -263,7 +264,7 @@ func (c *Cluster) startMons(targetCount int) error { // fixed by updating the mon deployments. Instead of returning error here, log a // warning, and retry setting this later. setConfigsNeedsRetry = true - logger.Warningf("failed to set Rook and/or user-defined Ceph config options before starting mons; will retry after starting mons. %v", err) + log.NamespacedWarning(c.Namespace, logger, "failed to set Rook and/or user-defined Ceph config options before starting mons; will retry after starting mons. %v", err) } } @@ -321,7 +322,7 @@ func (c *Cluster) startMons(targetCount int) error { } } - logger.Debugf("mon endpoints used are: %s", flattenMonEndpoints(c.ClusterInfo.AllMonitors())) + log.NamespacedDebug(c.Namespace, logger, "mon endpoints used are: %s", flattenMonEndpoints(c.ClusterInfo.AllMonitors())) // reconcile mon PDB if err := c.reconcileMonPDB(); err != nil { @@ -376,14 +377,14 @@ func (c *Cluster) ConfigureArbiter() error { monDump, err := cephclient.GetMonDump(c.context, c.ClusterInfo) if err != nil { - logger.Warningf("attempting to enable arbiter after failed to detect if already enabled. %v", err) + log.NamespacedWarning(c.Namespace, logger, "attempting to enable arbiter after failed to detect if already enabled. %v", err) } else if monDump.StretchMode { if monDump.TiebreakerMon == c.arbiterMon { - logger.Infof("stretch mode is already enabled with tiebreaker %q", c.arbiterMon) + log.NamespacedInfo(c.Namespace, logger, "stretch mode is already enabled with tiebreaker %q", c.arbiterMon) return nil } // Set the new mon tiebreaker - logger.Infof("updating tiebreaker mon from %q to %q", monDump.TiebreakerMon, c.arbiterMon) + log.NamespacedInfo(c.Namespace, logger, "updating tiebreaker mon from %q to %q", monDump.TiebreakerMon, c.arbiterMon) if err := cephclient.SetNewTiebreaker(c.context, c.ClusterInfo, c.arbiterMon); err != nil { return errors.Wrap(err, "failed to set new mon tiebreaker") } @@ -394,7 +395,7 @@ func (c *Cluster) ConfigureArbiter() error { // The timeout is relatively short since the operator will requeue the reconcile // and try again at a higher level if not yet found failureDomain := c.getFailureDomainName() - logger.Infof("enabling stretch mode... waiting for two failure domains of type %q to be found in the CRUSH map after OSD initialization", failureDomain) + log.NamespacedInfo(c.Namespace, logger, "enabling stretch mode... waiting for two failure domains of type %q to be found in the CRUSH map after OSD initialization", failureDomain) pollInterval := 5 * time.Second totalWaitTime := 2 * time.Minute err = wait.PollUntilContextTimeout(c.ClusterInfo.Context, pollInterval, totalWaitTime, true, func(ctx context.Context) (bool, error) { @@ -408,7 +409,7 @@ func (c *Cluster) ConfigureArbiter() error { // Wait for the .mgr pool to be created, which we expect is defined as a CephBlockPool // We may be able to remove this code waiting for the pool once this is in the ceph release: // https://github.com/ceph/ceph/pull/61371 - logger.Info("enabling stretch mode... waiting for the builtin .mgr pool to be created") + log.NamespacedInfo(c.Namespace, logger, "enabling stretch mode... waiting for the builtin .mgr pool to be created") err = wait.PollUntilContextTimeout(c.ClusterInfo.Context, pollInterval, totalWaitTime, true, func(ctx context.Context) (bool, error) { return c.builtinMgrPoolExists(), nil }) @@ -443,7 +444,7 @@ func (c *Cluster) readyToConfigureArbiter(checkOSDPods bool) (bool, error) { return false, errors.Wrap(err, "failed to check whether all osds are running before enabling the arbiter") } if !allRunning { - logger.Infof("waiting for all OSD pods to be in running state") + log.NamespacedInfo(c.Namespace, logger, "waiting for all OSD pods to be in running state") return false, nil } } @@ -460,10 +461,10 @@ func (c *Cluster) readyToConfigureArbiter(checkOSDPods bool) (bool, error) { if bucket.TypeName == failureDomain { // skip zones specific to device classes if strings.Index(bucket.Name, "~") > 0 { - logger.Debugf("skipping device class bucket %q", bucket.Name) + log.NamespacedDebug(c.Namespace, logger, "skipping device class bucket %q", bucket.Name) continue } - logger.Infof("found %s %q in CRUSH map with weight %d", failureDomain, bucket.Name, bucket.Weight) + log.NamespacedInfo(c.Namespace, logger, "found %s %q in CRUSH map with weight %d", failureDomain, bucket.Name, bucket.Weight) zoneCount++ // check that the weights of the failure domains are all the same @@ -471,7 +472,7 @@ func (c *Cluster) readyToConfigureArbiter(checkOSDPods bool) (bool, error) { // found the first matching bucket zoneWeight = bucket.Weight } else if zoneWeight != bucket.Weight { - logger.Infof("found failure domains that have different weights") + log.NamespacedInfo(c.Namespace, logger, "found failure domains that have different weights") return false, nil } } @@ -483,7 +484,7 @@ func (c *Cluster) readyToConfigureArbiter(checkOSDPods bool) (bool, error) { if zoneCount > 2 { return false, fmt.Errorf("cannot configure stretch cluster with more than 2 failure domains, and found %d of type %q", zoneCount, failureDomain) } - logger.Infof("found two expected failure domains %q for the stretch cluster", failureDomain) + log.NamespacedInfo(c.Namespace, logger, "found two expected failure domains %q for the stretch cluster", failureDomain) return true, nil } @@ -494,9 +495,9 @@ func (c *Cluster) readyToConfigureArbiter(checkOSDPods bool) (bool, error) { // This is needed when the operator is restarted and all mons may not be up or in quorum. func (c *Cluster) ensureMonsRunning(mons []*monConfig, i, targetCount int, requireAllInQuorum bool) error { if requireAllInQuorum { - logger.Infof("creating mon %s", mons[i].DaemonName) + log.NamespacedInfo(c.Namespace, logger, "creating mon %s", mons[i].DaemonName) } else { - logger.Info("checking for basic quorum with existing mons") + log.NamespacedInfo(c.Namespace, logger, "checking for basic quorum with existing mons") } // Calculate how many mons we expected to exist after this method is completed. @@ -609,7 +610,7 @@ func (c *Cluster) clusterInfoToMonConfigWithExclude(excludedMon string) []*monCo isHostNetwork = true } } - logger.Debugf("Host network for mon %q is %t", monitor.Name, isHostNetwork) + log.NamespacedDebug(c.Namespace, logger, "Host network for mon %q is %t", monitor.Name, isHostNetwork) mons = append(mons, &monConfig{ ResourceName: resourceName(monitor.Name), @@ -739,10 +740,10 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) { _, err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Create(c.ClusterInfo.Context, pvc, metav1.CreateOptions{}) if err == nil { - logger.Infof("created canary monitor %s pvc %s", d.Name, pvc.Name) + log.NamespacedInfo(c.Namespace, logger, "created canary monitor %s pvc %s", d.Name, pvc.Name) } else { if kerrors.IsAlreadyExists(err) { - logger.Debugf("creating mon %s pvc %s: already exists.", d.Name, pvc.Name) + log.NamespacedDebug(c.Namespace, logger, "creating mon %s pvc %s: already exists.", d.Name, pvc.Name) } else { return nil, errors.Wrapf(err, "failed to create mon %s pvc %s", d.Name, pvc.Name) } @@ -763,13 +764,13 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) { _, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(c.ClusterInfo.Context, d, metav1.CreateOptions{}) if err == nil { createdDeployment = true - logger.Infof("created canary deployment %s", d.Name) + log.NamespacedInfo(c.Namespace, logger, "created canary deployment %s", d.Name) break } else if kerrors.IsAlreadyExists(err) { if err := k8sutil.DeleteDeployment(c.ClusterInfo.Context, c.context.Clientset, c.Namespace, d.Name); err != nil { return nil, errors.Wrapf(err, "failed to delete canary deployment %s", d.Name) } - logger.Infof("deleted existing canary deployment %s", d.Name) + log.NamespacedInfo(c.Namespace, logger, "deleted existing canary deployment %s", d.Name) time.Sleep(time.Second * canaryRetryDelaySeconds) } else { return nil, errors.Wrapf(err, "failed to create canary monitor deployment %s", d.Name) @@ -824,13 +825,13 @@ func realWaitForMonitorScheduling(c *Cluster, d *apps.Deployment) (SchedulingRes } if len(pods.Items) == 0 { - logger.Infof("waiting for canary pod creation %s", d.Name) + log.NamespacedInfo(c.Namespace, logger, "waiting for canary pod creation %s", d.Name) continue } pod := pods.Items[0] if pod.Spec.NodeName == "" { - logger.Debugf("monitor %s canary pod %s not yet scheduled", d.Name, pod.Name) + log.NamespacedDebug(c.Namespace, logger, "monitor %s canary pod %s not yet scheduled", d.Name, pod.Name) continue } @@ -840,7 +841,7 @@ func realWaitForMonitorScheduling(c *Cluster, d *apps.Deployment) (SchedulingRes } result.Node = node - logger.Infof("canary monitor deployment %s scheduled to %s", d.Name, node.Name) + log.NamespacedInfo(c.Namespace, logger, "canary monitor deployment %s scheduled to %s", d.Name, node.Name) return result, nil } @@ -853,7 +854,7 @@ func (c *Cluster) initMonIPs(mons []*monConfig) error { return c.ClusterInfo.Context.Err() } if m.UseHostNetwork { - logger.Infof("setting mon %q endpoints for hostnetwork mode", m.DaemonName) + log.NamespacedInfo(c.Namespace, logger, "setting mon %q endpoints for hostnetwork mode", m.DaemonName) node, ok := c.mapping.Schedule[m.DaemonName] if !ok || node == nil { return errors.Errorf("failed to find node for mon %q in assignment map", m.DaemonName) @@ -871,7 +872,7 @@ func (c *Cluster) initMonIPs(mons []*monConfig) error { if err != nil { return errors.Wrapf(err, "failed to export service %q", monService.Name) } - logger.Infof("mon %q exported IP is %s", m.DaemonName, exportedIP) + log.NamespacedInfo(c.Namespace, logger, "mon %q exported IP is %s", m.DaemonName, exportedIP) m.PublicIP = exportedIP } else { m.PublicIP = monService.Spec.ClusterIP @@ -889,15 +890,15 @@ func (c *Cluster) initMonIPs(mons []*monConfig) error { func (c *Cluster) removeCanaryDeployments(labelSelector string) { canaryDeployments, err := k8sutil.GetDeployments(c.ClusterInfo.Context, c.context.Clientset, c.Namespace, labelSelector) if err != nil { - logger.Warningf("failed to get the list of monitor canary deployments. %v", err) + log.NamespacedWarning(c.Namespace, logger, "failed to get the list of monitor canary deployments. %v", err) return } // Delete the canary mons for _, canary := range canaryDeployments.Items { - logger.Infof("cleaning up canary monitor deployment %q", canary.Name) + log.NamespacedInfo(c.Namespace, logger, "cleaning up canary monitor deployment %q", canary.Name) if err := k8sutil.DeleteDeployment(c.ClusterInfo.Context, c.context.Clientset, c.Namespace, canary.Name); err != nil { - logger.Warningf("failed to delete canary monitor deployment %q. %v", canary.Name, err) + log.NamespacedWarning(c.Namespace, logger, "failed to delete canary monitor deployment %q. %v", canary.Name, err) } } } @@ -929,7 +930,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error { } // scheduling for this monitor has already been completed if _, ok := c.mapping.Schedule[mon.DaemonName]; ok { - logger.Debugf("mon %s already scheduled", mon.DaemonName) + log.NamespacedDebug(c.Namespace, logger, "mon %s already scheduled", mon.DaemonName) continue } @@ -951,14 +952,14 @@ func (c *Cluster) assignMons(mons []*monConfig) error { result, err := waitForMonitorScheduling(c, deployment) if err != nil { - logger.Errorf("failed to schedule mon %q. %v", mon.DaemonName, err) + log.NamespacedError(c.Namespace, logger, "failed to schedule mon %q. %v", mon.DaemonName, err) failedMonSchedule = true return } nodeChoice := result.Node if nodeChoice == nil { - logger.Errorf("failed to schedule monitor %q", mon.DaemonName) + log.NamespacedError(c.Namespace, logger, "failed to schedule monitor %q", mon.DaemonName) failedMonSchedule = true return } @@ -968,21 +969,21 @@ func (c *Cluster) assignMons(mons []*monConfig) error { // directly to a node selector on the monitor pod. var schedule *controller.MonScheduleInfo if c.spec.Network.IsHost() || c.monVolumeClaimTemplate(mon) == nil { - logger.Infof("mon %s assigned to node %s", mon.DaemonName, nodeChoice.Name) + log.NamespacedInfo(c.Namespace, logger, "mon %s assigned to node %s", mon.DaemonName, nodeChoice.Name) schedule, err = getNodeInfoFromNode(*nodeChoice) if err != nil { - logger.Errorf("failed to get node info for node %q. %v", nodeChoice.Name, err) + log.NamespacedError(c.Namespace, logger, "failed to get node info for node %q. %v", nodeChoice.Name, err) failedMonSchedule = true return } } else { - logger.Infof("mon %q placement using native scheduler", mon.DaemonName) + log.NamespacedInfo(c.Namespace, logger, "mon %q placement using native scheduler", mon.DaemonName) } if c.spec.ZonesRequired() { if schedule == nil { schedule = &controller.MonScheduleInfo{} } - logger.Infof("mon %q is assigned to zone %q", mon.DaemonName, mon.Zone) + log.NamespacedInfo(c.Namespace, logger, "mon %q is assigned to zone %q", mon.DaemonName, mon.Zone) schedule.Zone = mon.Zone } @@ -998,7 +999,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error { return errors.New("failed to schedule mons") } - logger.Debug("mons have been scheduled") + log.NamespacedDebug(c.Namespace, logger, "mons have been scheduled") return nil } @@ -1040,10 +1041,10 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e deployments, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).List(c.ClusterInfo.Context, metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", AppName)}) if err != nil { if kerrors.IsNotFound(err) { - logger.Infof("0 of %d expected mon deployments exist. creating new deployment(s).", len(mons)) + log.NamespacedInfo(c.Namespace, logger, "0 of %d expected mon deployments exist. creating new deployment(s).", len(mons)) onlyCheckQuorumOnce = true } else { - logger.Warningf("failed to list mon deployments. attempting to continue. %v", err) + log.NamespacedWarning(c.Namespace, logger, "failed to list mon deployments. attempting to continue. %v", err) } } @@ -1055,10 +1056,10 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e } } if len(deployments.Items) < len(mons) { - logger.Infof("%d of %d expected mon deployments exist. creating new deployment(s).", len(deployments.Items), len(mons)) + log.NamespacedInfo(c.Namespace, logger, "%d of %d expected mon deployments exist. creating new deployment(s).", len(deployments.Items), len(mons)) onlyCheckQuorumOnce = true } else if readyReplicas == 0 { - logger.Infof("%d of %d expected mons are ready. creating or updating deployments without checking quorum in attempt to achieve a healthy mon cluster", readyReplicas, len(mons)) + log.NamespacedInfo(c.Namespace, logger, "%d of %d expected mons are ready. creating or updating deployments without checking quorum in attempt to achieve a healthy mon cluster", readyReplicas, len(mons)) onlyCheckQuorumOnce = true } @@ -1076,7 +1077,7 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e // we need to do everything possible to verify the basic health of a cluster, complete the first orchestration, // and start watching for all the CRs. If mons still have quorum we can continue with the orchestration even // if they aren't all up. - logger.Errorf("attempting to continue after failing to start mon %q. %v", mons[i].DaemonName, err) + log.NamespacedError(c.Namespace, logger, "attempting to continue after failing to start mon %q. %v", mons[i].DaemonName, err) } // For the initial deployment (first creation) it's expected to not have all the monitors in quorum @@ -1091,13 +1092,13 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e } } - logger.Infof("mons created: %d", len(mons)) + log.NamespacedInfo(c.Namespace, logger, "mons created: %d", len(mons)) // Final verification that **all** mons are in quorum // Do not proceed if one monitor is still syncing // Only do this when monitors versions are different so we don't block the orchestration if a mon is down. versions, err := cephclient.GetAllCephDaemonVersions(c.context, c.ClusterInfo) if err != nil { - logger.Warningf("failed to get ceph daemons versions; this likely means there is no cluster yet. %v", err) + log.NamespacedWarning(c.Namespace, logger, "failed to get ceph daemons versions; this likely means there is no cluster yet. %v", err) } else { if len(versions.Mon) != 1 { requireAllInQuorum = true @@ -1113,20 +1114,20 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e func (c *Cluster) checkForExtraMonResources(mons []*monConfig, deployments []apps.Deployment) string { // If there are fewer mon deployments than the desired count, no need to remove an extra. if len(deployments) <= c.spec.Mon.Count || len(deployments) <= len(mons) { - logger.Debug("no extra mon deployments to remove") + log.NamespacedDebug(c.Namespace, logger, "no extra mon deployments to remove") return "" } // If there are fewer mons in the list than expected, either new mons are being created for // a new cluster, or a mon failover is in progress and the list of mons only // includes the single mon that was just started if len(mons) < c.spec.Mon.Count { - logger.Debug("new cluster or mon failover in progress, not checking for extra mon deployments") + log.NamespacedDebug(c.Namespace, logger, "new cluster or mon failover in progress, not checking for extra mon deployments") return "" } // If there are more deployments than expected mons from the ceph quorum, // find the extra mon deployment and clean it up. - logger.Infof("there is an extra mon deployment that is not needed and not in quorum") + log.NamespacedInfo(c.Namespace, logger, "there is an extra mon deployment that is not needed and not in quorum") for _, deploy := range deployments { monName := deploy.Labels[controller.DaemonIDLabel] found := false @@ -1138,7 +1139,7 @@ func (c *Cluster) checkForExtraMonResources(mons []*monConfig, deployments []app } } if !found { - logger.Infof("deleting extra mon deployment %q", deploy.Name) + log.NamespacedInfo(c.Namespace, logger, "deleting extra mon deployment %q", deploy.Name) c.removeMonResources(monName) return monName } @@ -1227,7 +1228,7 @@ func (c *Cluster) persistExpectedMonDaemonsAsEndpointSlice() error { // Theoretically, we could now go ahead with the normal code path to // delete (both IPv4 and IPv6) EndpointSlices, but 0 mons is certainly // an error state, so it's better to do nothing destructive right now. - logger.Debug("no mon addresses found, skipping endpointslice resource reconciliation") + log.NamespacedDebug(c.Namespace, logger, "no mon addresses found, skipping endpointslice resource reconciliation") return nil } @@ -1241,7 +1242,7 @@ func (c *Cluster) persistExpectedMonDaemonsAsEndpointSlice() error { } ip := net.ParseIP(host) if ip == nil { - logger.Warningf("invalid IP parsed from mon endpoint: %s", mon.Endpoint) + log.NamespacedWarning(c.Namespace, logger, "invalid IP parsed from mon endpoint: %s", mon.Endpoint) continue } if ip.To4() != nil { @@ -1271,12 +1272,12 @@ func (c *Cluster) createEndpointSliceForAddresses(addresses []string, addressTyp } if len(addresses) == 0 { - logger.Debugf("no %s addresses found, deleting existing %q endpointslice if exists", addressType, sliceName) + log.NamespacedDebug(c.Namespace, logger, "no %s addresses found, deleting existing %q endpointslice if exists", addressType, sliceName) if err := client.Delete(c.ClusterInfo.Context, sliceName, metav1.DeleteOptions{}); err != nil { if kerrors.IsNotFound(err) { - logger.Debugf("endpointslice %q not found, nothing to delete", sliceName) + log.NamespacedDebug(c.Namespace, logger, "endpointslice %q not found, nothing to delete", sliceName) } else { - logger.Errorf("failed to delete endpointslice %q: %v", sliceName, err) + log.NamespacedError(c.Namespace, logger, "failed to delete endpointslice %q: %v", sliceName, err) } } return nil @@ -1324,13 +1325,13 @@ func (c *Cluster) createEndpointSliceForAddresses(addresses []string, addressTyp return errors.Wrapf(err, "failed to create %s endpointslice", addressType) } - logger.Debugf("updating existing %s endpointslice %s", addressType, sliceName) + log.NamespacedDebug(c.Namespace, logger, "updating existing %s endpointslice %s", addressType, sliceName) if _, err = client.Update(c.ClusterInfo.Context, endpointSlice, metav1.UpdateOptions{}); err != nil { return errors.Wrapf(err, "failed to update %s endpointslice", addressType) } } - logger.Infof("created/updated %s endpointslice with addresses: %+v", addressType, addresses) + log.NamespacedInfo(c.Namespace, logger, "created/updated %s endpointslice with addresses: %+v", addressType, addresses) return nil } @@ -1397,12 +1398,12 @@ func (c *Cluster) persistExpectedMonDaemonsInConfigMap() error { return errors.Wrap(err, "failed to create mon endpoint config map") } - logger.Debugf("updating config map %s that already exists", configMap.Name) + log.NamespacedDebug(c.Namespace, logger, "updating config map %s that already exists", configMap.Name) if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(c.ClusterInfo.Context, configMap, metav1.UpdateOptions{}); err != nil { return errors.Wrap(err, "failed to update mon endpoint config map") } } - logger.Infof("saved mon endpoints to config map %+v", configMap.Data) + log.NamespacedInfo(c.Namespace, logger, "saved mon endpoints to config map %+v", configMap.Data) return nil } @@ -1420,7 +1421,7 @@ func (c *Cluster) getStoredMaxMonID() (string, error) { // if the configmap cannot be loaded, assume a new cluster. If the mons have previously // been created, the maxMonID will anyway analyze them to ensure the index is correct // even if this error occurs. - logger.Infof("existing maxMonID not found or failed to load. %v", err) + log.NamespacedInfo(c.Namespace, logger, "existing maxMonID not found or failed to load. %v", err) return "-1", nil } @@ -1446,11 +1447,11 @@ func (c *Cluster) commitMaxMonIDRequireIncrementing(desiredMaxMonID int, require } if requireIncrementing && existingMax >= desiredMaxMonID { - logger.Infof("no need to commit maxMonID %d since it is not greater than existing maxMonID %d", desiredMaxMonID, existingMax) + log.NamespacedInfo(c.Namespace, logger, "no need to commit maxMonID %d since it is not greater than existing maxMonID %d", desiredMaxMonID, existingMax) return nil } - logger.Infof("updating maxMonID from %d to %d", existingMax, desiredMaxMonID) + log.NamespacedInfo(c.Namespace, logger, "updating maxMonID from %d to %d", existingMax, desiredMaxMonID) configmap.Data[controller.MaxMonIDKey] = strconv.Itoa(desiredMaxMonID) if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(c.ClusterInfo.Context, configmap, metav1.UpdateOptions{}); err != nil { @@ -1476,7 +1477,7 @@ func (c *Cluster) updateMon(m *monConfig, d *apps.Deployment) error { _ = k8sutil.ExpandPVCIfRequired(c.ClusterInfo.Context, c.context.Client, desiredPvc, existingPvc) } - logger.Infof("deployment for mon %s already exists. updating if needed", + log.NamespacedInfo(c.Namespace, logger, "deployment for mon %s already exists. updating if needed", d.Name) err := updateDeploymentAndWait(c.context, c.ClusterInfo, d, config.MonType, m.DaemonName, c.spec.SkipUpgradeChecks, false) @@ -1519,6 +1520,7 @@ func (c *Cluster) updateMon(m *monConfig, d *apps.Deployment) error { func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) error { // check if the monitor deployment already exists. if the deployment does // exist, also determine if it using pvc storage. + log.NamespacedInfo(c.Namespace, logger, "starting mon %q", m.DaemonName) pvcExists := false deploymentExists := false @@ -1554,10 +1556,10 @@ func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) e pvcName := m.ResourceName d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, controller.DaemonVolumesDataPVC(pvcName)) controller.AddVolumeMountSubPath(&d.Spec.Template.Spec, "ceph-daemon-data") - logger.Debugf("adding pvc volume source %s to mon deployment %s", pvcName, d.Name) + log.NamespacedDebug(c.Namespace, logger, "adding pvc volume source %s to mon deployment %s", pvcName, d.Name) } else { d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, controller.DaemonVolumesDataHostPath(m.DataPathMap)...) - logger.Debugf("adding host path volume source to mon deployment %s", d.Name) + log.NamespacedDebug(c.Namespace, logger, "adding host path volume source to mon deployment %s", d.Name) } // placement settings from the CRD @@ -1611,7 +1613,7 @@ func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) e _, err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Create(c.ClusterInfo.Context, pvc, metav1.CreateOptions{}) if err != nil { if kerrors.IsAlreadyExists(err) { - logger.Debugf("cannot create mon %s pvc %s: already exists.", d.Name, pvc.Name) + log.NamespacedDebug(c.Namespace, logger, "cannot create mon %s pvc %s: already exists.", d.Name, pvc.Name) } else { return errors.Wrapf(err, "failed to create mon %s pvc %s", d.Name, pvc.Name) } @@ -1631,7 +1633,7 @@ func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) e k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), k8sutil.LabelHostname(), map[string]string{k8sutil.AppAttr: AppName}, nodeSelector) - logger.Debugf("Starting mon: %+v", d.Name) + log.NamespacedDebug(c.Namespace, logger, "Starting mon: %+v", d.Name) _, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(c.ClusterInfo.Context, d, metav1.CreateOptions{}) if err != nil { return errors.Wrapf(err, "failed to create mon deployment %s", d.Name) @@ -1667,10 +1669,10 @@ func isMonIPUpdateRequiredForHostNetwork(mon string, isMonUsingHostNetwork bool, func hasMonPathChanged(d *apps.Deployment, claim *corev1.PersistentVolumeClaim) bool { if d.Labels["pvc_name"] == "" && claim != nil { - logger.Infof("skipping update for mon %q where path has changed from hostPath to pvc", d.Name) + log.NamespacedInfo(d.Namespace, logger, "skipping update for mon %q where path has changed from hostPath to pvc", d.Name) return true } else if d.Labels["pvc_name"] != "" && claim == nil { - logger.Infof("skipping update for mon %q where path has changed from pvc to hostPath", d.Name) + log.NamespacedInfo(d.Namespace, logger, "skipping update for mon %q where path has changed from pvc to hostPath", d.Name) return true } @@ -1678,7 +1680,7 @@ func hasMonPathChanged(d *apps.Deployment, claim *corev1.PersistentVolumeClaim) } func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, mons []string, sleepTime int, requireAllInQuorum bool) error { - logger.Infof("waiting for mon quorum with %v", mons) + log.NamespacedInfo(clusterInfo.Namespace, logger, "waiting for mon quorum with %v", mons) // wait for monitors to establish quorum retryCount := 0 @@ -1705,18 +1707,18 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.Cl for _, m := range mons { running, err := k8sutil.PodsRunningWithLabel(clusterInfo.Context, context.Clientset, clusterInfo.Namespace, fmt.Sprintf("app=%s,mon=%s", AppName, m)) if err != nil { - logger.Infof("failed to query mon pod status, trying again. %v", err) + log.NamespacedInfo(clusterInfo.Namespace, logger, "failed to query mon pod status, trying again. %v", err) continue } if running > 0 { runningMonNames = append(runningMonNames, m) } else { allPodsRunning = false - logger.Infof("mon %s is not yet running", m) + log.NamespacedInfo(clusterInfo.Namespace, logger, "mon %s is not yet running", m) } } - logger.Infof("mons running: %v", runningMonNames) + log.NamespacedInfo(clusterInfo.Namespace, logger, "mons running: %v", runningMonNames) if !allPodsRunning && requireAllInQuorum { continue } @@ -1725,12 +1727,12 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.Cl // their quorum status monQuorumStatusResp, err := cephclient.GetMonQuorumStatus(context, clusterInfo) if err != nil { - logger.Debugf("failed to get quorum_status. %v", err) + log.NamespacedDebug(clusterInfo.Namespace, logger, "failed to get quorum_status. %v", err) continue } if !requireAllInQuorum { - logQuorumMembers(monQuorumStatusResp) + logQuorumMembers(clusterInfo.Namespace, monQuorumStatusResp) break } @@ -1739,14 +1741,14 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.Cl for _, name := range mons { if !monFoundInQuorum(name, monQuorumStatusResp) { // found an initial monitor that is not in quorum, bail out of this retry - logger.Warningf("monitor %s is not in quorum list", name) + log.NamespacedWarning(clusterInfo.Namespace, logger, "monitor %s is not in quorum list", name) allInQuorum = false break } } if allInQuorum { - logQuorumMembers(monQuorumStatusResp) + logQuorumMembers(clusterInfo.Namespace, monQuorumStatusResp) break } } @@ -1754,14 +1756,14 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.Cl return nil } -func logQuorumMembers(monQuorumStatusResp cephclient.MonStatusResponse) { +func logQuorumMembers(namespace string, monQuorumStatusResp cephclient.MonStatusResponse) { var monsInQuorum []string for _, m := range monQuorumStatusResp.MonMap.Mons { if monFoundInQuorum(m.Name, monQuorumStatusResp) { monsInQuorum = append(monsInQuorum, m.Name) } } - logger.Infof("Monitors in quorum: %v", monsInQuorum) + log.NamespacedInfo(namespace, logger, "Monitors in quorum: %v", monsInQuorum) } func monFoundInQuorum(name string, monQuorumStatusResp cephclient.MonStatusResponse) bool { @@ -1796,14 +1798,14 @@ func requiredDuringScheduling(spec *cephv1.ClusterSpec) bool { } func (c *Cluster) acquireOrchestrationLock() { - logger.Debugf("Acquiring lock for mon orchestration") + log.NamespacedDebug(c.Namespace, logger, "Acquiring lock for mon orchestration") c.orchestrationMutex.Lock() - logger.Debugf("Acquired lock for mon orchestration") + log.NamespacedDebug(c.Namespace, logger, "Acquired lock for mon orchestration") } func (c *Cluster) releaseOrchestrationLock() { c.orchestrationMutex.Unlock() - logger.Debugf("Released lock for mon orchestration") + log.NamespacedDebug(c.Namespace, logger, "Released lock for mon orchestration") } func (c *Cluster) RotateMonCephxKeys(clusterObj *cephv1.CephCluster) (bool, error) { @@ -1817,16 +1819,16 @@ func (c *Cluster) RotateMonCephxKeys(clusterObj *cephv1.CephCluster) (bool, erro } if !shouldRotateMonKeys { - logger.Debugf("cephx key rotation for mon daemon in the namespace %q is not required", c.ClusterInfo.Namespace) + log.NamespacedDebug(c.Namespace, logger, "cephx key rotation for mon daemon in the namespace %q is not required", c.ClusterInfo.Namespace) return shouldRotateMonKeys, nil } if !c.ClusterInfo.CephVersion.IsAtLeast(keyring.CephAuthMonRotateSupportedVersion) { - logger.Debugf("cephx key rotation for mons in namespace %q is indicated, but ceph version %#v does not support mon key rotation", c.Namespace, c.ClusterInfo.CephVersion) + log.NamespacedDebug(c.Namespace, logger, "cephx key rotation for mons in namespace %q is indicated, but ceph version %#v does not support mon key rotation", c.Namespace, c.ClusterInfo.CephVersion) return false, nil } - logger.Infof("cephx keys for mon daemons in the namespace %q will be rotated", c.ClusterInfo.Namespace) + log.NamespacedInfo(c.Namespace, logger, "cephx keys for mon daemons in the namespace %q will be rotated", c.ClusterInfo.Namespace) k := keyring.GetSecretStore(c.context, c.ClusterInfo, c.ClusterInfo.OwnerInfo) newKey, err := k.RotateKey(controller.MonCephxUser) @@ -1847,7 +1849,7 @@ func (c *Cluster) RotateMonCephxKeys(clusterObj *cephv1.CephCluster) (bool, erro return shouldRotateMonKeys, errors.Wrapf(err, "failed to save mon keyring secret after rotating the mon cephx keys in the namespace %q", c.ClusterInfo.Namespace) } - logger.Infof("successfully rotated cephx keys for mon daemons in the namespace %q", c.ClusterInfo.Namespace) + log.NamespacedInfo(c.Namespace, logger, "successfully rotated cephx keys for mon daemons in the namespace %q", c.ClusterInfo.Namespace) return shouldRotateMonKeys, nil } @@ -1860,11 +1862,11 @@ func (c *Cluster) UpdateMonCephxStatus(didRotate bool) error { } updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, c.ClusterInfo.CephVersion, cluster.Status.Cephx.Mon) cluster.Status.Cephx.Mon = updatedStatus - logger.Debugf("updating mon daemon cephx status to %+v", cluster.Status.Cephx.Mgr) + log.NamespacedDebug(c.Namespace, logger, "updating mon daemon cephx status to %+v", cluster.Status.Cephx.Mgr) if err := reporting.UpdateStatus(c.context.Client, cluster); err != nil { return errors.Wrapf(err, "failed to update cluster cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace) } - logger.Infof("successfully updated the cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace) + log.NamespacedInfo(c.Namespace, logger, "successfully updated the cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace) return nil }) diff --git a/pkg/operator/ceph/cluster/mon/predicate.go b/pkg/operator/ceph/cluster/mon/predicate.go index c3438f0ce..bb1889d76 100644 --- a/pkg/operator/ceph/cluster/mon/predicate.go +++ b/pkg/operator/ceph/cluster/mon/predicate.go @@ -22,6 +22,7 @@ import ( "sort" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/util/log" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -42,8 +43,8 @@ func PredicateMonEndpointChanges[T *corev1.ConfigMap]() predicate.TypedFuncs[T] cmOld := (*corev1.ConfigMap)(e.ObjectOld) cmNew := (*corev1.ConfigMap)(e.ObjectNew) if cmNew.GetName() == EndpointConfigMapName { - if wereMonEndpointsUpdated(cmOld.Data, cmNew.Data) { - logger.Info("monitor endpoints changed, updating the bootstrap peer token") + if wereMonEndpointsUpdated(cmNew.Namespace, cmOld.Data, cmNew.Data) { + log.NamespacedInfo(cmNew.Namespace, logger, "monitor endpoints changed, updating the bootstrap peer token") return true } } @@ -52,7 +53,7 @@ func PredicateMonEndpointChanges[T *corev1.ConfigMap]() predicate.TypedFuncs[T] } } -func wereMonEndpointsUpdated(oldCMData, newCMData map[string]string) bool { +func wereMonEndpointsUpdated(namespace string, oldCMData, newCMData map[string]string) bool { // Check the mapping key first if oldMapping, ok := oldCMData["mapping"]; ok { if newMapping, ok := newCMData["mapping"]; ok { @@ -60,20 +61,20 @@ func wereMonEndpointsUpdated(oldCMData, newCMData map[string]string) bool { var oldMappingToGo opcontroller.Mapping err := json.Unmarshal([]byte(oldMapping), &oldMappingToGo) if err != nil { - logger.Debugf("failed to unmarshal new. %v", err) + log.NamespacedDebug(namespace, logger, "failed to unmarshal new. %v", err) return false } var newMappingToGo opcontroller.Mapping err = json.Unmarshal([]byte(newMapping), &newMappingToGo) if err != nil { - logger.Debugf("failed to unmarshal new. %v", err) + log.NamespacedDebug(namespace, logger, "failed to unmarshal new. %v", err) return false } // If length is different, monitors are different if len(oldMappingToGo.Schedule) != len(newMappingToGo.Schedule) { - logger.Debugf("mons were added or removed from the endpoints cm") + log.NamespacedDebug(namespace, logger, "mons were added or removed from the endpoints cm") return true } // Since Schedule is map, it's unordered, so let's order it @@ -92,7 +93,7 @@ func wereMonEndpointsUpdated(oldCMData, newCMData map[string]string) bool { // Iterate over the map and compare the values for _, v := range oldKeys { if !reflect.DeepEqual(oldMappingToGo.Schedule[v], newMappingToGo.Schedule[v]) { - logger.Debugf("oldMappingToGo.Schedule[v] AND newMappingToGo.Schedule[v]: %v | %v", oldMappingToGo.Schedule[v], newMappingToGo.Schedule[v]) + log.NamespacedDebug(namespace, logger, "oldMappingToGo.Schedule[v] AND newMappingToGo.Schedule[v]: %v | %v", oldMappingToGo.Schedule[v], newMappingToGo.Schedule[v]) return true } } diff --git a/pkg/operator/ceph/cluster/mon/predicate_test.go b/pkg/operator/ceph/cluster/mon/predicate_test.go index 505ca77be..7bec7c1a9 100644 --- a/pkg/operator/ceph/cluster/mon/predicate_test.go +++ b/pkg/operator/ceph/cluster/mon/predicate_test.go @@ -37,7 +37,7 @@ func TestWereMonEndpointsUpdated(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := wereMonEndpointsUpdated(tt.args.oldCMData, tt.args.newCMData); got != tt.want { + if got := wereMonEndpointsUpdated("ns", tt.args.oldCMData, tt.args.newCMData); got != tt.want { t.Errorf("whereMonEndpointsUpdated() = %v, want %v", got, tt.want) } }) diff --git a/pkg/operator/ceph/cluster/mon/service.go b/pkg/operator/ceph/cluster/mon/service.go index dccde5d2b..cb47005f5 100644 --- a/pkg/operator/ceph/cluster/mon/service.go +++ b/pkg/operator/ceph/cluster/mon/service.go @@ -21,6 +21,7 @@ import ( "github.com/pkg/errors" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -54,7 +55,7 @@ func (c *Cluster) createService(mon *monConfig) (*v1.Service, error) { if mon.PublicIP != "" { _, err := c.context.Clientset.CoreV1().Services(c.Namespace).Get(c.ClusterInfo.Context, svcDef.Name, metav1.GetOptions{}) if err != nil && kerrors.IsNotFound(err) { - logger.Infof("ensuring the clusterIP for mon %q is %q", mon.DaemonName, mon.PublicIP) + log.NamespacedInfo(c.Namespace, logger, "ensuring the clusterIP for mon %q is %q", mon.DaemonName, mon.PublicIP) svcDef.Spec.ClusterIP = mon.PublicIP } } @@ -69,7 +70,7 @@ func (c *Cluster) createService(mon *monConfig) (*v1.Service, error) { return nil, nil } - logger.Infof("mon %q cluster IP is %s", mon.DaemonName, s.Spec.ClusterIP) + log.NamespacedInfo(c.Namespace, logger, "mon %q cluster IP is %s", mon.DaemonName, s.Spec.ClusterIP) return s, nil } @@ -78,12 +79,12 @@ func (c *Cluster) exportService(service *v1.Service, monDaemon string) (string, // query on ..svc.clusterset.local requires the mon canary pod to be running defer c.removeCanaryDeployments(monCanaryLabelSelector + fmt.Sprintf(",mon=%s", monDaemon)) - logger.Infof("exporting service %q", service.Name) + log.NamespacedInfo(c.Namespace, logger, "exporting service %q", service.Name) exportedIP, err := k8sutil.ExportService(c.ClusterInfo.Context, c.context, service, c.spec.Network.MultiClusterService.ClusterID) if err != nil { return "", errors.Wrapf(err, "failed to export service %q", service.Name) } - logger.Infof("mon %q exported IP is %s", service.Name, exportedIP) + log.NamespacedInfo(c.Namespace, logger, "mon %q exported IP is %s", service.Name, exportedIP) return exportedIP, nil } diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index a99cc8f24..bdc29f609 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -28,6 +28,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -187,7 +188,7 @@ func makeMonSecurityContext() *corev1.PodSecurityContext { } func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*corev1.Pod, error) { - logger.Debugf("monConfig: %+v", monConfig) + log.NamespacedDebug(c.Namespace, logger, "monConfig: %+v", monConfig) podSpec := corev1.PodSpec{ InitContainers: []corev1.Container{ c.makeChownInitContainer(monConfig), @@ -393,17 +394,16 @@ func UpdateCephDeploymentAndWait(context *clusterd.Context, clusterInfo *client. callback := func(action string) error { // At this point, we are in an upgrade if skipUpgradeChecks { - logger.Warningf("this is a Ceph upgrade, not performing upgrade checks because skipUpgradeChecks is %t", skipUpgradeChecks) + log.NamespacedWarning(clusterInfo.Namespace, logger, "this is a Ceph upgrade, not performing upgrade checks because skipUpgradeChecks is %t", skipUpgradeChecks) return nil } - logger.Infof("checking if we can %s the deployment %s", action, deployment.Name) - + log.NamespacedInfo(clusterInfo.Namespace, logger, "checking if we can %s the deployment %s", action, deployment.Name) if action == "stop" { err := client.OkToStop(context, clusterInfo, deployment.Name, daemonType, daemonName) if err != nil { if continueUpgradeAfterChecksEvenIfNotHealthy { - logger.Infof("The %s daemon %s is not ok-to-stop but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so proceeding to stop...", daemonType, daemonName) + log.NamespacedInfo(clusterInfo.Namespace, logger, "The %s daemon %s is not ok-to-stop but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so proceeding to stop...", daemonType, daemonName) return nil } return errors.Wrapf(err, "failed to check if we can %s the deployment %s", action, deployment.Name) @@ -414,7 +414,7 @@ func UpdateCephDeploymentAndWait(context *clusterd.Context, clusterInfo *client. err := client.OkToContinue(context, clusterInfo, deployment.Name, daemonType, daemonName) if err != nil { if continueUpgradeAfterChecksEvenIfNotHealthy { - logger.Infof("The %s daemon %s is not ok-to-continue but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so continuing...", daemonType, daemonName) + log.NamespacedInfo(clusterInfo.Namespace, logger, "The %s daemon %s is not ok-to-continue but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so continuing...", daemonType, daemonName) return nil } return errors.Wrapf(err, "failed to check if we can %s the deployment %s", action, deployment.Name) diff --git a/pkg/operator/ceph/cluster/monitoring.go b/pkg/operator/ceph/cluster/monitoring.go index ef4116fd1..964e0572c 100644 --- a/pkg/operator/ceph/cluster/monitoring.go +++ b/pkg/operator/ceph/cluster/monitoring.go @@ -25,6 +25,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/cluster/osd" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/util/log" ) var monitorDaemonList = []string{"mon", "osd", "status"} @@ -37,7 +38,7 @@ func (c *ClusterController) configureCephMonitoring(cluster *cluster, clusterInf if health, ok := cluster.monitoringRoutines.Load(daemon); ok { // If the context Err() is nil this means it hasn't been cancelled yet if health.(*opcontroller.ClusterHealth).InternalCtx.Err() == nil { - logger.Debugf("monitoring routine for %q is already running", daemon) + log.NamespacedDebug(cluster.Namespace, logger, "monitoring routine for %q is already running", daemon) if !isEnabled { health.(*opcontroller.ClusterHealth).InternalCancel() } @@ -79,19 +80,19 @@ func (c *ClusterController) startMonitoringCheck(cluster *cluster, clusterInfo * switch daemon { case "mon": healthChecker := mon.NewHealthChecker(cluster.mons) - logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace) + log.NamespacedInfo(cluster.Namespace, logger, "enabling ceph %s monitoring goroutine", daemon) go healthChecker.Check(&cluster.monitoringRoutines, daemon) case "osd": if !cluster.Spec.External.Enable { osdChecker := osd.NewOSDHealthMonitor(c.context, clusterInfo, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Spec.HealthCheck) - logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace) + log.NamespacedInfo(cluster.Namespace, logger, "enabling ceph %s monitoring goroutine", daemon) go osdChecker.Start(&cluster.monitoringRoutines, daemon) } case "status": cephChecker := newCephStatusChecker(c.context, clusterInfo, cluster.Spec) - logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace) + log.NamespacedInfo(cluster.Namespace, logger, "enabling ceph %s monitoring goroutine", daemon) go cephChecker.checkCephStatus(&cluster.monitoringRoutines, daemon) } } diff --git a/pkg/operator/ceph/cluster/nodedaemon/exporter.go b/pkg/operator/ceph/cluster/nodedaemon/exporter.go index a2298c002..93b8fefef 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/exporter.go +++ b/pkg/operator/ceph/cluster/nodedaemon/exporter.go @@ -28,6 +28,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/util/log" cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" @@ -57,11 +58,11 @@ func (r *ReconcileNode) createOrUpdateCephExporter(node corev1.Node, tolerations // CephVersion change is done temporarily, as some regression was detected in Ceph version 17.2.6 which is summarised here https://github.com/ceph/ceph/pull/50718#issuecomment-1505608312. // Thus, disabling ceph-exporter for now until all the regression are fixed. if !cephVersion.IsAtLeast(MinVersionForCephExporter) { - logger.Infof("Skipping exporter reconcile on ceph version %q", cephVersion.String()) + log.NamespacedInfo(cephCluster.Namespace, logger, "Skipping exporter reconcile on ceph version %q", cephVersion.String()) return controllerutil.OperationResultNone, nil } if cephCluster.Spec.Monitoring.MetricsDisabled { - logger.Info("Skipping exporter reconcile since monitoring.metricsDisabled is true") + log.NamespacedInfo(cephCluster.Namespace, logger, "Skipping exporter reconcile since monitoring.metricsDisabled is true") return controllerutil.OperationResultNone, nil } @@ -283,10 +284,10 @@ func applyCephExporterLabels(cephCluster cephv1.CephCluster, serviceMonitor *mon serviceMonitor.Spec.Endpoints[0].RelabelConfigs = append( serviceMonitor.Spec.Endpoints[0].RelabelConfigs, relabelConfig) } else { - logger.Info("rook.io/managedBy not specified in ceph-exporter labels") + log.NamespacedInfo(cephCluster.Namespace, logger, "rook.io/managedBy not specified in ceph-exporter labels") } } else { - logger.Debug("ceph-exporter labels not specified") + log.NamespacedDebug(cephCluster.Namespace, logger, "ceph-exporter labels not specified") } } } diff --git a/pkg/operator/ceph/cluster/nodedaemon/keyring.go b/pkg/operator/ceph/cluster/nodedaemon/keyring.go index 5a35464ed..6a8469c26 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/keyring.go +++ b/pkg/operator/ceph/cluster/nodedaemon/keyring.go @@ -26,6 +26,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config/keyring" "github.com/rook/rook/pkg/operator/ceph/reporting" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" @@ -96,7 +97,7 @@ func createCrashCollectorKeyring(s *keyring.SecretStore, context *clusterd.Conte } if shouldRotateCephxKeys { - logger.Infof("rotating cephx key for crash collector %q", crashCollectorKeyringUsername) + log.NamespacedInfo(clusterInfo.Namespace, logger, "rotating cephx key for crash collector %q", crashCollectorKeyringUsername) newKey, err := s.RotateKey(crashCollectorKeyringUsername) if err != nil { return "", errors.Wrapf(err, "failed to rotate cephx key for crash collector %q", crashCollectorKeyringUsername) @@ -124,7 +125,7 @@ func updateCrashCollectorCephxStatus(context *clusterd.Context, clusterInfo *cli if err := reporting.UpdateStatus(context.Client, cluster); err != nil { return errors.Wrap(err, "failed to update cluster cephx status for crash collector daemon") } - logger.Debugf("successfully updated the crash collector cephx status for cluster in namespace %q to %+v", cluster.Namespace, cluster.Status.Cephx.CrashCollector) + log.NamespacedDebug(clusterInfo.Namespace, logger, "successfully updated the crash collector cephx status for cluster in namespace %q to %+v", cluster.Namespace, cluster.Status.Cephx.CrashCollector) return nil }) @@ -161,7 +162,7 @@ func createOrUpdateCrashCollectorSecret(clusterInfo *client.ClusterInfo, crashCo return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", s.Name, clusterInfo.Namespace) } - logger.Infof("created kubernetes crash collector secret for cluster %q", clusterInfo.Namespace) + log.NamespacedInfo(clusterInfo.Namespace, logger, "created kubernetes crash collector secret") return nil } @@ -212,7 +213,7 @@ func createExporterKeyring(s *keyring.SecretStore, context *clusterd.Context, cl } if shouldRotateCephxKeys { - logger.Infof("rotating cephx key for ceph exporter %q", exporterKeyringUsername) + log.NamespacedInfo(clusterInfo.Namespace, logger, "rotating cephx key for ceph exporter %q", exporterKeyringUsername) newKey, err := s.RotateKey(exporterKeyringUsername) if err != nil { return "", errors.Wrapf(err, "failed to rotate cephx key for ceph exporter %q", exporterKeyringUsername) @@ -239,7 +240,7 @@ func updateCephExporterCephxStatus(context *clusterd.Context, clusterInfo *clien if err := reporting.UpdateStatus(context.Client, cluster); err != nil { return errors.Wrap(err, "failed to update cluster cephx status for ceph exporter daemon") } - logger.Debugf("successfully updated the ceph exporter cephx status for cluster in namespace %q to %+v", cluster.Namespace, cluster.Status.Cephx.CephExporter) + log.NamespacedDebug(clusterInfo.Namespace, logger, "successfully updated the ceph exporter cephx status for cluster in namespace %q to %+v", cluster.Namespace, cluster.Status.Cephx.CephExporter) return nil }) @@ -276,6 +277,6 @@ func createOrUpdateExporterSecret(clusterInfo *client.ClusterInfo, exporterSecre return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", s.Name, clusterInfo.Namespace) } - logger.Infof("created kubernetes exporter secret for cluster %q", clusterInfo.Namespace) + log.NamespacedInfo(clusterInfo.Namespace, logger, "created kubernetes exporter secret for cluster %q", clusterInfo.Namespace) return nil } diff --git a/pkg/operator/ceph/cluster/nodedaemon/pruner.go b/pkg/operator/ceph/cluster/nodedaemon/pruner.go index 19f8714bf..858cd29d2 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/pruner.go +++ b/pkg/operator/ceph/cluster/nodedaemon/pruner.go @@ -25,6 +25,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config/keyring" "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" v1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -34,7 +35,7 @@ import ( func (r *ReconcileNode) reconcileCrashPruner(namespace string, cephCluster cephv1.CephCluster, tolerations []corev1.Toleration) error { if cephCluster.Spec.CrashCollector.Disable { - logger.Debugf("crash collector is disabled in namespace %q so skipping crash retention reconcile", namespace) + log.NamespacedDebug(cephCluster.Namespace, logger, "crash collector is disabled in namespace %q so skipping crash retention reconcile", namespace) return nil } @@ -44,27 +45,27 @@ func (r *ReconcileNode) reconcileCrashPruner(namespace string, cephCluster cephv } if cephCluster.Spec.CrashCollector.DaysToRetain == 0 { - logger.Debug("deleting cronjob if it exists...") + log.NamespacedDebug(cephCluster.Namespace, logger, "deleting cronjob if it exists...") cronJob := &v1.CronJob{ObjectMeta: objectMeta} err := r.client.Delete(r.opManagerContext, cronJob) if err != nil { if kerrors.IsNotFound(err) { - logger.Debug("cronJob resource not found. Ignoring since object must be deleted.") + log.NamespacedDebug(cephCluster.Namespace, logger, "cronJob resource not found. Ignoring since object must be deleted.") } else { return err } } else { - logger.Debug("successfully deleted crash pruner cronjob.") + log.NamespacedDebug(cephCluster.Namespace, logger, "successfully deleted crash pruner cronjob.") } } else { - logger.Debugf("daysToRetain set to: %d", cephCluster.Spec.CrashCollector.DaysToRetain) + log.NamespacedDebug(cephCluster.Namespace, logger, "daysToRetain set to: %d", cephCluster.Spec.CrashCollector.DaysToRetain) op, err := r.createOrUpdateCephCron(cephCluster, tolerations) if err != nil { return errors.Wrapf(err, "node reconcile failed on op %q", op) } - logger.Debugf("cronjob successfully reconciled. operation: %q", op) + log.NamespacedDebug(cephCluster.Namespace, logger, "cronjob successfully reconciled. operation: %q", op) } return nil } diff --git a/pkg/operator/ceph/cluster/nodedaemon/reconcile.go b/pkg/operator/ceph/cluster/nodedaemon/reconcile.go index 5fe180f2a..30ae6e38f 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/reconcile.go +++ b/pkg/operator/ceph/cluster/nodedaemon/reconcile.go @@ -26,6 +26,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/cluster/rbd" cephver "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/rook/rook/pkg/util/log" appsv1 "k8s.io/api/apps/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" @@ -77,7 +78,7 @@ func (r *ReconcileNode) Reconcile(context context.Context, request reconcile.Req // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface result, err := r.reconcile(request) if err != nil { - logger.Error(err) + log.NamespacedError(request.Namespace, logger, "%s", err.Error()) } return result, err } @@ -95,7 +96,7 @@ func (r *ReconcileNode) cleanupExporterResources(ns string, nodeName string) (re } func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result, error) { - logger.Debugf("reconciling node: %q", request.Name) + log.NamespacedDebug(request.Namespace, logger, "reconciling node: %q", request.Name) // get the node object node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: request.Name}} @@ -105,7 +106,7 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result, // if a node is not present, check if there are any node daemons to remove err := r.listNodeDaemonsAndDelete(request.Name, "") if err != nil { - logger.Errorf("failed to list and delete deployment on node %q; user should delete them manually. %v", request.Name, err) + log.NamespacedError(request.Namespace, logger, "failed to list and delete deployment on node %q; user should delete them manually. %v", request.Name, err) } } else { return reconcile.Result{}, errors.Wrapf(err, "could not get node %q", request.Name) @@ -144,13 +145,13 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result, return reconcile.Result{}, errors.Wrapf(err, "could not get cephcluster in namespaces %q", namespace) } if len(cephClusters.Items) < 1 { - logger.Debugf("no CephCluster found in the namespace %q", namespace) + log.NamespacedDebug(namespace, logger, "no CephCluster found in the namespace %q", namespace) return reconcile.Result{}, nil } cephCluster := cephClusters.Items[0] if len(cephClusters.Items) > 1 { - logger.Errorf("more than one CephCluster found in the namespace %q, choosing the first one %q", namespace, cephCluster.GetName()) + log.NamespacedError(request.Namespace, logger, "more than one CephCluster found in the namespace %q, choosing the first one %q", namespace, cephCluster.GetName()) } allDisabled := r.removeDisabledCrashCollectorDaemons(cephCluster.Spec, namespace) && r.removeDisabledCephExporterDaemons(cephCluster.Spec, namespace) @@ -164,7 +165,7 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result, err = r.client.Get(r.opManagerContext, types.NamespacedName{Name: crashCollectorKeyName, Namespace: namespace}, secret) if err != nil { if kerrors.IsNotFound(err) { - logger.Debugf("secret %q in namespace %q not found. retrying in %q. %v", crashCollectorKeyName, namespace, waitForRequeueIfSecretNotCreated.RequeueAfter.String(), err) + log.NamespacedDebug(namespace, logger, "secret %q in namespace %q not found. retrying in %q. %v", crashCollectorKeyName, namespace, waitForRequeueIfSecretNotCreated.RequeueAfter.String(), err) return waitForRequeueIfSecretNotCreated, nil } @@ -174,7 +175,7 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result, clusterImage := cephCluster.Spec.CephVersion.Image cephVersion, err := opcontroller.GetImageVersion(cephCluster) if err != nil { - logger.Errorf("ceph version not found for image %q used by cluster %q in namespace %q. %v", clusterImage, cephCluster.Name, cephCluster.Namespace, err) + log.NamespacedError(request.Namespace, logger, "ceph version not found for image %q used by cluster %q in namespace %q. %v", clusterImage, cephCluster.Name, cephCluster.Namespace, err) return reconcile.Result{}, nil } @@ -233,19 +234,19 @@ func (r *ReconcileNode) createOrUpdateNodeDaemons(node corev1.Node, tolerations op, err := r.createOrUpdateCephCrash(node, tolerations, cephCluster, cephVersion) if err != nil { if op == "unchanged" { - logger.Debugf("crash collector unchanged on node %q", node.Name) + log.NamespacedDebug(cephCluster.Namespace, logger, "crash collector unchanged on node %q", node.Name) } else { return errors.Wrapf(err, "crash collector reconcile failed on op %q", op) } } else { - logger.Debugf("crash collector successfully reconciled for node %q. operation: %q", node.Name, op) + log.NamespacedDebug(cephCluster.Namespace, logger, "crash collector successfully reconciled for node %q. operation: %q", node.Name, op) } } if cephVersion.IsAtLeast(MinVersionForCephExporter) && !cephCluster.Spec.Monitoring.MetricsDisabled { op, err := r.createOrUpdateCephExporter(node, tolerations, cephCluster, cephVersion) if err != nil { if op == "unchanged" { - logger.Debugf("ceph exporter unchanged on node %q", node.Name) + log.NamespacedDebug(cephCluster.Namespace, logger, "ceph exporter unchanged on node %q", node.Name) } else { return errors.Wrapf(err, "ceph exporter reconcile failed on op %q", op) } @@ -253,7 +254,7 @@ func (r *ReconcileNode) createOrUpdateNodeDaemons(node corev1.Node, tolerations // CephVersion change is done temporarily, as some regression was detected in Ceph version 17.2.6 which is summarised here https://github.com/ceph/ceph/pull/50718#issuecomment-1505608312. // Thus, disabling ceph-exporter for now until all the regression are fixed. if cephVersion.IsAtLeast(MinVersionForCephExporter) { - logger.Debugf("ceph exporter successfully reconciled for node %q. operation: %q", node.Name, op) + log.NamespacedDebug(cephCluster.Namespace, logger, "ceph exporter successfully reconciled for node %q. operation: %q", node.Name, op) // create the metrics service service, err := MakeCephExporterMetricsService(cephCluster, exporterServiceMetricName, r.scheme) if err != nil { @@ -267,7 +268,7 @@ func (r *ReconcileNode) createOrUpdateNodeDaemons(node corev1.Node, tolerations if err := EnableCephExporterServiceMonitor(r.context, cephCluster, r.scheme, r.opManagerContext, exporterServiceMetricName); err != nil { return errors.Wrap(err, "failed to enable service monitor") } - logger.Debug("service monitor for ceph exporter was enabled successfully") + log.NamespacedDebug(cephCluster.Namespace, logger, "service monitor for ceph exporter was enabled successfully") } } } @@ -302,12 +303,12 @@ func (r *ReconcileNode) listDeploymentAndDelete(appName, nodeName, ns string) er return errors.Wrapf(err, "failed to list deployments in namespace %q", ns) } for _, d := range deploymentList.Items { - logger.Infof("deleting deployment %q for node %q", d.ObjectMeta.Name, nodeName) + log.NamespacedInfo(d.Namespace, logger, "deleting deployment %q for node %q", d.ObjectMeta.Name, nodeName) err := r.deleteDeployment(d) if err != nil { return errors.Wrapf(err, "failed to delete deployment %q in namespace %q", d.Name, d.Namespace) } - logger.Infof("successfully removed deployment %q in namespace %q from node %q", d.Name, d.Namespace, nodeName) + log.NamespacedInfo(d.Namespace, logger, "successfully removed deployment %q in namespace %q from node %q", d.Name, d.Namespace, nodeName) } return nil @@ -320,7 +321,7 @@ func (r *ReconcileNode) deleteNodeDaemon(appName, namespace string) { // Try to fetch the list of existing deployment and remove them err := r.client.List(r.opManagerContext, deploymentList, client.MatchingLabels{k8sutil.AppAttr: appName}, namespaceListOpts) if err != nil { - logger.Errorf("failed to list deployments in namespace %q, delete it/them manually. %v", namespace, err) + log.NamespacedError(namespace, logger, "failed to list deployments in namespace %q, delete it/them manually. %v", namespace, err) return } @@ -328,10 +329,10 @@ func (r *ReconcileNode) deleteNodeDaemon(appName, namespace string) { for _, d := range deploymentList.Items { err := r.deleteDeployment(d) if err != nil { - logger.Errorf("failed to delete deployment %q in namespace %q, delete it manually. %v", d.Name, d.Namespace, err) + log.NamespacedError(namespace, logger, "failed to delete deployment %q in namespace %q, delete it manually. %v", d.Name, d.Namespace, err) continue } - logger.Infof("Deployments %q in namespace %q successfully removed", d.Name, d.Namespace) + log.NamespacedInfo(d.Namespace, logger, "Deployments %q in namespace %q successfully removed", d.Name, d.Namespace) } } diff --git a/pkg/operator/ceph/cluster/osd/create.go b/pkg/operator/ceph/cluster/osd/create.go index 92120947f..80b839305 100644 --- a/pkg/operator/ceph/cluster/osd/create.go +++ b/pkg/operator/ceph/cluster/osd/create.go @@ -27,6 +27,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config/keyring" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" ) @@ -79,20 +80,20 @@ func (c *createConfig) createNewOSDsFromStatus( ) { if !c.awaitingStatusConfigMaps.Has(statusConfigMapName(nodeOrPVCName)) { // If there is a dangling OSD prepare configmap from another reconcile, don't process it - logger.Infof("not creating deployments for OSD prepare results found in ConfigMap %q which was not created for the latest storage spec", statusConfigMapName(nodeOrPVCName)) + log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "not creating deployments for OSD prepare results found in ConfigMap %q which was not created for the latest storage spec", statusConfigMapName(nodeOrPVCName)) return } if c.finishedStatusConfigMaps.Has(statusConfigMapName(nodeOrPVCName)) { // If we have already processed this configmap, don't process it again - logger.Infof("not creating deployments for OSD prepare results found in ConfigMap %q which was already processed", statusConfigMapName(nodeOrPVCName)) + log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "not creating deployments for OSD prepare results found in ConfigMap %q which was already processed", statusConfigMapName(nodeOrPVCName)) return } for i, osd := range status.OSDs { if c.deployments.Exists(osd.ID) { // This OSD will be handled by the updater - logger.Debugf("not creating deployment for OSD %d which already exists", osd.ID) + log.NamespacedDebug(c.cluster.clusterInfo.Namespace, logger, "not creating deployment for OSD %d which already exists", osd.ID) continue } @@ -102,13 +103,13 @@ func (c *createConfig) createNewOSDsFromStatus( c.cluster.clusterInfo.CephVersion, keyring.UninitializedCephxStatus()) if status.PvcBackedOSD { - logger.Infof("creating OSD %d on PVC %q", osd.ID, nodeOrPVCName) + log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "creating OSD %d on PVC %q", osd.ID, nodeOrPVCName) err := createDaemonOnPVCFunc(c.cluster, &status.OSDs[i], nodeOrPVCName, c.provisionConfig) if err != nil { errs.addError("%v", errors.Wrapf(err, "failed to create OSD %d on PVC %q", osd.ID, nodeOrPVCName)) } } else { - logger.Infof("creating OSD %d on node %q", osd.ID, nodeOrPVCName) + log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "creating OSD %d on node %q", osd.ID, nodeOrPVCName) err := createDaemonOnNodeFunc(c.cluster, &status.OSDs[i], nodeOrPVCName, c.provisionConfig) if err != nil { errs.addError("%v", errors.Wrapf(err, "failed to create OSD %d on node %q", osd.ID, nodeOrPVCName)) @@ -136,7 +137,7 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig, errs *provi // no valid VolumeSource is ready to run an osd if len(c.deviceSets) == 0 { - logger.Info("no storageClassDeviceSets defined to configure OSDs on PVCs") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "no storageClassDeviceSets defined to configure OSDs on PVCs") return sets.New[string](), nil } @@ -161,14 +162,14 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig, errs *provi metadataSource, metadataOK := volume.PVCSources[bluestorePVCMetadata] if metadataOK { - logger.Infof("OSD will have its main bluestore block on %q and its metadata device on %q", dataSource.ClaimName, metadataSource.ClaimName) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its main bluestore block on %q and its metadata device on %q", dataSource.ClaimName, metadataSource.ClaimName) } else { - logger.Infof("OSD will have its main bluestore block on %q", dataSource.ClaimName) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its main bluestore block on %q", dataSource.ClaimName) } walSource, walOK := volume.PVCSources[bluestorePVCWal] if walOK { - logger.Infof("OSD will have its wal device on %q", walSource.ClaimName) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its wal device on %q", walSource.ClaimName) } osdProps := osdProperties{ @@ -196,13 +197,13 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig, errs *provi // Allow updating OSD prepare pod if the OSD needs migration if c.migrateOSD != nil { if strings.Contains(c.migrateOSD.BlockPath, dataSource.ClaimName) { - logger.Infof("updating OSD prepare pod to replace OSD.%d", c.migrateOSD.ID) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "updating OSD prepare pod to replace OSD.%d", c.migrateOSD.ID) skipPreparePod = false } } if skipPreparePod { - logger.Infof("skipping OSD prepare job creation for PVC %q because OSD daemon using the PVC already exists", osdProps.crushHostname) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "skipping OSD prepare job creation for PVC %q because OSD daemon using the PVC already exists", osdProps.crushHostname) continue } @@ -264,13 +265,13 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig, errs *provi // usage of awaitingStatusConfigMaps in this file. func (c *Cluster) startProvisioningOverNodes(config *provisionConfig, errs *provisionErrors) (sets.Set[string], error) { if !c.spec.Storage.UseAllNodes && len(c.spec.Storage.Nodes) == 0 { - logger.Info("no nodes are defined for configuring OSDs on raw devices") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "no nodes are defined for configuring OSDs on raw devices") return sets.New[string](), nil } if c.spec.Storage.UseAllNodes { if len(c.spec.Storage.Nodes) > 0 { - logger.Warningf("useAllNodes is TRUE, but nodes are specified. NODES in the cluster CR will be IGNORED unless useAllNodes is FALSE.") + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "useAllNodes is TRUE, but nodes are specified. NODES in the cluster CR will be IGNORED unless useAllNodes is FALSE.") } // Get the list of all nodes in the cluster. The placement settings will be applied below. @@ -286,19 +287,19 @@ func (c *Cluster) startProvisioningOverNodes(config *provisionConfig, errs *prov } c.spec.Storage.Nodes = append(c.spec.Storage.Nodes, storageNode) } - logger.Debugf("storage nodes: %+v", c.spec.Storage.Nodes) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "storage nodes: %+v", c.spec.Storage.Nodes) } // generally speaking, this finds nodes which are capable of running new osds validNodes := k8sutil.GetValidNodes(c.clusterInfo.Context, c.spec.Storage, c.context.Clientset, cephv1.GetOSDPlacement(c.spec.Placement)) - logger.Infof("%d of the %d storage nodes are valid", len(validNodes), len(c.spec.Storage.Nodes)) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "%d of the %d storage nodes are valid", len(validNodes), len(c.spec.Storage.Nodes)) c.ValidStorage = *c.spec.Storage.DeepCopy() c.ValidStorage.Nodes = validNodes // no valid node is ready to run an osd if len(validNodes) == 0 { - logger.Warningf("no valid nodes available to run osds on nodes in namespace %q", c.clusterInfo.Namespace) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "no valid nodes available to run osds on nodes in namespace %q", c.clusterInfo.Namespace) return sets.New[string](), nil } @@ -316,12 +317,12 @@ func (c *Cluster) startProvisioningOverNodes(config *provisionConfig, errs *prov // don't care about osd device class resources since it will be overwritten later for prepareosd resources n := c.resolveNode(node.Name, "") if n == nil { - logger.Warningf("node %q did not resolve", node.Name) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "node %q did not resolve", node.Name) continue } if n.Name == "" { - logger.Warningf("skipping node with a blank name! %+v", n) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "skipping node with a blank name! %+v", n) continue } @@ -372,7 +373,7 @@ func (c *Cluster) runPrepareJob(osdProps *osdProperties, config *provisionConfig return errors.Wrapf(err, "failed to run osd provisioning job for %s %q", nodeOrPVC, nodeOrPVCName) } - logger.Infof("started OSD provisioning job for %s %q", nodeOrPVC, nodeOrPVCName) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "started OSD provisioning job for %s %q", nodeOrPVC, nodeOrPVCName) return nil } diff --git a/pkg/operator/ceph/cluster/osd/deviceSet.go b/pkg/operator/ceph/cluster/osd/deviceSet.go index 9cc1ce373..b4f1bd2c7 100644 --- a/pkg/operator/ceph/cluster/osd/deviceSet.go +++ b/pkg/operator/ceph/cluster/osd/deviceSet.go @@ -29,6 +29,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -109,7 +110,7 @@ func (c *Cluster) prepareStorageClassDeviceSets(errs *provisionErrors) { highestExistingID := -1 countInDeviceSet := 0 if existingIDs, ok := uniqueOSDsPerDeviceSet[deviceSet.Name]; ok { - logger.Infof("verifying PVCs exist for %d OSDs in device set %q", existingIDs.Len(), deviceSet.Name) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "verifying PVCs exist for %d OSDs in device set %q", existingIDs.Len(), deviceSet.Name) for existingID := range existingIDs { pvcID, err := strconv.Atoi(existingID) if err != nil { @@ -129,7 +130,7 @@ func (c *Cluster) prepareStorageClassDeviceSets(errs *provisionErrors) { // No new PVCs will be created if we have too many pvcsToCreate := deviceSet.Count - countInDeviceSet if pvcsToCreate > 0 { - logger.Infof("creating %d new PVCs for device set %q", pvcsToCreate, deviceSet.Name) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "creating %d new PVCs for device set %q", pvcsToCreate, deviceSet.Name) } for i := 0; i < pvcsToCreate; i++ { pvcID := highestExistingID + i + 1 @@ -234,12 +235,12 @@ func (c *Cluster) createDeviceSetPVC(existingPVCs map[string]*v1.PersistentVolum } if existingPVC != nil { - logger.Infof("OSD PVC %q already exists", existingPVC.Name) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD PVC %q already exists", existingPVC.Name) desiredPvcSize, desiredOK := pvc.Spec.Resources.Requests[v1.ResourceStorage] actualPvcSize, currentOK := existingPVC.Spec.Resources.Requests[v1.ResourceStorage] if !desiredOK || !currentOK { - logger.Debugf("desired or current size are not specified for PVC %q", existingPVC.Name) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "desired or current size are not specified for PVC %q", existingPVC.Name) return existingPVC, nil } @@ -260,7 +261,7 @@ func (c *Cluster) createDeviceSetPVC(existingPVCs map[string]*v1.PersistentVolum if err != nil { return nil, errors.Wrapf(err, "failed to create PVC %q for device set %q", pvc.Name, deviceSetName) } - logger.Infof("successfully provisioned PVC %q", deployedPVC.Name) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "successfully provisioned PVC %q", deployedPVC.Name) return deployedPVC, nil } @@ -345,13 +346,13 @@ func waitForPvcToExpandWithTimeout(ctx context.Context, client client.Client, pv timeout := currentTime.Add(pvcResizeWaitTimeOut) for { if time.Now().UTC().After(timeout) { - logger.Warning("timed out waiting for PVC size to be updated, osds may need to be restarted manually after resize") + log.NamespacedWarning(namespace, logger, "timed out waiting for PVC size to be updated, osds may need to be restarted manually after resize") return } allPvcResized := checkAllPvcResize(ctx, client, namespace, pvcResizeMap) if allPvcResized { - logger.Info("all osd pvcs resized") + log.NamespacedInfo(namespace, logger, "all osd pvcs resized") return } @@ -373,7 +374,7 @@ func checkAllPvcResize(ctx context.Context, client client.Client, namespace stri currentPVC := &v1.PersistentVolumeClaim{} err := client.Get(ctx, types.NamespacedName{Name: pvcName, Namespace: namespace}, currentPVC) if err != nil { - logger.Errorf("failed to get PVC %q. %v", currentPVC.Name, err) + log.NamespacedError(namespace, logger, "failed to get PVC %q. %v", currentPVC.Name, err) continue } @@ -385,7 +386,7 @@ func checkAllPvcResize(ctx context.Context, client client.Client, namespace stri if currentPVC.Status.Capacity != nil { if currentPVC.Status.Capacity[v1.ResourceStorage] == pvcSize { - logger.Infof("PVC %q size is updated to %s", currentPVC.Name, pvcSize.String()) + log.NamespacedInfo(namespace, logger, "PVC %q size is updated to %s", currentPVC.Name, pvcSize.String()) numberOfPvcResized++ pvcResizeMap[pvcName] = pvcResize{ desiredSize: pvc.desiredSize, @@ -393,11 +394,11 @@ func checkAllPvcResize(ctx context.Context, client client.Client, namespace stri resizeConfirmed: true, } } else { - logger.Debugf("PVC %q size is not updated to %v", currentPVC.Name, pvcSize.String()) + log.NamespacedDebug(namespace, logger, "PVC %q size is not updated to %v", currentPVC.Name, pvcSize.String()) } } } - logger.Infof("%d of osd pvcs resized and %d remain to be resized", numberOfPvcResized, len(pvcResizeMap)-numberOfPvcResized) + log.NamespacedInfo(namespace, logger, "%d of osd pvcs resized and %d remain to be resized", numberOfPvcResized, len(pvcResizeMap)-numberOfPvcResized) return numberOfPvcResized == len(pvcResizeMap) } diff --git a/pkg/operator/ceph/cluster/osd/health.go b/pkg/operator/ceph/cluster/osd/health.go index e769f0755..88e15428a 100644 --- a/pkg/operator/ceph/cluster/osd/health.go +++ b/pkg/operator/ceph/cluster/osd/health.go @@ -27,6 +27,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" kerrors "k8s.io/apimachinery/pkg/api/errors" ) @@ -58,7 +59,7 @@ func NewOSDHealthMonitor(context *clusterd.Context, clusterInfo *client.ClusterI // allow overriding the check interval checkInterval := healthCheck.DaemonHealth.ObjectStorageDaemon.Interval if checkInterval != nil { - logger.Infof("ceph osd status in namespace %q check interval %q", h.clusterInfo.Namespace, checkInterval.Duration.String()) + log.NamespacedInfo(h.clusterInfo.Namespace, logger, "ceph osd status in namespace %q check interval %q", h.clusterInfo.Namespace, checkInterval.Duration.String()) h.interval = &checkInterval.Duration } @@ -72,17 +73,17 @@ func (m *OSDHealthMonitor) Start(monitoringRoutines *sync.Map, daemon string) { // we will get an invalid pointer error and the go routine will panic v, ok := monitoringRoutines.Load(daemon) if !ok { - logger.Infof("ceph cluster %q has been deleted. stopping monitoring of OSDs", m.clusterInfo.Namespace) + log.NamespacedInfo(m.clusterInfo.Namespace, logger, "ceph cluster %q has been deleted. stopping monitoring of OSDs", m.clusterInfo.Namespace) return } health := v.(*opcontroller.ClusterHealth) select { case <-time.After(*m.interval): - logger.Debug("checking osd processes status.") + log.NamespacedDebug(m.clusterInfo.Namespace, logger, "checking osd processes status.") m.checkOSDHealth() case <-health.InternalCtx.Done(): - logger.Infof("stopping monitoring of OSDs in namespace %q", m.clusterInfo.Namespace) + log.NamespacedInfo(m.clusterInfo.Namespace, logger, "stopping monitoring of OSDs in namespace %q", m.clusterInfo.Namespace) monitoringRoutines.Delete(daemon) return } @@ -98,7 +99,7 @@ func (m *OSDHealthMonitor) Update(removeOSDsIfOUTAndSafeToRemove bool) { func (m *OSDHealthMonitor) checkOSDHealth() { err := m.checkOSDDump() if err != nil { - logger.Debugf("failed to check OSD Dump. %v", err) + log.NamespacedDebug(m.clusterInfo.Namespace, logger, "failed to check OSD Dump. %v", err) } } @@ -115,7 +116,7 @@ func (m *OSDHealthMonitor) checkOSDDump() error { } id := int(id64) - logger.Debugf("validating status of osd.%d", id) + log.NamespacedDebug(m.clusterInfo.Namespace, logger, "validating status of osd.%d", id) status, in, err := osdDump.StatusByID(int64(id)) if err != nil { @@ -123,17 +124,17 @@ func (m *OSDHealthMonitor) checkOSDDump() error { } if status == upStatus { - logger.Debugf("osd.%d is healthy.", id) + log.NamespacedDebug(m.clusterInfo.Namespace, logger, "osd.%d is healthy.", id) continue } - logger.Debugf("osd.%d is marked 'DOWN'", id) + log.NamespacedDebug(m.clusterInfo.Namespace, logger, "osd.%d is marked 'DOWN'", id) if in != inStatus { - logger.Debugf("osd.%d is marked 'OUT'", id) + log.NamespacedDebug(m.clusterInfo.Namespace, logger, "osd.%d is marked 'OUT'", id) if m.removeOSDsIfOUTAndSafeToRemove { if err := m.removeOSDDeploymentIfSafeToDestroy(id); err != nil { - logger.Errorf("error handling marked out osd osd.%d. %v", id, err) + log.NamespacedError(m.clusterInfo.Namespace, logger, "error handling marked out osd osd.%d. %v", id, err) } } } @@ -162,7 +163,7 @@ func (m *OSDHealthMonitor) removeOSDDeploymentIfSafeToDestroy(outOSDid int) erro podDeletionTimeStamp := podCreationTimestamp.Add(graceTime) currentTime := time.Now().UTC() if podDeletionTimeStamp.Before(currentTime) { - logger.Infof("osd.%d is 'safe-to-destroy'. removing the osd deployment.", outOSDid) + log.NamespacedInfo(m.clusterInfo.Namespace, logger, "osd.%d is 'safe-to-destroy'. removing the osd deployment.", outOSDid) if err := k8sutil.DeleteDeployment(m.clusterInfo.Context, m.context.Clientset, dp.Items[0].Namespace, dp.Items[0].Name); err != nil { return errors.Wrapf(err, "failed to delete osd deployment %s", dp.Items[0].Name) } diff --git a/pkg/operator/ceph/cluster/osd/key_rotation.go b/pkg/operator/ceph/cluster/osd/key_rotation.go index 07d84d7f1..c957e2bf8 100644 --- a/pkg/operator/ceph/cluster/osd/key_rotation.go +++ b/pkg/operator/ceph/cluster/osd/key_rotation.go @@ -25,6 +25,7 @@ import ( kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms" "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" batch "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -247,7 +248,7 @@ func (c *Cluster) reconcileKeyRotationCronJob() error { } return nil } - logger.Debugf("successfully deleted key rotation cron jobs") + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "successfully deleted key rotation cron jobs") return nil } @@ -259,7 +260,7 @@ func (c *Cluster) reconcileKeyRotationCronJob() error { return errors.Wrap(err, "failed to query existing OSD deployments") } - logger.Debugf("found %d osd deployments", len(deployments.Items)) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "found %d osd deployments", len(deployments.Items)) for i := range deployments.Items { osdDep := deployments.Items[i] osd, err := c.getOSDInfo(&osdDep) @@ -279,7 +280,7 @@ func (c *Cluster) reconcileKeyRotationCronJob() error { continue } - logger.Infof("starting OSD key rotation cron job for osd %d", osd.ID) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "starting OSD key rotation cron job for osd %d", osd.ID) cj, err := c.makeKeyRotationCronJob(pvcName, osd, osdProps) if err != nil { return errors.Wrap(err, "failed to make key rotation cron job") @@ -294,9 +295,9 @@ func (c *Cluster) reconcileKeyRotationCronJob() error { if err != nil { return errors.Wrapf(err, "failed to create or update key rotation cron job %q", cj.Name) } - logger.Infof("started OSD key rotation cron job %q", cj.Name) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "started OSD key rotation cron job %q", cj.Name) } - logger.Infof("successfully started OSD key rotation cron jobs") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "successfully started OSD key rotation cron jobs") return nil } diff --git a/pkg/operator/ceph/cluster/osd/migrate.go b/pkg/operator/ceph/cluster/osd/migrate.go index 555df56a8..c7c0eb83e 100644 --- a/pkg/operator/ceph/cluster/osd/migrate.go +++ b/pkg/operator/ceph/cluster/osd/migrate.go @@ -26,6 +26,7 @@ import ( "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -94,7 +95,7 @@ func (m *migrationConfig) migrateForEncryption(c *Cluster, osdDeployments *appsv if err != nil { return errors.Wrapf(err, "failed to details about the OSD %q", osdDeployments.Items[i].Name) } - logger.Infof("migration is required for OSD.%d due to change in encryption settings from %t to %t in storageClassDeviceSet %q", osdInfo.ID, actualEncryptedSetting, requestedEncryptionSetting, osdDeviceSetName) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "migration is required for OSD.%d due to change in encryption settings from %t to %t in storageClassDeviceSet %q", osdInfo.ID, actualEncryptedSetting, requestedEncryptionSetting, osdDeviceSetName) if _, exists := m.osds[osdInfo.ID]; !exists { m.osds[osdInfo.ID] = &osdInfo } @@ -113,7 +114,7 @@ func (m *migrationConfig) migrateForOSDStore(c *Cluster, osdDeployments *appsv1. if err != nil { return errors.Wrapf(err, "failed to details about the OSD %q", osdDeployments.Items[i].Name) } - logger.Infof("migration is required for OSD.%d to update storeType from %q to %q", osdInfo.ID, osdStore, desiredOSDStore) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "migration is required for OSD.%d to update storeType from %q to %q", osdInfo.ID, osdStore, desiredOSDStore) if _, exists := m.osds[osdInfo.ID]; !exists { m.osds[osdInfo.ID] = &osdInfo } @@ -185,12 +186,12 @@ func isLastOSDMigrationComplete(c *Cluster) (bool, error) { _, err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Get(c.clusterInfo.Context, deploymentName, metav1.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { - logger.Infof("deployment for the last migrated OSD with ID - %d is not found.", migratedOSDId) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "deployment for the last migrated OSD with ID - %d is not found.", migratedOSDId) return false, nil } } - logger.Infof("migration of OSD.%d was successful", migratedOSDId) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "migration of OSD.%d was successful", migratedOSDId) return true, nil } @@ -205,7 +206,7 @@ func getLastMigratedOSDId(context *clusterd.Context, clusterInfo *cephclient.Clu osdID, ok := cm.Data[OSDIdKey] if !ok || osdID == "" { - logger.Debugf("empty config map %q", OSDMigrationConfigName) + log.NamespacedDebug(clusterInfo.Namespace, logger, "empty config map %q", OSDMigrationConfigName) return -1, nil } diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 583fd3f2f..c155dc600 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -38,6 +38,7 @@ import ( cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util" + "github.com/rook/rook/pkg/util/log" "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" @@ -206,47 +207,47 @@ func (c *Cluster) validateOSDSettings() error { } func (c *Cluster) validateTopologyAcrossNodes() error { - logger.Info("Starting validateTopologyAcrossNodes()") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Starting validateTopologyAcrossNodes()") if os.Getenv("ROOK_SKIP_OSD_TOPOLOGY_CHECK") == "true" { - logger.Debugf("Skipping topology validation due to ROOK_SKIP_OSD_TOPOLOGY_CHECK=true") + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "Skipping topology validation due to ROOK_SKIP_OSD_TOPOLOGY_CHECK=true") return nil } if topologyValidated { - logger.Debug("Skipping topology validation because it was already validated") + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "Skipping topology validation because it was already validated") return nil } - logger.Info("Validating node topology across all cluster nodes") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Validating node topology across all cluster nodes") nodelist, err := c.context.Clientset.CoreV1().Nodes().List(c.clusterInfo.Context, metav1.ListOptions{}) if err != nil { - logger.Errorf("Failed to list nodes for topology validation: %v", err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "Failed to list nodes for topology validation: %v", err) return errors.Wrap(err, "failed to list nodes for topology validation") } - logger.Infof("Fetched nodelist with %d nodes", len(nodelist.Items)) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Fetched nodelist with %d nodes", len(nodelist.Items)) if err := topology.CheckTopologyConflicts(&nodelist.Items); err != nil { - logger.Infof("Topology conflict detected: %v", err) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Topology conflict detected: %v", err) // Check if there are any existing OSDs osdRunning, errPods := k8sutil.PodsRunningWithLabel(c.clusterInfo.Context, c.context.Clientset, c.clusterInfo.Namespace, "app=rook-ceph-osd") if errPods != nil { - logger.Errorf("Failed to list OSD pods: %v", errPods) + log.NamespacedError(c.clusterInfo.Namespace, logger, "Failed to list OSD pods: %v", errPods) return errPods // fail safe } - logger.Infof("Found %d existing OSDs", osdRunning) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Found %d existing OSDs", osdRunning) if osdRunning == 0 { - logger.Errorf("Topology conflict detected in new cluster: %v", err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "Topology conflict detected in new cluster: %v", err) return errors.Wrap(err, "topology conflict detected in new cluster") } // Existing OSDs found — log and continue - logger.Warningf("Topology conflict detected, but skipping failure since OSDs already exist. %v", err) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "Topology conflict detected, but skipping failure since OSDs already exist. %v", err) return nil } - logger.Info("Node topology validation passed without conflicts") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Node topology validation passed without conflicts") topologyValidated = true - logger.Info("validateTopologyAcrossNodes() completed successfully") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "validateTopologyAcrossNodes() completed successfully") return nil } @@ -265,10 +266,10 @@ func (c *Cluster) Start() error { if err := c.initializeNodeConfigmaps(); err != nil { return err } - logger.Infof("start running osds in namespace %q", namespace) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "start running osds in namespace %q", namespace) if !c.spec.Storage.UseAllNodes && len(c.spec.Storage.Nodes) == 0 && len(c.spec.Storage.StorageClassDeviceSets) == 0 { - logger.Warningf("useAllNodes is set to false and no nodes, storageClassDevicesets or volumeSources are specified, no OSD pods are going to be created") + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "useAllNodes is set to false and no nodes, storageClassDevicesets or volumeSources are specified, no OSD pods are going to be created") } if c.spec.WaitTimeoutForHealthyOSDInMinutes != 0 { @@ -276,11 +277,11 @@ func (c *Cluster) Start() error { } else { c.clusterInfo.OsdUpgradeTimeout = defaultWaitTimeoutForHealthyOSD } - logger.Infof("wait timeout for healthy OSDs during upgrade or restart is %q", c.clusterInfo.OsdUpgradeTimeout) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "wait timeout for healthy OSDs during upgrade or restart is %q", c.clusterInfo.OsdUpgradeTimeout) osdsToSkipReconcile, err := controller.GetDaemonsToSkipReconcile(c.clusterInfo.Context, c.context, c.clusterInfo.Namespace, OsdIdLabelKey, AppName) if err != nil { - logger.Warningf("failed to get osds to skip reconcile. %v", err) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to get osds to skip reconcile. %v", err) } migrationConfig, err := c.startOSDMigration() @@ -301,20 +302,20 @@ func (c *Cluster) Start() error { } } - logger.Debugf("%d of %d OSD Deployments need update", updateQueue.Len(), deployments.Len()) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "%d of %d OSD Deployments need update", updateQueue.Len(), deployments.Len()) updateConfig := c.newUpdateConfig(config, updateQueue, deployments, osdsToSkipReconcile) // prepare for creating new OSDs statusConfigMaps := sets.New[string]() - logger.Info("start provisioning the OSDs on PVCs, if needed") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "start provisioning the OSDs on PVCs, if needed") pvcConfigMaps, err := c.startProvisioningOverPVCs(config, errs) if err != nil { return err } statusConfigMaps = statusConfigMaps.Union(pvcConfigMaps) - logger.Info("start provisioning the OSDs on nodes, if needed") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "start provisioning the OSDs on nodes, if needed") nodeConfigMaps, err := c.startProvisioningOverNodes(config, errs) if err != nil { return err @@ -360,26 +361,26 @@ func (c *Cluster) Start() error { // remove the osdBootstrapKey as the osds are running c.deleteOsdBootstrapKeyring() - logger.Infof("finished running OSDs in namespace %q", namespace) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "finished running OSDs in namespace %q", namespace) return nil } func (c *Cluster) deleteOsdBootstrapKeyring() { err := cephclient.AuthDelete(c.context, c.clusterInfo, "client.bootstrap-osd") if err != nil { - logger.Debugf("failed to delete client.bootstrap-osd bootstrap key in the namespace %q. Error: %v", c.clusterInfo.Namespace, err) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "failed to delete client.bootstrap-osd bootstrap key in the namespace %q. Error: %v", c.clusterInfo.Namespace, err) } else { - logger.Debugf("successfully deleted client.bootstrap-osd bootstrap key in the namespace %q", c.clusterInfo.Namespace) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "successfully deleted client.bootstrap-osd bootstrap key in the namespace %q", c.clusterInfo.Namespace) } } func (c *Cluster) startOSDMigration() (*migrationConfig, error) { if !c.isMigrationRequested() { - logger.Debug("no OSD migration is requested") + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "no OSD migration is requested") return nil, nil } - logger.Info("osd migration is requested") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "osd migration is requested") // start migration only if PGs are active+clean pgsHealhty, err := c.waitForHealthyPGs() @@ -409,7 +410,7 @@ func (c *Cluster) startOSDMigration() (*migrationConfig, error) { // delete deployment of the osd that needs migration if migrationConfig != nil && len(migrationConfig.osds) > 0 { osdToMigrate := migrationConfig.getOSDToMigrate() - logger.Infof("deleting OSD.%d deployment for migration ", osdToMigrate.ID) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "deleting OSD.%d deployment for migration ", osdToMigrate.ID) err = c.deleteOSDDeployment(osdToMigrate.ID) if err != nil { return nil, errors.Wrapf(err, "failed to delete deployment for osd.%d that needs migration %q", osdToMigrate.ID, c.clusterInfo.Namespace) @@ -437,13 +438,13 @@ func (c *Cluster) postReconcileUpdateOSDProperties(desiredOSDs map[int]*OSDInfo) if err != nil { return errors.Wrap(err, "failed to get osd usage") } - logger.Debugf("post processing osd properties with %d actual osds from ceph osd df and %d existing osds found during reconcile", len(osdUsage.OSDNodes), len(desiredOSDs)) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "post processing osd properties with %d actual osds from ceph osd df and %d existing osds found during reconcile", len(osdUsage.OSDNodes), len(desiredOSDs)) for _, actualOSD := range osdUsage.OSDNodes { if c.spec.Storage.AllowOsdCrushWeightUpdate { _, err := cephclient.ResizeOsdCrushWeight(actualOSD, c.context, c.clusterInfo) if err != nil { // Log the error and allow other updates to continue - logger.Errorf("failed to resize osd crush weight on cluster in namespace %s: %v", c.clusterInfo.Namespace, err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to resize osd crush weight on cluster in namespace %s: %v", c.clusterInfo.Namespace, err) } } @@ -453,7 +454,7 @@ func (c *Cluster) postReconcileUpdateOSDProperties(desiredOSDs map[int]*OSDInfo) } if err := c.updateDeviceClassIfChanged(actualOSD.ID, desiredOSD.DeviceClass, actualOSD.DeviceClass); err != nil { // Log the error and allow other updates to continue - logger.Errorf("failed to update device class on cluster in namespace %s: %v", c.clusterInfo.Namespace, err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to update device class on cluster in namespace %s: %v", c.clusterInfo.Namespace, err) } } @@ -466,14 +467,14 @@ func (c *Cluster) updateDeviceClassIfChanged(osdID int, desiredDeviceClass, actu return nil } if desiredDeviceClass != "" && desiredDeviceClass != actualDeviceClass { - logger.Infof("updating osd.%d device class from %q to %q", osdID, actualDeviceClass, desiredDeviceClass) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "updating osd.%d device class from %q to %q", osdID, actualDeviceClass, desiredDeviceClass) err := cephclient.SetDeviceClass(c.context, c.clusterInfo, osdID, desiredDeviceClass) if err != nil { return errors.Wrapf(err, "failed to set device class on osd %d", osdID) } return nil } - logger.Debugf("no device class change needed for osd.%d. desired=%q, actual=%q", osdID, desiredDeviceClass, actualDeviceClass) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "no device class change needed for osd.%d. desired=%q, actual=%q", osdID, desiredDeviceClass, actualDeviceClass) return nil } @@ -584,21 +585,21 @@ func (c *Cluster) getOSDPropsForPVC(pvcName string) (osdProperties, error) { // The data PVC template is required. dataSource, dataOK := deviceSet.PVCSources[bluestorePVCData] if !dataOK { - logger.Warningf("failed to find data source daemon for device set %q, missing the data template", deviceSet.Name) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to find data source daemon for device set %q, missing the data template", deviceSet.Name) continue } if pvcName == dataSource.ClaimName { metadataSource, metadataOK := deviceSet.PVCSources[bluestorePVCMetadata] if metadataOK { - logger.Infof("OSD will have its main bluestore block on %q and its metadata device on %q", dataSource.ClaimName, metadataSource.ClaimName) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its main bluestore block on %q and its metadata device on %q", dataSource.ClaimName, metadataSource.ClaimName) } else { - logger.Infof("OSD will have its main bluestore block on %q", dataSource.ClaimName) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its main bluestore block on %q", dataSource.ClaimName) } walSource, walOK := deviceSet.PVCSources[bluestorePVCWal] if walOK { - logger.Infof("OSD will have its wal device on %q", walSource.ClaimName) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its wal device on %q", walSource.ClaimName) } if deviceSet.Resources.Limits == nil && deviceSet.Resources.Requests == nil { @@ -670,7 +671,7 @@ func (c *Cluster) getPVCHostName(pvcName string) (string, error) { for _, pod := range pods.Items { name, err := k8sutil.GetNodeHostName(c.clusterInfo.Context, c.context.Clientset, pod.Spec.NodeName) if err != nil { - logger.Warningf("falling back to node name %s since hostname not found for node", pod.Spec.NodeName) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "falling back to node name %s since hostname not found for node", pod.Spec.NodeName) name = pod.Spec.NodeName } if name == "" { @@ -754,7 +755,7 @@ func (c *Cluster) getOSDInfo(d *appsv1.Deployment) (OSDInfo, error) { // If CVMode is empty, this likely means we upgraded Rook // This property did not exist before so we need to initialize it if osd.CVMode == "" { - logger.Infof("required CVMode for OSD %d was not found. assuming this is an LVM OSD", osd.ID) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "required CVMode for OSD %d was not found. assuming this is an LVM OSD", osd.ID) osd.CVMode = "lvm" } @@ -762,7 +763,7 @@ func (c *Cluster) getOSDInfo(d *appsv1.Deployment) (OSDInfo, error) { if isPVC && osd.TopologyAffinity == "" { osd.TopologyAffinity, err = getTopologyFromNode(c.clusterInfo.Context, c.context.Clientset, d, osd) if err != nil { - logger.Errorf("failed to get topology affinity for osd %d. %v", osd.ID, err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get topology affinity for osd %d. %v", osd.ID, err) } } @@ -772,7 +773,7 @@ func (c *Cluster) getOSDInfo(d *appsv1.Deployment) (OSDInfo, error) { if !locationFound { location, _, err := getLocationFromPod(c.clusterInfo.Context, c.context.Clientset, d, cephclient.GetCrushRootFromSpec(&c.spec)) if err != nil { - logger.Errorf("failed to get location. %v", err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get location. %v", err) } else { osd.Location = location } @@ -884,7 +885,7 @@ func getTopologyFromNode(ctx context.Context, clientset kubernetes.Interface, d // osd is not portable, no need to load the topology affinity return "", nil } - logger.Infof("detecting topology affinity for osd %d after upgrade", osd.ID) + log.NamespacedInfo(d.Namespace, logger, "detecting topology affinity for osd %d after upgrade", osd.ID) // Get the osd pod and its assigned node, then look up the node labels pods, err := clientset.CoreV1().Pods(d.Namespace).List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", OsdIdLabelKey, d.Labels[OsdIdLabelKey])}) @@ -903,7 +904,7 @@ func getTopologyFromNode(ctx context.Context, clientset kubernetes.Interface, d return "", errors.Wrap(err, "failed to get the node for topology affinity") } _, topologyAffinity := topology.ExtractOSDTopologyFromLabels(node.Labels) - logger.Infof("found osd %d topology affinity at %q", osd.ID, topologyAffinity) + log.NamespacedInfo(d.Namespace, logger, "found osd %d topology affinity at %q", osd.ID, topologyAffinity) return topologyAffinity, nil } @@ -984,7 +985,7 @@ func (c *Cluster) applyUpgradeOSDFunctionality() { // Get all the daemons versions versions, err := cephclient.GetAllCephDaemonVersions(c.context, c.clusterInfo) if err != nil { - logger.Warningf("failed to get ceph daemons versions; this likely means there are no osds yet. %v", err) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to get ceph daemons versions; this likely means there are no osds yet. %v", err) return } @@ -997,7 +998,7 @@ func (c *Cluster) applyUpgradeOSDFunctionality() { for v := range versions.Osd { osdVersion, err = cephver.ExtractCephVersion(v) if err != nil { - logger.Warningf("failed to extract ceph version. %v", err) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to extract ceph version. %v", err) return } // Ensure the required version of OSDs is set to the current consistent version, @@ -1005,7 +1006,7 @@ func (c *Cluster) applyUpgradeOSDFunctionality() { // previous major ceph version. err = cephclient.EnableReleaseOSDFunctionality(c.context, c.clusterInfo, osdVersion.ReleaseName()) if err != nil { - logger.Warningf("failed to enable new osd functionality. %v", err) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to enable new osd functionality. %v", err) return } } @@ -1016,12 +1017,12 @@ func (c *Cluster) applyUpgradeOSDFunctionality() { func (c *Cluster) deleteOSDDeployment(osdID int) error { // Delete the OSD deployment deploymentName := fmt.Sprintf("rook-ceph-osd-%d", osdID) - logger.Infof("removing the OSD deployment %q", deploymentName) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "removing the OSD deployment %q", deploymentName) if err := k8sutil.DeleteDeployment(c.clusterInfo.Context, c.context.Clientset, c.clusterInfo.Namespace, deploymentName); err != nil { if !kerrors.IsNotFound(err) { return errors.Wrapf(err, "failed to delete OSD deployment %q.", deploymentName) } - logger.Debugf("osd deployment %q not found. Ignoring since object must be deleted.", deploymentName) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "osd deployment %q not found. Ignoring since object must be deleted.", deploymentName) } return nil } @@ -1035,7 +1036,7 @@ func (c *Cluster) waitForHealthyPGs() (bool, error) { if pgClean { return true, nil } - logger.Infof("waiting for PGs to be healthy. PG status: %q", pgHealthMsg) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "waiting for PGs to be healthy. PG status: %q", pgHealthMsg) return false, nil } @@ -1083,7 +1084,7 @@ func (c *Cluster) updateCephOsdStorageStatus() error { err := c.context.Client.Get(c.clusterInfo.Context, c.clusterInfo.NamespacedName(), &cephCluster) if err != nil { if kerrors.IsNotFound(err) { - logger.Debug("CephCluster resource not found. Ignoring since object must be deleted.") + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "CephCluster resource not found. Ignoring since object must be deleted.") return nil } return errors.Wrapf(err, "failed to retrieve ceph cluster %q to update ceph Storage", c.clusterInfo.NamespacedName().Name) @@ -1177,7 +1178,7 @@ func (c *Cluster) initializeNodeConfigmaps() error { for _, configmap := range configmaps.Items { if strings.HasPrefix(configmap.Name, prefix) { nodeName := strings.TrimPrefix(configmap.Name, prefix) - logger.Infof("found node %q configmap, will mount it for alternate ceph conf overrides for osds on this node", nodeName) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "found node %q configmap, will mount it for alternate ceph conf overrides for osds on this node", nodeName) c.nodeConfigmaps[nodeName] = struct{}{} } } diff --git a/pkg/operator/ceph/cluster/osd/provision_spec.go b/pkg/operator/ceph/cluster/osd/provision_spec.go index 2c5e2bfbf..1087c8201 100644 --- a/pkg/operator/ceph/cluster/osd/provision_spec.go +++ b/pkg/operator/ceph/cluster/osd/provision_spec.go @@ -31,6 +31,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" batch "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -365,13 +366,13 @@ func (c *Cluster) deleteAllOrphanedPrepareJobs() { } jobs, err := c.context.Clientset.BatchV1().Jobs(c.clusterInfo.Namespace).List(c.clusterInfo.Context, listOpts) if err != nil { - logger.Warningf("failed to clean up any orphaned OSD prepare jobs. failed to list OSD prepare jobs. %v", err) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to clean up any orphaned OSD prepare jobs. failed to list OSD prepare jobs. %v", err) return } nodes, err := c.context.Clientset.CoreV1().Nodes().List(c.clusterInfo.Context, metav1.ListOptions{}) if err != nil { - logger.Warningf("failed to clean up any orphaned OSD prepare jobs. failed to list nodes. %v", err) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to clean up any orphaned OSD prepare jobs. failed to list nodes. %v", err) return } @@ -387,9 +388,9 @@ func (c *Cluster) deleteAllOrphanedPrepareJobs() { nodeSelector := job.Spec.Template.Spec.NodeSelector if jobHostName, ok := nodeSelector[k8sutil.LabelHostname()]; ok { if !hostNames.Has(jobHostName) { - logger.Infof("cleaning up orphaned OSD prepare job %q.", job.Name) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "cleaning up orphaned OSD prepare job %q.", job.Name) if err := c.context.Clientset.BatchV1().Jobs(c.clusterInfo.Namespace).Delete(c.clusterInfo.Context, job.Name, metav1.DeleteOptions{}); err != nil { - logger.Warningf("failed to clean up OSD prepare job %q. %v", job.Name, err) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to clean up OSD prepare job %q. %v", job.Name, err) } } } diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 10153c6c2..64e86cdeb 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -31,6 +31,7 @@ import ( cephkey "github.com/rook/rook/pkg/operator/ceph/config/keyring" "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -312,14 +313,14 @@ func deploymentName(osdID int) string { func (c *Cluster) updateCephConfigVolume(volumes []v1.Volume, nodeName string) []v1.Volume { if _, ok := c.nodeConfigmaps[nodeName]; !ok { - logger.Debugf("no configmap to override for node %q", nodeName) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "no configmap to override for node %q", nodeName) return volumes } updatedVolumes := []v1.Volume{} for _, volume := range volumes { if volume.Name == k8sutil.ConfigOverrideName { - logger.Debugf("found configmap volume to override for node %q", nodeName) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "found configmap volume to override for node %q", nodeName) name := fmt.Sprintf("%s-%s", k8sutil.ConfigOverrideName, nodeName) volume.VolumeSource.Projected.Sources[0].ConfigMap.LocalObjectReference.Name = name } @@ -351,12 +352,12 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provision } if c.spec.Storage.AllowDeviceClassUpdate && osdProps.storeConfig.DeviceClass != "" && osdProps.storeConfig.DeviceClass != osd.DeviceClass { - logger.Infof("The device class for osd %d is changing from %q to %q", osd.ID, osd.DeviceClass, osdProps.storeConfig.DeviceClass) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "The device class for osd %d is changing from %q to %q", osd.ID, osd.DeviceClass, osdProps.storeConfig.DeviceClass) osd.DeviceClass = osdProps.storeConfig.DeviceClass } // Assign the resources specific to this device class if resources, ok := cephv1.GetOSDResourcesForDeviceClass(c.spec.Resources, osd.DeviceClass); ok { - logger.Debugf("assigning resources for device class %q to osd %d: %+v", osd.DeviceClass, osd.ID, resources) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "assigning resources for device class %q to osd %d: %+v", osd.DeviceClass, osd.ID, resources) osdProps.resources = resources } @@ -560,7 +561,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provision return nil, errors.Wrapf(err, "failed to export service %q", osdService.Name) } - logger.Infof("osd.%d exported IP is %q", osd.ID, exportedIP) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "osd.%d exported IP is %q", osd.ID, exportedIP) args = append(args, []string{ fmt.Sprintf("--public-addr=%s", exportedIP), @@ -793,7 +794,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provision } if osdProps.portable { // portable OSDs must have affinity to the topology where the osd prepare job was executed - if err := applyTopologyAffinity(&deployment.Spec.Template.Spec, *osd); err != nil { + if err := c.applyTopologyAffinity(&deployment.Spec.Template.Spec, *osd); err != nil { return nil, err } } else { @@ -848,7 +849,7 @@ func (c *Cluster) createOSDService(osd OSDInfo, labels map[string]string) (*v1.S return nil, err } - logger.Infof("osd.%d cluster IP is %q", osd.ID, svc.Spec.ClusterIP) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "osd.%d cluster IP is %q", osd.ID, svc.Spec.ClusterIP) return svc, nil } @@ -869,12 +870,12 @@ func (c *Cluster) applyAllPlacementIfNeeded(d *v1.PodSpec) { } } -func applyTopologyAffinity(spec *v1.PodSpec, osd OSDInfo) error { +func (c *Cluster) applyTopologyAffinity(spec *v1.PodSpec, osd OSDInfo) error { if osd.TopologyAffinity == "" { - logger.Debugf("no topology affinity to set for osd %d", osd.ID) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "no topology affinity to set for osd %d", osd.ID) return nil } - logger.Infof("assigning osd %d topology affinity to %q", osd.ID, osd.TopologyAffinity) + log.NamespacedInfo(osd.Cluster, logger, "assigning osd %d topology affinity to %q", osd.ID, osd.TopologyAffinity) nodeAffinity, err := k8sutil.GenerateNodeAffinity(osd.TopologyAffinity) if err != nil { return errors.Wrapf(err, "failed to generate osd %d topology affinity", osd.ID) diff --git a/pkg/operator/ceph/cluster/osd/status.go b/pkg/operator/ceph/cluster/osd/status.go index d4c0b6437..d88a39d6a 100644 --- a/pkg/operator/ceph/cluster/osd/status.go +++ b/pkg/operator/ceph/cluster/osd/status.go @@ -26,6 +26,7 @@ import ( "github.com/pkg/errors" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -176,7 +177,7 @@ func (c *Cluster) updateAndCreateOSDs( if !doLoop { return err } - logger.Errorf("%v", err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "%v", err) } } @@ -248,20 +249,20 @@ func (c *Cluster) updateAndCreateOSDsLoop( select { case event, ok := <-watcher.ResultChan(): if !ok { - logger.Infof("restarting watcher for OSD provisioning status ConfigMaps. the watcher closed the channel") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "restarting watcher for OSD provisioning status ConfigMaps. the watcher closed the channel") return true, nil } if !isAddOrModifyEvent(event.Type) { // We don't want to process delete events when we delete configmaps after having // processed them. We also don't want to process BOOKMARK or ERROR events. - logger.Debugf("not processing %s event for object %q", event.Type, eventObjectName(event)) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "not processing %s event for object %q", event.Type, eventObjectName(event)) break // case } configMap, ok := event.Object.(*corev1.ConfigMap) if !ok { - logger.Errorf("recovering. %s. expected type ConfigMap but found %T", watchErrMsg, configMap) + log.NamespacedError(c.clusterInfo.Namespace, logger, "recovering. %s. expected type ConfigMap but found %T", watchErrMsg, configMap) break // case } @@ -275,11 +276,11 @@ func (c *Cluster) updateAndCreateOSDsLoop( // Log progress c, cExp := createConfig.progress() u, uExp := updateConfig.progress() - logger.Infof("waiting... %d of %d OSD prepare jobs have finished processing and %d of %d OSDs have been updated", c, cExp, u, uExp) + log.NamespacedInfo(namespace, logger, "waiting... %d of %d OSD prepare jobs have finished processing and %d of %d OSDs have been updated", c, cExp, u, uExp) // keep track of each time progress is made if c != prevCreatedCount || u != prevUpdatedCount { - logger.Debugf("cluster in namespace %q, progress was made processing OSDs: %d nodes/OSDs were processed for creation and %d updated", namespace, c-prevCreatedCount, u-prevUpdatedCount) + log.NamespacedDebug(namespace, logger, "cluster in namespace %q, progress was made processing OSDs: %d nodes/OSDs were processed for creation and %d updated", namespace, c-prevCreatedCount, u-prevUpdatedCount) prevCreatedCount = c prevUpdatedCount = u prevChangeTime = time.Now().UTC() @@ -292,7 +293,7 @@ func (c *Cluster) updateAndCreateOSDsLoop( } case <-c.clusterInfo.Context.Done(): - logger.Infof("context cancelled, exiting OSD update and create loop") + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "context cancelled, exiting OSD update and create loop") return false, c.clusterInfo.Context.Err() } } @@ -327,7 +328,7 @@ func (c *Cluster) createOSDsForStatusMap( ) { nodeOrPVCName, ok := configMap.Labels[nodeLabelKey] if !ok { - logger.Warningf("missing node label on configmap %s", configMap.Name) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "missing node label on configmap %s", configMap.Name) return } @@ -340,7 +341,7 @@ func (c *Cluster) createOSDsForStatusMap( nodeOrPVC = "PVC" } - logger.Infof("OSD orchestration status for %s %s is %q", nodeOrPVC, nodeOrPVCName, status.Status) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD orchestration status for %s %s is %q", nodeOrPVC, nodeOrPVCName, status.Status) if status.Status == OrchestrationStatusCompleted { createConfig.createNewOSDsFromStatus(status, nodeOrPVCName, errs) @@ -362,7 +363,7 @@ func statusConfigMapName(nodeOrPVCName string) string { func (c *Cluster) deleteStatusConfigMap(nodeOrPVCName string) { if err := c.kv.ClearStore(c.clusterInfo.Context, statusConfigMapName(nodeOrPVCName)); err != nil { - logger.Errorf("failed to remove the status configmap %q. %v", statusConfigMapName(nodeOrPVCName), err) + log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to remove the status configmap %q. %v", statusConfigMapName(nodeOrPVCName), err) } } @@ -373,14 +374,14 @@ func (c *Cluster) deleteAllStatusConfigMaps() { cmClientset := c.context.Clientset.CoreV1().ConfigMaps(c.clusterInfo.Namespace) cms, err := cmClientset.List(c.clusterInfo.Context, listOpts) if err != nil { - logger.Warningf("failed to clean up any dangling OSD prepare status configmaps. failed to list OSD prepare status configmaps. %v", err) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to clean up any dangling OSD prepare status configmaps. failed to list OSD prepare status configmaps. %v", err) return } for _, cm := range cms.Items { - logger.Debugf("cleaning up dangling OSD prepare status configmap %q", cm.Name) + log.NamespacedDebug(c.clusterInfo.Namespace, logger, "cleaning up dangling OSD prepare status configmap %q", cm.Name) err := cmClientset.Delete(c.clusterInfo.Context, cm.Name, metav1.DeleteOptions{}) if err != nil { - logger.Warningf("failed to clean up dangling OSD prepare status configmap %q. %v", cm.Name, err) + log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to clean up dangling OSD prepare status configmap %q. %v", cm.Name, err) } } } diff --git a/pkg/operator/ceph/cluster/osd/update.go b/pkg/operator/ceph/cluster/osd/update.go index 621140d81..3354e327b 100644 --- a/pkg/operator/ceph/cluster/osd/update.go +++ b/pkg/operator/ceph/cluster/osd/update.go @@ -26,6 +26,7 @@ import ( cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/config/keyring" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -85,14 +86,14 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) { if !c.cluster.spec.SkipUpgradeChecks && c.cluster.spec.UpgradeOSDRequiresHealthyPGs { pgHealthMsg, pgClean, err := cephclient.IsClusterClean(c.cluster.context, c.cluster.clusterInfo, c.cluster.spec.DisruptionManagement.PGHealthyRegex) if err != nil { - logger.Warningf("failed to check PGs status to update OSDs, will try updating it again later. %v", err) + log.NamespacedWarning(c.cluster.clusterInfo.Namespace, logger, "failed to check PGs status to update OSDs, will try updating it again later. %v", err) return } if !pgClean { - logger.Infof("PGs are not healthy to update OSDs, will try updating it again later. PGs status: %q", pgHealthMsg) + log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "PGs are not healthy to update OSDs, will try updating it again later. PGs status: %q", pgHealthMsg) return } - logger.Infof("PGs are healthy to proceed updating OSDs. %v", pgHealthMsg) + log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "PGs are healthy to proceed updating OSDs. %v", pgHealthMsg) } osdIDQuery, _ := c.queue.Pop() @@ -102,7 +103,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) { if c.cluster.spec.SkipUpgradeChecks || !shouldCheckOkToStopFunc(c.cluster.context, c.cluster.clusterInfo) { // If we should not check ok-to-stop, then only process one OSD at a time. There are likely // less than 3 OSDs in the cluster or the cluster is on a single node. E.g., in CI :wink:. - logger.Infof("skipping osd checks for ok-to-stop") + log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "skipping osd checks for ok-to-stop") osdIDs = []int{osdIDQuery} } else { osdMaxUpdatesInParallel := c.cluster.spec.Storage.OSDMaxUpdatesInParallel @@ -112,29 +113,29 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) { osdIDs, err = cephclient.OSDOkToStop(c.cluster.context, c.cluster.clusterInfo, osdIDQuery, osdMaxUpdatesInParallel) if err != nil { if c.cluster.spec.ContinueUpgradeAfterChecksEvenIfNotHealthy { - logger.Infof("OSD %d is not ok-to-stop but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so continuing to update it", osdIDQuery) + log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "OSD %d is not ok-to-stop but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so continuing to update it", osdIDQuery) osdIDs = []int{osdIDQuery} // make sure to update the queried OSD } else { - logger.Infof("OSD %d is not ok-to-stop. will try updating it again later", osdIDQuery) + log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "OSD %d is not ok-to-stop. will try updating it again later", osdIDQuery) c.queue.Push(osdIDQuery) // push back onto queue to make sure we retry it later return } } } - logger.Debugf("updating OSDs: %v", osdIDs) + log.NamespacedDebug(c.cluster.clusterInfo.Namespace, logger, "updating OSDs: %v", osdIDs) updatedDeployments := make([]*appsv1.Deployment, 0, len(osdIDs)) listIDs := []string{} // use this to build the k8s api selector query for _, osdID := range osdIDs { if !c.deployments.Exists(osdID) { - logger.Debugf("not updating deployment for OSD %d that is newly created", osdID) + log.NamespacedDebug(c.cluster.clusterInfo.Namespace, logger, "not updating deployment for OSD %d that is newly created", osdID) continue } // osdIDQuery which has been popped off the queue but it does need to be updated if osdID != osdIDQuery && !c.queue.Exists(osdID) { - logger.Debugf("not updating deployment for OSD %d that is not in the update queue. the OSD has already been updated", osdID) + log.NamespacedDebug(c.cluster.clusterInfo.Namespace, logger, "not updating deployment for OSD %d that is not in the update queue. the OSD has already been updated", osdID) continue } @@ -152,7 +153,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) { c.osdDesiredState[osdID] = &osdInfo if c.osdsToSkipReconcile.Has(strconv.Itoa(osdID)) { - logger.Warningf("Skipping update for OSD %d since labeled with %s", osdID, cephv1.SkipReconcileLabelKey) + log.NamespacedWarning(c.cluster.clusterInfo.Namespace, logger, "Skipping update for OSD %d since labeled with %s", osdID, cephv1.SkipReconcileLabelKey) continue } @@ -162,7 +163,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) { if osdInfo.DeviceClass == "" || osdInfo.DeviceClass == "None" { deviceClassInfo, err := cephclient.OSDDeviceClasses(c.cluster.context, c.cluster.clusterInfo, []string{strconv.Itoa(osdID)}) if err != nil { - logger.Errorf("failed to get device class for existing deployment %q. %v", depName, err) + log.NamespacedError(c.cluster.clusterInfo.Namespace, logger, "failed to get device class for existing deployment %q. %v", depName, err) } else { osdInfo.DeviceClass = deviceClassInfo[0].DeviceClass } @@ -188,14 +189,14 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) { } if osdIsOnPVC(dep) { - logger.Infof("updating OSD %d on PVC %q", osdID, nodeOrPVCName) + log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "updating OSD %d on PVC %q", osdID, nodeOrPVCName) updatedDep, err = deploymentOnPVCFunc(c.cluster, &osdInfo, nodeOrPVCName, c.provisionConfig) message := fmt.Sprintf("Processing OSD %d on PVC %q", osdID, nodeOrPVCName) updateConditionFunc(c.cluster.clusterInfo.Context, c.cluster.context, c.cluster.clusterInfo.NamespacedName(), k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, message) } else { if !c.cluster.ValidStorage.NodeExists(nodeOrPVCName) { // node will not reconcile, so don't update the deployment - logger.Warningf( + log.NamespacedWarning(c.cluster.clusterInfo.Namespace, logger, "not updating OSD %d on node %q. node no longer exists in the storage spec. "+ "if the user wishes to remove OSDs from the node, they must do so manually. "+ "Rook will not remove OSDs from nodes that are removed from the storage spec in order to prevent accidental data loss", @@ -203,7 +204,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) { continue } - logger.Infof("updating OSD %d on node %q", osdID, nodeOrPVCName) + log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "updating OSD %d on node %q", osdID, nodeOrPVCName) updatedDep, err = deploymentOnNodeFunc(c.cluster, &osdInfo, nodeOrPVCName, c.provisionConfig) message := fmt.Sprintf("Processing OSD %d on node %q", osdID, nodeOrPVCName) updateConditionFunc(c.cluster.clusterInfo.Context, c.cluster.context, c.cluster.clusterInfo.NamespacedName(), k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, message) @@ -424,7 +425,7 @@ func (c *Cluster) rotateCephxKey(osdInfo OSDInfo) (cephv1.CephxStatus, error) { return didNotRotateCephxStatus, nil } - logger.Infof("rotating cephx key of OSD %d for CephCluster in namespace %q", osdInfo.ID, c.clusterInfo.Namespace) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "rotating cephx key of OSD %d for CephCluster in namespace %q", osdInfo.ID, c.clusterInfo.Namespace) user := fmt.Sprintf("osd.%d", osdInfo.ID) // Note: OSD key is not stored in k8s secret; rotated key is picked up by OSD init container _, err = cephclient.AuthRotate(c.context, c.clusterInfo, user) @@ -435,7 +436,7 @@ func (c *Cluster) rotateCephxKey(osdInfo OSDInfo) (cephv1.CephxStatus, error) { // rotating the `client.osd-lockbox.$OSD_UUID` keys created for luks-encrypted OSDs by ceph-volume if osdInfo.Encrypted { osdLockBoxUser := fmt.Sprintf("client.osd-lockbox.%s", osdInfo.UUID) - logger.Infof("rotating osd-lockbox cephx key of encrypted OSD %d for CephCluster in namespace %q", osdInfo.ID, c.clusterInfo.Namespace) + log.NamespacedInfo(c.clusterInfo.Namespace, logger, "rotating osd-lockbox cephx key of encrypted OSD %d for CephCluster in namespace %q", osdInfo.ID, c.clusterInfo.Namespace) _, err = cephclient.AuthRotate(c.context, c.clusterInfo, osdLockBoxUser) if err != nil { return didNotRotateCephxStatus, errors.Wrapf(err, "failed to rotate osd-lockbox cephx key for the encrypted OSD %d", osdInfo.ID) diff --git a/pkg/operator/ceph/cluster/osd/volumes.go b/pkg/operator/ceph/cluster/osd/volumes.go index d6c397cb1..268a03fe8 100644 --- a/pkg/operator/ceph/cluster/osd/volumes.go +++ b/pkg/operator/ceph/cluster/osd/volumes.go @@ -23,6 +23,7 @@ import ( kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms" "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/util/log" v1 "k8s.io/api/core/v1" ) @@ -169,7 +170,7 @@ func getPVCOSDVolumes(osdProps *osdProperties, configDir string, namespace strin volumes = append(volumes, walPVCVolume...) } - logger.Debugf("volumes are %+v", volumes) + log.NamespacedDebug(namespace, logger, "volumes are %+v", volumes) return volumes } diff --git a/pkg/operator/ceph/cluster/predicate.go b/pkg/operator/ceph/cluster/predicate.go index 713bc0b76..8ad7d6fb2 100644 --- a/pkg/operator/ceph/cluster/predicate.go +++ b/pkg/operator/ceph/cluster/predicate.go @@ -27,6 +27,7 @@ import ( discoverDaemon "github.com/rook/rook/pkg/daemon/discover" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -139,7 +140,7 @@ func predicateForOperatorConfigCMWatcher[T *corev1.ConfigMap]() predicate.TypedF } if objOld.Data["ROOK_USE_CSI_OPERATOR"] != objNew.Data["ROOK_USE_CSI_OPERATOR"] { - logger.Infof("ROOK_USE_CSI_OPERATOR changed from %q to %q", objOld.Data["ROOK_USE_CSI_OPERATOR"], objNew.Data["ROOK_USE_CSI_OPERATOR"]) + log.NamespacedInfo(objNew.Namespace, logger, "ROOK_USE_CSI_OPERATOR changed from %q to %q", objOld.Data["ROOK_USE_CSI_OPERATOR"], objNew.Data["ROOK_USE_CSI_OPERATOR"]) return true } @@ -196,18 +197,18 @@ func watchControllerPredicate[T *cephv1.CephCluster](ctx context.Context, c clie objOld := (*cephv1.CephCluster)(e.ObjectOld) objNew := (*cephv1.CephCluster)(e.ObjectNew) - logger.Debug("update event on CephCluster CR") + log.NamespacedDebug(objNew.Namespace, logger, "update event on CephCluster CR") // If the labels "do_not_reconcile" is set on the object, let's not reconcile that request if opcontroller.IsDoNotReconcile(objNew.GetLabels()) { - logger.Debugf("object %q matched on update but %q label is set, doing nothing", opcontroller.DoNotReconcileLabelName, objNew.Name) + log.NamespacedDebug(objNew.Namespace, logger, "object %q matched on update but %q label is set, doing nothing", opcontroller.DoNotReconcileLabelName, objNew.Name) return false } diff := cmp.Diff(objOld.Spec, objNew.Spec, resourceQtyComparer) if diff != "" { - logger.Infof("CR has changed for %q. diff=%s", objNew.Name, diff) + log.NamespacedInfo(objNew.Namespace, logger, "CR has changed for %q. diff=%s", objNew.Name, diff) if objNew.Spec.CleanupPolicy.HasDataDirCleanPolicy() { - logger.Infof("skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set. not reloading the manager", objNew.GetName(), objNew.GetNamespace()) + log.NamespacedInfo(objNew.Namespace, logger, "skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set. not reloading the manager", objNew.GetName(), objNew.GetNamespace()) return false } @@ -217,7 +218,7 @@ func watchControllerPredicate[T *cephv1.CephCluster](ctx context.Context, c clie return false } else if !objOld.GetDeletionTimestamp().Equal(objNew.GetDeletionTimestamp()) { - logger.Infof("CR %q is going be deleted, cancelling any ongoing orchestration", objNew.Name) + log.NamespacedInfo(objNew.Namespace, logger, "CR %q is going be deleted, cancelling any ongoing orchestration", objNew.Name) // Stop any ongoing orchestration opcontroller.ReloadManager() @@ -225,7 +226,7 @@ func watchControllerPredicate[T *cephv1.CephCluster](ctx context.Context, c clie return false } else if objOld.GetGeneration() != objNew.GetGeneration() { - logger.Debugf("reconciling CephCluster %q with changed generation", objNew.Name) + log.NamespacedDebug(objNew.Namespace, logger, "reconciling CephCluster %q with changed generation", objNew.Name) return true } diff --git a/pkg/operator/ceph/cluster/rbd/config.go b/pkg/operator/ceph/cluster/rbd/config.go index bfc445962..d59746baf 100644 --- a/pkg/operator/ceph/cluster/rbd/config.go +++ b/pkg/operator/ceph/cluster/rbd/config.go @@ -26,6 +26,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config/keyring" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -59,7 +60,7 @@ func (r *ReconcileCephRBDMirror) generateKeyring(clusterInfo *client.ClusterInfo } if r.shouldRotateCephxKeys { - logger.Infof("rotating cephx key for CephRBDMirror %q in the namespace %q", daemonConfig.ResourceName, clusterInfo.Namespace) + log.NamespacedInfo(clusterInfo.Namespace, logger, "rotating cephx key for CephRBDMirror %q in the namespace %q", daemonConfig.ResourceName, clusterInfo.Namespace) newKey, err := s.RotateKey(user) if err != nil { return "", errors.Wrapf(err, "failed to rotate cephx key for CephRDBMirror %q in the namespace %q", daemonConfig.ResourceName, clusterInfo.Namespace) @@ -79,9 +80,9 @@ func (r *ReconcileCephRBDMirror) reconcileAddBootstrapPeer(cephRBDMirror *cephv1 // List all the peers secret, we can have more than one peer we might want to configure // For each, get the Kubernetes Secret and import the "peer token" so that we can configure the mirroring - logger.Warning("(DEPRECATED) use of peer secret names in CephRBDMirror is deprecated. Please use CephBlockPool CR to configure peer secret names and import peers.") + log.NamespacedWarning(cephRBDMirror.Namespace, logger, "(DEPRECATED) use of peer secret names in CephRBDMirror is deprecated. Please use CephBlockPool CR to configure peer secret names and import peers.") for _, peerSecret := range cephRBDMirror.Spec.Peers.SecretNames { - logger.Debugf("fetching bootstrap peer kubernetes secret %q", peerSecret) + log.NamespacedDebug(cephRBDMirror.Namespace, logger, "fetching bootstrap peer kubernetes secret %q", peerSecret) s, err := r.context.Clientset.CoreV1().Secrets(r.clusterInfo.Namespace).Get(r.opManagerContext, peerSecret, metav1.GetOptions{}) // We don't care about IsNotFound here, we still need to fail if err != nil { diff --git a/pkg/operator/ceph/cluster/rbd/controller.go b/pkg/operator/ceph/cluster/rbd/controller.go index ddecd4575..824cd18fd 100644 --- a/pkg/operator/ceph/cluster/rbd/controller.go +++ b/pkg/operator/ceph/cluster/rbd/controller.go @@ -31,6 +31,7 @@ import ( opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/reporting" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -168,7 +169,7 @@ func (r *ReconcileCephRBDMirror) Reconcile(context context.Context, request reco if err != nil { return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed set ready failure status to the cephRBDMirror %q", request.NamespacedName) } - logger.Errorf("failed to reconcile %v", err) + log.NamespacedError(request.Namespace, logger, "failed to reconcile %v", err) } return reporting.ReportReconcileResult(logger, r.recorder, request, &cephRBDMirror, reconcileResponse, err) @@ -180,7 +181,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile err := r.client.Get(r.opManagerContext, request.NamespacedName, cephRBDMirror) if err != nil { if kerrors.IsNotFound(err) { - logger.Debug("cephRBDMirror resource not found. Ignoring since object must be deleted.") + log.NamespacedDebug(request.Namespace, logger, "cephRBDMirror resource not found. Ignoring since object must be deleted.") return reconcile.Result{}, *cephRBDMirror, nil } // Error reading the object - requeue the request. @@ -213,7 +214,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile // Make sure a CephCluster is present otherwise do nothing cephCluster, isReadyToReconcile, _, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { - logger.Debugf("CephCluster resource not ready in namespace %q, retrying in %q.", request.NamespacedName.Namespace, reconcileResponse.RequeueAfter.String()) + log.NamedDebug(request.NamespacedName, logger, "CephCluster resource not ready, retrying in %q.", reconcileResponse.RequeueAfter.String()) return reconcileResponse, *cephRBDMirror, nil } r.cephClusterSpec = &cephCluster.Spec @@ -238,7 +239,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile ) if err != nil { if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { - logger.Info(opcontroller.OperatorNotInitializedMessage) + log.NamespacedInfo(request.Namespace, logger, opcontroller.OperatorNotInitializedMessage) return opcontroller.WaitForRequeueIfOperatorNotInitialized, *cephRBDMirror, nil } return reconcile.Result{}, *cephRBDMirror, errors.Wrap(err, "failed to detect running and desired ceph version") @@ -256,7 +257,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile r.clusterInfo.CephVersion = *runningCephVersion // Add bootstrap peer if any - logger.Debug("reconciling ceph rbd mirror peers addition") + log.NamespacedDebug(request.Namespace, logger, "reconciling ceph rbd mirror peers addition") reconcileResponse, err = r.reconcileAddBootstrapPeer(cephRBDMirror, request.NamespacedName) if err != nil { return opcontroller.ImmediateRetryResult, *cephRBDMirror, errors.Wrap(err, "failed to add ceph rbd mirror peer") @@ -268,11 +269,11 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile return reconcile.Result{}, *cephRBDMirror, errors.Wrapf(err, "failed to determine if cephx keys should be rotated for the cephRBDMirror %q", request.NamespacedName) } if r.shouldRotateCephxKeys { - logger.Infof("cephx keys for CephRBDMirror %q will be rotated", request.NamespacedName) + log.NamedInfo(request.NamespacedName, logger, "cephx keys for CephRBDMirror will be rotated") } // CREATE/UPDATE - logger.Debug("reconciling ceph rbd mirror deployments") + log.NamespacedDebug(request.Namespace, logger, "reconciling ceph rbd mirror deployments") reconcileResponse, err = r.reconcileCreateCephRBDMirror(cephRBDMirror) if err != nil { return opcontroller.ImmediateRetryResult, *cephRBDMirror, errors.Wrap(err, "failed to create ceph rbd mirror deployments") @@ -288,7 +289,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile } // Return and do not requeue - logger.Debug("done reconciling ceph rbd mirror") + log.NamespacedDebug(request.Namespace, logger, "done reconciling ceph rbd mirror") return reconcile.Result{}, *cephRBDMirror, nil } @@ -318,7 +319,7 @@ func (r *ReconcileCephRBDMirror) updateStatus(observedGeneration int64, namespac err := r.client.Get(r.opManagerContext, namespacedName, rbdMirror) if err != nil { if kerrors.IsNotFound(err) { - logger.Debug("CephRBDMirror resource not found. Ignoring since object must be deleted.") + log.NamespacedDebug(namespacedName.Namespace, logger, "CephRBDMirror resource not found. Ignoring since object must be deleted.") return nil } return errors.Wrapf(err, "failed to retrieve CephRBDMirror %q to update status to %q", namespacedName, status) @@ -347,6 +348,6 @@ func (r *ReconcileCephRBDMirror) updateStatus(observedGeneration int64, namespac return err } - logger.Debugf("successfully updated cephRBDMirror %q status updated to %+v", namespacedName, rbdMirror.Status) + log.NamedDebug(namespacedName, logger, "successfully updated cephRBDMirror status updated to %+v", rbdMirror.Status) return nil } diff --git a/pkg/operator/ceph/cluster/rbd/mirror.go b/pkg/operator/ceph/cluster/rbd/mirror.go index 6ec03bd36..308a95c1c 100644 --- a/pkg/operator/ceph/cluster/rbd/mirror.go +++ b/pkg/operator/ceph/cluster/rbd/mirror.go @@ -28,6 +28,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config/keyring" "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -51,7 +52,7 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro return errors.Wrap(err, "error checking pod memory") } - logger.Infof("configure rbd-mirroring with %d workers", cephRBDMirror.Spec.Count) + log.NamespacedInfo(cephRBDMirror.Namespace, logger, "configure rbd-mirroring with %d workers", cephRBDMirror.Spec.Count) ownerInfo := k8sutil.NewOwnerInfo(cephRBDMirror, r.scheme) daemonID := k8sutil.IndexToName(0) @@ -68,7 +69,7 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro return errors.Wrap(err, "failed to check for RBD Mirror to skip reconcile") } if rbdMirrorToSkipReconcile.Len() > 0 { - logger.Warningf("skipping RBD mirror reconcile since RBD mirror daemons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(rbdMirrorToSkipReconcile)) + log.NamespacedWarning(cephRBDMirror.Namespace, logger, "skipping RBD mirror reconcile since RBD mirror daemons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(rbdMirrorToSkipReconcile)) return nil } @@ -102,11 +103,11 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro if !kerrors.IsAlreadyExists(err) { return errors.Wrapf(err, "failed to create %q deployment", resourceName) } - logger.Infof("deployment for rbd-mirror %q already exists. updating if needed", resourceName) + log.NamespacedInfo(cephRBDMirror.Namespace, logger, "deployment for rbd-mirror %q already exists. updating if needed", resourceName) if err := updateDeploymentAndWait(r.context, r.clusterInfo, d, config.RbdMirrorType, daemonConf.DaemonID, r.cephClusterSpec.SkipUpgradeChecks, false); err != nil { // fail could be an issue updating label selector (immutable), so try del and recreate - logger.Debugf("updateDeploymentAndWait failed for rbd-mirror %q. Attempting del-and-recreate. %v", resourceName, err) + log.NamespacedDebug(cephRBDMirror.Namespace, logger, "updateDeploymentAndWait failed for rbd-mirror %q. Attempting del-and-recreate. %v", resourceName, err) err = r.context.Clientset.AppsV1().Deployments(cephRBDMirror.Namespace).Delete(r.opManagerContext, d.Name, metav1.DeleteOptions{}) if err != nil && !kerrors.IsNotFound(err) { return errors.Wrapf(err, "failed to delete rbd-mirror %q during del-and-recreate update attempt", resourceName) @@ -117,6 +118,6 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro } } - logger.Infof("%q deployment started", resourceName) + log.NamespacedInfo(cephRBDMirror.Namespace, logger, "%q deployment started", resourceName) return nil } diff --git a/pkg/operator/ceph/cluster/telemetry/telemetry.go b/pkg/operator/ceph/cluster/telemetry/telemetry.go index da3a3239b..b848b4c62 100644 --- a/pkg/operator/ceph/cluster/telemetry/telemetry.go +++ b/pkg/operator/ceph/cluster/telemetry/telemetry.go @@ -23,6 +23,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/util/log" ) var logger = capnslog.NewPackageLogger("github.com/rook/rook", "telemetry") @@ -53,8 +54,8 @@ var CSIVersion string func ReportKeyValue(context *clusterd.Context, clusterInfo *client.ClusterInfo, key, value string) { ms := config.GetMonStore(context, clusterInfo) if err := ms.SetKeyValue(key, value); err != nil { - logger.Warningf("failed to set telemetry key %q. %v", key, err) + log.NamespacedWarning(clusterInfo.Namespace, logger, "failed to set telemetry key %q. %v", key, err) return } - logger.Debugf("set telemetry key: %s=%s", key, value) + log.NamespacedDebug(clusterInfo.Namespace, logger, "set telemetry key: %s=%s", key, value) } diff --git a/pkg/operator/ceph/cluster/utils.go b/pkg/operator/ceph/cluster/utils.go index d5a728610..a1917b8a9 100644 --- a/pkg/operator/ceph/cluster/utils.go +++ b/pkg/operator/ceph/cluster/utils.go @@ -23,6 +23,7 @@ import ( "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/log" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,7 +37,7 @@ func populateConfigOverrideConfigMap(clusterdContext *clusterd.Context, namespac existingCM, err := clusterdContext.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, k8sutil.ConfigOverrideName, metav1.GetOptions{}) if err != nil { if !kerrors.IsNotFound(err) { - logger.Warningf("failed to get cm %q to check labels and annotations", k8sutil.ConfigOverrideName) + log.NamespacedWarning(namespace, logger, "failed to get cm %q to check labels and annotations", k8sutil.ConfigOverrideName) return nil } @@ -66,7 +67,7 @@ func populateConfigOverrideConfigMap(clusterdContext *clusterd.Context, namespac if err != nil { return errors.Wrapf(err, "failed to create override configmap %s", namespace) } - logger.Infof("created placeholder configmap for ceph overrides %q", cm.Name) + log.NamespacedInfo(namespace, logger, "created placeholder configmap for ceph overrides %q", cm.Name) return nil } @@ -83,9 +84,9 @@ func populateConfigOverrideConfigMap(clusterdContext *clusterd.Context, namespac if updateRequired { _, err = clusterdContext.Clientset.CoreV1().ConfigMaps(namespace).Update(ctx, existingCM, metav1.UpdateOptions{}) if err != nil { - logger.Warningf("failed to add recommended labels and annotations to configmap %q. %v", existingCM.Name, err) + log.NamespacedWarning(namespace, logger, "failed to add recommended labels and annotations to configmap %q. %v", existingCM.Name, err) } else { - logger.Infof("added expected labels and annotations to configmap %q", existingCM.Name) + log.NamespacedInfo(namespace, logger, "added expected labels and annotations to configmap %q", existingCM.Name) } } return nil @@ -96,15 +97,15 @@ func initRequiredMetadata(metadata metav1.ObjectMeta, labels, annotations map[st releaseNameAttr := "meta.helm.sh/release-name" chartName, ok := metadata.Annotations[releaseNameAttr] if !ok { - logger.Debug("cluster helm chart is not configured, not adding helm annotations to configmap") + log.NamespacedDebug(metadata.Namespace, logger, "cluster helm chart is not configured, not adding helm annotations to configmap") return false } if _, ok := annotations[releaseNameAttr]; ok { - logger.Debug("cluster helm chart helm annotations already added to configmap") + log.NamespacedDebug(metadata.Namespace, logger, "cluster helm chart helm annotations already added to configmap") return false } - logger.Infof("adding helm chart name %q annotation to configmap", chartName) + log.NamespacedInfo(metadata.Namespace, logger, "adding helm chart name %q annotation to configmap", chartName) labels["app.kubernetes.io/managed-by"] = "Helm" annotations[releaseNameAttr] = chartName annotations["meta.helm.sh/release-namespace"] = metadata.Namespace diff --git a/pkg/operator/ceph/cluster/version.go b/pkg/operator/ceph/cluster/version.go index bef0ab249..d3c478202 100644 --- a/pkg/operator/ceph/cluster/version.go +++ b/pkg/operator/ceph/cluster/version.go @@ -24,6 +24,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/controller" cephver "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/rook/rook/pkg/util/log" ) func (c *ClusterController) detectAndValidateCephVersion(cluster *cluster) (*cephver.CephVersion, bool, error) { @@ -40,7 +41,7 @@ func (c *ClusterController) detectAndValidateCephVersion(cluster *cluster) (*cep return nil, false, err } - logger.Info("validating ceph version from provided image") + log.NamespacedInfo(cluster.Namespace, logger, "validating ceph version from provided image") if err := cluster.validateCephVersion(version); err != nil { return nil, cluster.isUpgrade, err } @@ -54,7 +55,7 @@ func (c *ClusterController) detectAndValidateCephVersion(cluster *cluster) (*cep func (c *cluster) printOverallCephVersion() { versions, err := daemonclient.GetAllCephDaemonVersions(c.context, c.ClusterInfo) if err != nil { - logger.Errorf("failed to get ceph daemons versions. %v", err) + log.NamespacedError(c.Namespace, logger, "failed to get ceph daemons versions. %v", err) return } @@ -62,21 +63,21 @@ func (c *cluster) printOverallCephVersion() { for v := range versions.Overall { version, err := cephver.ExtractCephVersion(v) if err != nil { - logger.Errorf("failed to extract ceph version. %v", err) + log.NamespacedError(c.Namespace, logger, "failed to extract ceph version. %v", err) return } vv := *version - logger.Infof("successfully upgraded cluster to version: %q", vv.String()) + log.NamespacedInfo(c.Namespace, logger, "successfully upgraded cluster to version: %q", vv.String()) } } else { // This shouldn't happen, but let's log just in case - logger.Warningf("upgrade orchestration completed but somehow we still have more than one Ceph version running. %v:", versions.Overall) + log.NamespacedWarning(c.Namespace, logger, "upgrade orchestration completed but somehow we still have more than one Ceph version running. %v:", versions.Overall) } } // This function compare the Ceph spec image and the cluster running version // It returns true if the image is different and false if identical -func diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion, runningVersions cephv1.CephDaemonsVersions) (bool, error) { +func (c *cluster) diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion, runningVersions cephv1.CephDaemonsVersions) (bool, error) { numberOfCephVersions := len(runningVersions.Overall) if numberOfCephVersions == 0 { // let's return immediately @@ -85,7 +86,7 @@ func diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion, if numberOfCephVersions > 1 { // let's return immediately - logger.Warningf("it looks like we have more than one ceph version running. triggering upgrade. %+v:", runningVersions.Overall) + log.NamespacedWarning(c.Namespace, logger, "it looks like we have more than one ceph version running. triggering upgrade. %+v:", runningVersions.Overall) return true, nil } @@ -93,24 +94,24 @@ func diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion, for v := range runningVersions.Overall { version, err := cephver.ExtractCephVersion(v) if err != nil { - logger.Errorf("failed to extract ceph version. %v", err) + log.NamespacedError(c.Namespace, logger, "failed to extract ceph version. %v", err) return false, err } clusterRunningVersion := *version // If this is the same version if cephver.IsIdentical(clusterRunningVersion, imageSpecVersion) { - logger.Debugf("both cluster and image spec versions are identical, doing nothing %s", imageSpecVersion.String()) + log.NamespacedDebug(c.Namespace, logger, "both cluster and image spec versions are identical, doing nothing %s", imageSpecVersion.String()) return false, nil } if cephver.IsSuperior(imageSpecVersion, clusterRunningVersion) { - logger.Infof("image spec version %s is higher than the running cluster version %s, upgrading", imageSpecVersion.String(), clusterRunningVersion.String()) + log.NamespacedInfo(c.Namespace, logger, "image spec version %s is higher than the running cluster version %s, upgrading", imageSpecVersion.String(), clusterRunningVersion.String()) return true, nil } if cephver.IsInferior(imageSpecVersion, clusterRunningVersion) { - logger.Warningf("image spec version %s is lower than the running cluster version %s, downgrading is not supported", imageSpecVersion.String(), clusterRunningVersion.String()) + log.NamespacedWarning(c.Namespace, logger, "image spec version %s is lower than the running cluster version %s, downgrading is not supported", imageSpecVersion.String(), clusterRunningVersion.String()) return true, nil } } @@ -129,11 +130,11 @@ func (c *cluster) validateCephVersion(version *cephver.CephVersion) error { if !c.Spec.CephVersion.AllowUnsupported { return errors.Errorf("allowUnsupported must be set to true to run with this version %q", version.String()) } - logger.Warningf("unsupported ceph version detected: %q, pursuing", version) + log.NamespacedWarning(c.Namespace, logger, "unsupported ceph version detected: %q, pursuing", version) } if version.Unsupported() { - logger.Errorf("UNSUPPORTED: ceph version %q detected, it is recommended to rollback to the previous pin-point stable release, pursuing anyways", version) + log.NamespacedError(c.Namespace, logger, "UNSUPPORTED: ceph version %q detected, it is recommended to rollback to the previous pin-point stable release, pursuing anyways", version) } } @@ -149,13 +150,13 @@ func (c *cluster) validateCephVersion(version *cephver.CephVersion) error { // Write connection info (ceph config file and keyring) for ceph commands err = mon.WriteConnectionConfig(c.context, clusterInfo) if err != nil { - logger.Errorf("failed to write config. attempting to continue. %v", err) + log.NamespacedError(c.Namespace, logger, "failed to write config. attempting to continue. %v", err) } } if err := clusterInfo.IsInitialized(); err != nil { // If not initialized, this is likely a new cluster so there is nothing to do - logger.Debugf("cluster not initialized, nothing to validate. %v", err) + log.NamespacedDebug(c.Namespace, logger, "cluster not initialized, nothing to validate. %v", err) return nil } @@ -170,21 +171,21 @@ func (c *cluster) validateCephVersion(version *cephver.CephVersion) error { // On external cluster setup, if we don't bootstrap any resources in the Kubernetes cluster then // there is no need to validate the Ceph image further if c.Spec.External.Enable && c.Spec.CephVersion.Image == "" { - logger.Debug("no spec image specified on external cluster, not validating Ceph version.") + log.NamespacedDebug(c.Namespace, logger, "no spec image specified on external cluster, not validating Ceph version.") return nil } // Get cluster running versions versions, err := daemonclient.GetAllCephDaemonVersions(c.context, c.ClusterInfo) if err != nil { - logger.Errorf("failed to get ceph daemons versions, this typically happens during the first cluster initialization. %v", err) + log.NamespacedError(c.Namespace, logger, "failed to get ceph daemons versions, this typically happens during the first cluster initialization. %v", err) return nil } runningVersions := *versions - differentImages, err := diffImageSpecAndClusterRunningVersion(*version, runningVersions) + differentImages, err := c.diffImageSpecAndClusterRunningVersion(*version, runningVersions) if err != nil { - logger.Errorf("failed to determine if we should upgrade or not. %v", err) + log.NamespacedError(c.Namespace, logger, "failed to determine if we should upgrade or not. %v", err) // we shouldn't block the orchestration if we can't determine the version of the image spec, we proceed anyway in best effort // we won't be able to check if there is an update or not and what to do, so we don't check the cluster status either // This will happen if someone uses ceph/daemon:latest-master for instance @@ -197,13 +198,13 @@ func (c *cluster) validateCephVersion(version *cephver.CephVersion) error { cephHealthy := daemonclient.IsCephHealthy(c.context, c.ClusterInfo) if !cephHealthy { if c.Spec.SkipUpgradeChecks { - logger.Warning("ceph is not healthy but SkipUpgradeChecks is set, forcing upgrade.") + log.NamespacedWarning(c.Namespace, logger, "ceph is not healthy but SkipUpgradeChecks is set, forcing upgrade.") } else { return errors.Errorf("ceph status in namespace %s is not healthy, refusing to upgrade. Either fix the health issue or force an update by setting skipUpgradeChecks to true in the cluster CR", c.Namespace) } } // This is an upgrade - logger.Infof("upgrading ceph cluster to %q", version.String()) + log.NamespacedInfo(c.Namespace, logger, "upgrading ceph cluster to %q", version.String()) c.isUpgrade = true } diff --git a/pkg/operator/ceph/cluster/version_test.go b/pkg/operator/ceph/cluster/version_test.go index 29a00f2c0..68bf979b9 100755 --- a/pkg/operator/ceph/cluster/version_test.go +++ b/pkg/operator/ceph/cluster/version_test.go @@ -44,7 +44,8 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) { err := json.Unmarshal(fakeRunningVersions, &dummyRunningVersions) assert.NoError(t, err) - m, err := diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions) + c := testSpec(t) + m, err := c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions) assert.Error(t, err) // Overall is absent assert.False(t, m) @@ -60,7 +61,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) { err = json.Unmarshal(fakeRunningVersions, &dummyRunningVersions2) assert.NoError(t, err) - m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions2) + m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions2) assert.NoError(t, err) assert.True(t, m) @@ -76,7 +77,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) { assert.NoError(t, err) // Allow the downgrade - m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions3) + m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions3) assert.NoError(t, err) assert.True(t, m) @@ -92,7 +93,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) { err = json.Unmarshal(fakeRunningVersions, &dummyRunningVersions4) assert.NoError(t, err) - m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions4) + m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions4) assert.NoError(t, err) assert.True(t, m) @@ -111,7 +112,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) { err = json.Unmarshal(fakeRunningVersions, &dummyRunningVersions5) assert.NoError(t, err) - m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions5) + m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions5) assert.NoError(t, err) assert.False(t, m) @@ -130,7 +131,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) { err = json.Unmarshal(fakeRunningVersions, &dummyRunningVersions6) assert.NoError(t, err) - m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions6) + m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions6) assert.NoError(t, err) assert.True(t, m) @@ -149,7 +150,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) { err = json.Unmarshal(fakeRunningVersions, &dummyRunningVersions7) assert.NoError(t, err) - m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions7) + m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions7) assert.NoError(t, err) assert.False(t, m) } diff --git a/pkg/util/log/log.go b/pkg/util/log/log.go new file mode 100644 index 000000000..6c186b007 --- /dev/null +++ b/pkg/util/log/log.go @@ -0,0 +1,63 @@ +/* +Copyright 2025 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package log + +import ( + "fmt" + + "github.com/coreos/pkg/capnslog" + "k8s.io/apimachinery/pkg/types" +) + +func NamedInfo(name types.NamespacedName, logger *capnslog.PackageLogger, message string, args ...interface{}) { + NamespacedInfo(name.String(), logger, message, args...) +} + +func NamedWarning(name types.NamespacedName, logger *capnslog.PackageLogger, message string, args ...interface{}) { + NamespacedWarning(name.String(), logger, message, args...) +} + +func NamedError(name types.NamespacedName, logger *capnslog.PackageLogger, message string, args ...interface{}) { + NamespacedError(name.String(), logger, message, args...) +} + +func NamedDebug(name types.NamespacedName, logger *capnslog.PackageLogger, message string, args ...interface{}) { + NamespacedDebug(name.String(), logger, message, args...) +} + +func NamedTrace(name types.NamespacedName, logger *capnslog.PackageLogger, message string, args ...interface{}) { + NamespacedTrace(name.String(), logger, message, args...) +} + +func NamespacedInfo(namespace string, logger *capnslog.PackageLogger, message string, args ...interface{}) { + logger.Infof("[%s] %s", namespace, fmt.Sprintf(message, args...)) +} + +func NamespacedWarning(namespace string, logger *capnslog.PackageLogger, message string, args ...interface{}) { + logger.Warningf("[%s] %s", namespace, fmt.Sprintf(message, args...)) +} + +func NamespacedError(namespace string, logger *capnslog.PackageLogger, message string, args ...interface{}) { + logger.Errorf("[%s] %s", namespace, fmt.Sprintf(message, args...)) +} + +func NamespacedDebug(namespace string, logger *capnslog.PackageLogger, message string, args ...interface{}) { + logger.Debugf("[%s] %s", namespace, fmt.Sprintf(message, args...)) +} + +func NamespacedTrace(namespace string, logger *capnslog.PackageLogger, message string, args ...interface{}) { + logger.Tracef("[%s] %s", namespace, fmt.Sprintf(message, args...)) +}