core: include namespace more consistently in logging

When writing to the operator log, where possible
the namespace of the resource on which the operator
is working will be logged as a standard prefix
to more easily troubleshoot what resource is being reconciled

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
Travis Nielsen committed 2025-11-20 15:49:43 -07:00
1 parent a086077d05
commit 93adc5dd6b
43 files changed
+727 -622

No files matched your search

+25 -24
View File
@@ -34,6 +34,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/reporting"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
v1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -75,7 +76,7 @@ func newCephStatusChecker(context *clusterd.Context, clusterInfo *cephclient.Clu
checkInterval = &checkIntervalCRSetting.Duration
}
if checkInterval != nil {
logger.Infof("ceph status check interval is %s", checkInterval.String())
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "ceph status check interval is %s", checkInterval.String())
c.interval = checkInterval
}
@@ -87,7 +88,7 @@ func (c *cephStatusChecker) checkCephStatus(monitoringRoutines *sync.Map, daemon
// check the status immediately before starting the loop
v, ok := monitoringRoutines.Load(daemon)
if !ok {
logger.Infof("ceph cluster %q has been deleted. stopping ceph status check", c.clusterInfo.Namespace)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "ceph cluster %q has been deleted. stopping ceph status check", c.clusterInfo.Namespace)
return
}
c.checkStatus(v.(*opcontroller.ClusterHealth).InternalCtx)
@@ -97,13 +98,13 @@ func (c *cephStatusChecker) checkCephStatus(monitoringRoutines *sync.Map, daemon
// we will get an invalid pointer error and the go routine will panic
v, ok := monitoringRoutines.Load(daemon)
if !ok {
logger.Infof("ceph cluster %q has been deleted. stopping ceph status check", c.clusterInfo.Namespace)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "ceph cluster %q has been deleted. stopping ceph status check", c.clusterInfo.Namespace)
return
}
health := v.(*opcontroller.ClusterHealth)
select {
case <-health.InternalCtx.Done():
logger.Infof("stopping monitoring of ceph status")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "stopping monitoring of ceph status")
monitoringRoutines.Delete(daemon)
return
@@ -118,27 +119,27 @@ func (c *cephStatusChecker) checkStatus(ctx context.Context) {
var status cephclient.CephStatus
var err error
logger.Debugf("checking health of cluster")
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "checking health of cluster")
// Check ceph's status
status, err = cephclient.StatusWithUser(c.context, c.clusterInfo)
if err != nil {
if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
logger.Info("skipping ceph status since operator is still initializing")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "skipping ceph status since operator is still initializing")
return
}
logger.Errorf("failed to get ceph status. %v", err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get ceph status. %v", err)
status := cephStatusOnError(err.Error())
c.updateCephStatus(status)
return
}
logger.Debugf("cluster status: %+v", status)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "cluster status: %+v", status)
c.updateCephStatus(&status)
if status.Health.Status != "HEALTH_OK" {
logger.Debug("checking for stuck pods on not ready nodes")
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "checking for stuck pods on not ready nodes")
if err := c.forceDeleteStuckRookPodsOnNotReadyNodes(ctx); err != nil {
logger.Errorf("failed to delete pod on not ready nodes. %v", err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to delete pod on not ready nodes. %v", err)
}
}
@@ -148,16 +149,16 @@ func (c *cephStatusChecker) checkStatus(ctx context.Context) {
func (c *cephStatusChecker) configureHealthSettings(status cephclient.CephStatus) {
// loop through the health codes and log what we find
for healthCode, check := range status.Health.Checks {
logger.Debugf("Health: %q, code: %q, message: %q", check.Severity, healthCode, check.Summary.Message)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "Health: %q, code: %q, message: %q", check.Severity, healthCode, check.Summary.Message)
}
// disable the insecure global id if there are no old clients
if _, ok := status.Health.Checks["AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED"]; ok {
if _, ok := status.Health.Checks["AUTH_INSECURE_GLOBAL_ID_RECLAIM"]; !ok {
logger.Info("Disabling the insecure global ID as no legacy clients are currently connected. If you still require the insecure connections, see the CVE to suppress the health warning and re-enable the insecure connections. https://docs.ceph.com/en/latest/security/CVE-2021-20288/")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Disabling the insecure global ID as no legacy clients are currently connected. If you still require the insecure connections, see the CVE to suppress the health warning and re-enable the insecure connections. https://docs.ceph.com/en/latest/security/CVE-2021-20288/")
config.DisableInsecureGlobalID(c.context, c.clusterInfo)
} else {
logger.Warning("insecure clients are connected to the cluster, to resolve the AUTH_INSECURE_GLOBAL_ID_RECLAIM health warning please refer to the upgrade guide to ensure all Ceph daemons are updated.")
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "insecure clients are connected to the cluster, to resolve the AUTH_INSECURE_GLOBAL_ID_RECLAIM health warning please refer to the upgrade guide to ensure all Ceph daemons are updated.")
}
}
}
@@ -168,10 +169,10 @@ func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus) {
cephCluster, err := c.context.RookClientset.CephV1().CephClusters(clusterName.Namespace).Get(c.clusterInfo.Context, clusterName.Name, metav1.GetOptions{})
if err != nil {
if kerrors.IsNotFound(err) {
logger.Debug("CephCluster resource not found. Ignoring since object must be deleted.")
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "CephCluster resource not found. Ignoring since object must be deleted.")
return
}
logger.Errorf("failed to retrieve ceph cluster %q in namespace %q to update status to %+v", clusterName.Name, clusterName.Namespace, status)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to retrieve ceph cluster %q in namespace %q to update status to %+v", clusterName.Name, clusterName.Namespace, status)
return
}
@@ -181,16 +182,16 @@ func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus) {
// versions store the ceph version of all the ceph daemons and overall cluster version
versions, err := cephclient.GetAllCephDaemonVersions(c.context, c.clusterInfo)
if err != nil {
logger.Errorf("failed to get ceph daemons versions. %v", err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get ceph daemons versions. %v", err)
} else {
// Update status with Ceph versions
cephCluster.Status.CephStatus.Versions = versions
}
// Update condition
logger.Debugf("updating ceph cluster %q status to %+v", clusterName.Namespace, status)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "updating ceph cluster %q status to %+v", clusterName.Namespace, status)
if err := reporting.UpdateStatus(c.context.Client, cephCluster); err != nil {
logger.Errorf("failed to update cluster status to %+v. %v", status, err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to update cluster status to %+v. %v", status, err)
}
}
@@ -237,15 +238,15 @@ func formatTime(t time.Time) string {
}
func (c *ClusterController) updateClusterCephVersion(cluster *cluster, cephVersion cephver.CephVersion) {
logger.Infof("cluster %q: version %q detected for image %q", cluster.namespacedName.Namespace, cephVersion.String(), cluster.Spec.CephVersion.Image)
log.NamespacedInfo(cluster.Namespace, logger, "cluster %q: version %q detected for image %q", cluster.namespacedName.Namespace, cephVersion.String(), cluster.Spec.CephVersion.Image)
cephCluster, err := c.context.RookClientset.CephV1().CephClusters(cluster.namespacedName.Namespace).Get(c.OpManagerCtx, cluster.namespacedName.Name, metav1.GetOptions{})
if err != nil {
if kerrors.IsNotFound(err) {
logger.Debug("CephCluster resource not found. Ignoring since object must be deleted.")
log.NamespacedDebug(cluster.Namespace, logger, "CephCluster resource not found. Ignoring since object must be deleted.")
return
}
logger.Errorf("failed to retrieve ceph cluster %q to update ceph version to %+v. %v", cluster.namespacedName.Name, cephVersion, err)
log.NamespacedError(cluster.Namespace, logger, "failed to retrieve ceph cluster %q to update ceph version to %+v. %v", cluster.namespacedName.Name, cephVersion, err)
return
}
@@ -257,7 +258,7 @@ func (c *ClusterController) updateClusterCephVersion(cluster *cluster, cephVersi
// do not overwrite the ceph status that is updated in a separate goroutine
cephCluster.Status.CephVersion = cephClusterVersion
if err := reporting.UpdateStatus(c.client, cephCluster); err != nil {
logger.Errorf("failed to update cluster %q version. %v", cluster.namespacedName.Name, err)
log.NamespacedError(cluster.Namespace, logger, "failed to update cluster %q version. %v", cluster.namespacedName.Name, err)
return
}
}
@@ -289,11 +290,11 @@ func (c *cephStatusChecker) forceDeleteStuckRookPodsOnNotReadyNodes(ctx context.
for _, node := range nodes {
pods, err := c.getRookPodsOnNode(node.Name)
if err != nil {
logger.Errorf("failed to get pods on NotReady node %q. %v", node.Name, err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get pods on NotReady node %q. %v", node.Name, err)
}
for _, pod := range pods {
if err := k8sutil.ForceDeletePodIfStuck(ctx, c.context, pod); err != nil {
logger.Warningf("skipping forced delete of stuck pod %q. %v", pod.Name, err)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "skipping forced delete of stuck pod %q. %v", pod.Name, err)
}
}
}
+17 -16
View File
@@ -33,6 +33,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/ceph/reporting"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
kerrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/client-go/util/retry"
)
@@ -73,9 +74,9 @@ func claimAdminRotationLock(namespace string) error {
adminRotationMutex.Lock()
defer adminRotationMutex.Unlock()
if _, ok := adminRotationInProgress[namespace]; ok {
return fmt.Errorf("failed to claim admin rotation lock for cluster in namespace %q", namespace)
return fmt.Errorf("failed to claim admin rotation lock")
}
logger.Debugf("claimed admin rotation lock for cluster in namespace %q", namespace)
log.NamespacedDebug(namespace, logger, "claimed admin rotation lock")
adminRotationInProgress[namespace] = struct{}{}
return nil
}
@@ -118,7 +119,7 @@ func rotateAdminCephxKey(
) error {
if clusterInfo.CephCred.Username != cephclient.AdminUsername {
// this shouldn't happen during normal runtime - this could indicate external mode cluster
logger.Infof("cannot rotate admin cephx key with non-rotatable username %q for cluster in namespace %q", clusterInfo.CephCred.Username, clusterInfo.Namespace)
log.NamespacedInfo(clusterInfo.Namespace, logger, "cannot rotate admin cephx key with non-rotatable username %q", clusterInfo.CephCred.Username)
return nil
}
@@ -129,7 +130,7 @@ func rotateAdminCephxKey(
return errors.Wrap(err, "failed to determine if admin cephx key should be rotated")
}
if !shouldRotate {
logger.Debugf("not rotating admin cephx key for cluster in namespace %q", clusterInfo.Namespace)
log.NamespacedDebug(clusterInfo.Namespace, logger, "not rotating admin cephx key")
// no rotation, but still update cephx status - critical for greenfield Uninitialized clusters
err := updateCephClusterAdminCephxStatus(clusterdCtx, clusterInfo, false)
if err != nil {
@@ -143,7 +144,7 @@ func rotateAdminCephxKey(
}
defer releaseAdminRotationLock(clusterInfo.Namespace)
logger.Infof("beginning admin cephx key rotation for cluster in namespace %q", clusterInfo.Namespace)
log.NamespacedInfo(clusterInfo.Namespace, logger, "beginning admin cephx key rotation")
// As an optimization, use a (shallow) copy of clusterInfo with a context that won't be canceled
// during normal runtime. This disallows the rotation process from being interrupted partway
@@ -194,7 +195,7 @@ func recoverPriorAdminCephxKeyRotation(
) error {
if clusterInfo == nil {
// new cluster hasn't been deployed yet. there should be no way for recovery to be needed
logger.Debugf("no admin cephx key recovery possible for cluster with empty clusterInfo in namespace %q", clusterNamespace)
log.NamespacedDebug(clusterNamespace, logger, "no admin cephx key recovery possible for cluster with empty clusterInfo")
return nil
}
@@ -203,7 +204,7 @@ func recoverPriorAdminCephxKeyRotation(
rotatorKeyring, err := s.GetKeyringFromSecret(adminRotatorSecretPrefix)
if err != nil {
if kerrors.IsNotFound(err) {
logger.Debugf("no admin cephx key recovery needed for cluster in namespace %q", clusterInfo.Namespace)
log.NamespacedDebug(clusterNamespace, logger, "no admin cephx key recovery needed")
return nil
}
return errors.Wrap(err, "failed to get admin rotator cephx key secret to check if admin cephx key rotation recovery is needed")
@@ -214,7 +215,7 @@ func recoverPriorAdminCephxKeyRotation(
}
defer releaseAdminRotationLock(clusterNamespace)
logger.Infof("recovering from interrupted admin cephx key rotation for cluster in namespace %q", clusterInfo.Namespace)
log.NamespacedInfo(clusterNamespace, logger, "recovering from interrupted admin cephx key rotation")
// As an optimization, use a (shallow) copy of clusterInfo with a context that won't be canceled
// during normal runtime. This disallows the recovery process from being interrupted partway
@@ -239,7 +240,7 @@ func recoverPriorAdminCephxKeyRotation(
authList, err := cephclient.AuthList(clusterdCtx, clusterInfoCopy)
if err != nil {
logger.Debugf("primary admin user failed to list ceph auth - this is an expected condition when recovering from admin cephx key rotation")
log.NamespacedDebug(clusterNamespace, logger, "primary admin user failed to list ceph auth - this is an expected condition when recovering from admin cephx key rotation")
} else if !entityExistsInAuthList(adminRotatorUsername, authList) { // err == nil
// corner case: if the primary client.admin works AND client.admin-rotator is not present in
// the `auth ls` output, it means rotation succeeded and was interrupted during cleanup.
@@ -271,7 +272,7 @@ func rotateAdminCephxKeyUsingRotator(
) error {
// write admin rotator keyring file to tempdir so it can be used for rotation
rotatorKeyringTmpfile := filepath.Join(tmpDir, adminRotatorUsername+".keyring")
logger.Infof("temporarily storing admin rotator keyring at %q for cluster in namespace %q", rotatorKeyringTmpfile, clusterInfo.Namespace)
log.NamespacedInfo(clusterInfo.Namespace, logger, "temporarily storing admin rotator keyring at %q", rotatorKeyringTmpfile)
err := cephclient.WriteKeyring(rotatorKeyringTmpfile, rotatorKeyring)
if err != nil {
return errors.Wrapf(err, "failed to write admin rotator keyring to %q", rotatorKeyringTmpfile)
@@ -287,7 +288,7 @@ func rotateAdminCephxKeyUsingRotator(
}
// as `client.admin-rotator`: run `ceph auth rotate client.admin`
logger.Infof("admin cephx key will be rotated for cluster in namespace %q. rook will restart afterwards. some reconciles and health checks may fail in between - this is normal", clusterInfo.Namespace)
log.NamespacedInfo(clusterInfo.Namespace, logger, "admin cephx key will be rotated. rook will restart afterwards. some reconciles and health checks may fail in between - this is normal")
newAdminKey, err := cephclient.AuthRotate(clusterdCtx, rotatorInfo, cephclient.AdminUsername)
if err != nil {
return errors.Wrapf(err, "failed to rotate admin key using admin rotator client")
@@ -296,7 +297,7 @@ func rotateAdminCephxKeyUsingRotator(
// write new admin keyring to tempdir so it can be verified before it's stored permanently
newAdminKeyringTmpfile := filepath.Join(tmpDir, cephclient.AdminUsername+".keyring")
logger.Infof("temporarily storing rotated admin cephx keyring at %q for cluster in namespace %q", newAdminKeyringTmpfile, clusterInfo.Namespace)
log.NamespacedInfo(clusterInfo.Namespace, logger, "temporarily storing rotated admin cephx keyring at %q", newAdminKeyringTmpfile)
err = cephclient.WriteKeyring(newAdminKeyringTmpfile, newAdminKeyring)
if err != nil {
return errors.Wrapf(err, "failed to write admin rotator keyring to %q", newAdminKeyringTmpfile)
@@ -345,7 +346,7 @@ func finalizeAdminKeyRotation(
// clean up tmpDir now that it's no longer useful for recovery
if err := os.RemoveAll(tmpDir); err != nil {
// most likely, operator pod got rescheduled to a different node where dir isn't present
logger.Infof("non-critical failure removing admin cephx key rotation temp dir %q. %v", tmpDir, err)
log.NamespacedInfo(clusterInfo.Namespace, logger, "non-critical failure removing admin cephx key rotation temp dir %q. %v", tmpDir, err)
}
// delete the `rook-ceph-admin-rotator-keyring` secret (store.Delete)
@@ -361,7 +362,7 @@ func finalizeAdminKeyRotation(
if err != nil {
// this should be rare, and failure here doesn't need to block finalization of admin key
// rotation. status will be missing ceph version, but cluster can continue to reconcile
logger.Errorf("non-critical failure to determine missing ceph version of cluster in namespace %q for admin cephx key status", clusterInfo.Namespace)
log.NamespacedError(clusterInfo.Namespace, logger, "non-critical failure to determine missing ceph version of cluster for admin cephx key status")
} else {
clusterInfo.CephVersion = ver
}
@@ -375,7 +376,7 @@ func finalizeAdminKeyRotation(
// restart Rook to force health checkers to restart with latest clusterInfo
// do this even during rotation recovery since the health checkers could still be running
logger.Infof("restarting rook operator after successful admin cephx key rotation for cluster in namespace %q", clusterInfo.Namespace)
log.NamespacedInfo(clusterInfo.Namespace, logger, "restarting rook operator after successful admin cephx key rotation")
reloadManagerFunc()
return errSuccessfulAdminKeyRotation
@@ -390,7 +391,7 @@ func updateCephClusterAdminCephxStatus(clusterdCtx *clusterd.Context, clusterInf
}
cephx := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, clusterInfo.CephVersion, cluster.Status.Cephx.Admin)
cluster.Status.Cephx.Admin = cephx
logger.Debugf("updating admin key cephx status to %+v", cephx)
log.NamespacedDebug(clusterInfo.Namespace, logger, "updating admin key cephx status to %+v", cephx)
if err := reporting.UpdateStatus(clusterdCtx.Client, cluster); err != nil {
return errors.Wrap(err, "failed to update admin key cephx status")
}
+9 -8
View File
@@ -34,6 +34,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/file/mirror"
"github.com/rook/rook/pkg/operator/ceph/object"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
batch "k8s.io/api/batch/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -59,10 +60,10 @@ var (
)
func (c *ClusterController) startClusterCleanUp(context context.Context, cluster *cephv1.CephCluster, cephHosts []string, monSecret, clusterFSID string) {
logger.Infof("starting clean up for cluster %q", cluster.Name)
log.NamespacedInfo(cluster.Namespace, logger, "starting clean up for cluster %q", cluster.Name)
err := c.waitForCephDaemonCleanUp(context, cluster, time.Duration(clusterCleanUpPolicyRetryInterval)*time.Second)
if err != nil {
logger.Errorf("failed to wait till ceph daemons are destroyed. %v", err)
log.NamespacedError(cluster.Namespace, logger, "failed to wait till ceph daemons are destroyed. %v", err)
return
}
@@ -71,7 +72,7 @@ func (c *ClusterController) startClusterCleanUp(context context.Context, cluster
func (c *ClusterController) startCleanUpJobs(cluster *cephv1.CephCluster, cephHosts []string, monSecret, clusterFSID string) {
for _, hostName := range cephHosts {
logger.Infof("starting clean up job on node %q", hostName)
log.NamespacedInfo(cluster.Namespace, logger, "starting clean up job on node %q", hostName)
jobName := k8sutil.TruncateNodeNameForJob("cluster-cleanup-job-%s", hostName)
podSpec := c.cleanUpJobTemplateSpec(cluster, monSecret, clusterFSID)
podSpec.Spec.NodeSelector = map[string]string{k8sutil.LabelHostname(): hostName}
@@ -93,7 +94,7 @@ func (c *ClusterController) startCleanUpJobs(cluster *cephv1.CephCluster, cephHo
cephv1.GetCleanupLabels(cluster.Spec.Labels).ApplyToObjectMeta(&job.ObjectMeta)
if err := k8sutil.RunReplaceableJob(c.OpManagerCtx, c.context.Clientset, job, true); err != nil {
logger.Errorf("failed to run cluster clean up job on node %q. %v", hostName, err)
log.NamespacedError(cluster.Namespace, logger, "failed to run cluster clean up job on node %q. %v", hostName, err)
}
}
}
@@ -205,7 +206,7 @@ func getCleanupPlacement(c cephv1.ClusterSpec) cephv1.Placement {
}
func (c *ClusterController) waitForCephDaemonCleanUp(context context.Context, cluster *cephv1.CephCluster, retryInterval time.Duration) error {
logger.Infof("waiting for all the ceph daemons to be cleaned up in the cluster %q", cluster.Namespace)
log.NamespacedInfo(cluster.Namespace, logger, "waiting for all the ceph daemons to be cleaned up in the cluster %q", cluster.Namespace)
for {
select {
case <-time.After(retryInterval):
@@ -215,11 +216,11 @@ func (c *ClusterController) waitForCephDaemonCleanUp(context context.Context, cl
}
if len(cephHosts) == 0 {
logger.Info("all ceph daemons are cleaned up")
log.NamespacedInfo(cluster.Namespace, logger, "all ceph daemons are cleaned up")
return nil
}
logger.Debugf("waiting for ceph daemons in cluster %q to be cleaned up. Retrying in %q",
log.NamespacedDebug(cluster.Namespace, logger, "waiting for ceph daemons in cluster %q to be cleaned up. Retrying in %q",
cluster.Namespace, retryInterval.String())
case <-context.Done():
return errors.Errorf("cancelling the host cleanup job. %s", context.Err())
@@ -250,7 +251,7 @@ func (c *ClusterController) getCephHosts(namespace string) ([]string, error) {
fmt.Fprintf(&b, "%s: %d. ", app, len(podList.Items))
}
logger.Infof("existing ceph daemons in the namespace %q. %s", namespace, b.String())
log.NamespacedInfo(namespace, logger, "existing ceph daemons in the namespace %q. %s", namespace, b.String())
for nodeName := range nodeNameList {
podHostName, err := k8sutil.GetNodeHostName(c.OpManagerCtx, c.context.Clientset, nodeName)
+33 -32
View File
@@ -46,6 +46,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/reporting"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
rookversion "github.com/rook/rook/pkg/version"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -104,7 +105,7 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep
// Execute actions before the monitors are up and running, if needed during upgrades.
// These actions would be skipped in a new cluster.
logger.Debug("monitors are about to reconcile, executing pre actions")
log.NamespacedDebug(c.Namespace, logger, "monitors are about to reconcile, executing pre actions")
err = c.preMonStartupActions(cephVersion)
if err != nil {
return errors.Wrap(err, "failed to execute actions before reconciling the ceph monitors")
@@ -132,7 +133,7 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep
}
// Execute actions after the monitors are up and running
logger.Debug("monitors are up and running, executing post actions")
log.NamespacedDebug(c.Namespace, logger, "monitors are up and running, executing post actions")
err = c.postMonStartupActions()
if err != nil {
return errors.Wrap(err, "failed to execute post actions after all the ceph monitors started")
@@ -147,7 +148,7 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep
}
// Execute actions after the managers are up and running
logger.Debug("managers are up and running, executing post actions")
log.NamespacedDebug(c.Namespace, logger, "managers are up and running, executing post actions")
err = c.postMgrStartupActions()
if err != nil {
return errors.Wrap(err, "failed to execute post actions after all the ceph managers started")
@@ -168,7 +169,7 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep
}
}
logger.Infof("done reconciling ceph cluster in namespace %q", c.Namespace)
log.NamespacedInfo(c.Namespace, logger, "done reconciling ceph cluster")
// We should be done updating by now
if c.isUpgrade {
@@ -186,7 +187,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error {
cleanDataDirHostPath := path.Clean(cluster.Spec.DataDirHostPath)
for _, b := range disallowedHostDirectories {
if cleanDataDirHostPath == b {
logger.Errorf("dataDirHostPath (given: %q) must not be used, conflicts with %q internal path", cluster.Spec.DataDirHostPath, b)
log.NamespacedError(cluster.Namespace, logger, "dataDirHostPath (given: %q) must not be used, conflicts with %q internal path", cluster.Spec.DataDirHostPath, b)
return nil
}
}
@@ -202,7 +203,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error {
clusterInfo, _, _, err := controller.LoadClusterInfo(c.context, c.OpManagerCtx, cluster.Namespace, cluster.Spec)
if err != nil {
if errors.Is(err, controller.ClusterInfoNoClusterNoSecret) {
logger.Info("clusterInfo not yet found, must be a new cluster.")
log.NamespacedInfo(cluster.Namespace, logger, "clusterInfo not yet found, must be a new cluster.")
// ClusterInfoNoClusterNoSecret should return nil clusterInfo, but this is a
// mandatory condition for recoverPriorAdminCephxKeyRotation() below, so ensure it
clusterInfo = nil
@@ -216,7 +217,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error {
if cluster.mons.ClusterInfo == nil {
// ClusterInfo stored in cluster.mons can be lost from the object stored in the
// clusterMap during admin key rotation corner cases. rehydrate it if needed
logger.Debugf("applying missing ClusterInfo to tracked mons info for cluster in namespace %q", cluster.Namespace)
log.NamespacedDebug(cluster.Namespace, logger, "applying missing ClusterInfo to tracked mons info")
cluster.mons.ClusterInfo = clusterInfo
}
}
@@ -301,7 +302,7 @@ func (c *ClusterController) configureLocalCephCluster(cluster *cluster) error {
// Validate the cluster Specs
func preClusterStartValidation(cluster *cluster) error {
if cluster.Spec.Mon.Count == 0 {
logger.Warningf("mon count should be at least 1, will use default value of %d", mon.DefaultMonCount)
log.NamespacedWarning(cluster.Namespace, logger, "mon count should be at least 1, will use default value of %d", mon.DefaultMonCount)
cluster.Spec.Mon.Count = mon.DefaultMonCount
}
if !cluster.Spec.Mon.AllowMultiplePerNode {
@@ -328,7 +329,7 @@ func preClusterStartValidation(cluster *cluster) error {
}
}
logger.Debug("cluster spec successfully validated")
log.NamespacedDebug(cluster.Namespace, logger, "cluster spec successfully validated")
return nil
}
@@ -391,7 +392,7 @@ func (c *cluster) replaceDefaultReplicationRule(newRoot string) error {
// CRUSH root
// in both cases, we cannot do anything about it either way and
// we’ll assume that the user knows what they’re doing.
logger.Warning("replicated_rule is in use, not replaced")
log.NamespacedWarning(c.Namespace, logger, "replicated_rule is in use, not replaced")
return nil
}
// the error does not refer to EBUSY -> return as error
@@ -432,7 +433,7 @@ func (c *cluster) removeDefaultCrushRoot() error {
//
// in all cases, we cannot do anything about it either way and
// we’ll assume that the user knows what they’re doing.
logger.Debug("default is not empty or is still in use, not removed")
log.NamespacedDebug(c.Namespace, logger, "default is not empty or is still in use, not removed")
return nil
}
}
@@ -446,19 +447,19 @@ func (c *cluster) removeDefaultCrushRoot() error {
// Those objects may interfere with the normal operation of the cluster.
// Note that errors which indicate that the objects are in use are ignored and the objects will continue to exist in that case.
func (c *cluster) replaceDefaultCrushMap(newRoot string) (err error) {
logger.Info("creating new CRUSH root if it does not exist")
log.NamespacedInfo(c.Namespace, logger, "creating new CRUSH root if it does not exist")
err = c.createCrushRoot(newRoot)
if err != nil {
return errors.Wrap(err, "failed to create CRUSH root")
}
logger.Info("replacing default replicated_rule CRUSH rule for use of non-default CRUSH root")
log.NamespacedInfo(c.Namespace, logger, "replacing default replicated_rule CRUSH rule for use of non-default CRUSH root")
err = c.replaceDefaultReplicationRule(newRoot)
if err != nil {
return errors.Wrap(err, "failed to replace default rule")
}
logger.Info("replacing default CRUSH node if applicable")
log.NamespacedInfo(c.Namespace, logger, "replacing default CRUSH node if applicable")
err = c.removeDefaultCrushRoot()
if err != nil {
return errors.Wrap(err, "failed to remove default CRUSH root")
@@ -589,12 +590,12 @@ func (c *cluster) configureStorageSettings() error {
func (c *cluster) setClusterFullRatio(ratioCommand string, desiredRatio *float64, actualRatio float64) error {
if !shouldUpdateFloatSetting(desiredRatio, actualRatio) {
if desiredRatio != nil {
logger.Infof("desired value %s=%.2f is already set", ratioCommand, *desiredRatio)
log.NamespacedInfo(c.Namespace, logger, "desired value %s=%.2f is already set", ratioCommand, *desiredRatio)
}
return nil
}
desiredStringVal := fmt.Sprintf("%.2f", *desiredRatio)
logger.Infof("updating %s from %.2f to %s", ratioCommand, actualRatio, desiredStringVal)
log.NamespacedInfo(c.Namespace, logger, "updating %s from %.2f to %s", ratioCommand, actualRatio, desiredStringVal)
args := []string{"osd", ratioCommand, desiredStringVal}
cephCmd := client.NewCephCommand(c.context, c.ClusterInfo, args)
output, err := cephCmd.Run()
@@ -648,7 +649,7 @@ func (c *cluster) reportTelemetry() {
}
func reportClusterTelemetry(c *cluster) {
logger.Info("reporting cluster telemetry")
log.NamespacedInfo(c.Namespace, logger, "reporting cluster telemetry")
// Identify this as a rook cluster for Ceph telemetry by setting the Rook version.
telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.RookVersionKey, rookversion.Version)
@@ -656,7 +657,7 @@ func reportClusterTelemetry(c *cluster) {
// Report the K8s version
serverVersion, err := c.context.Clientset.Discovery().ServerVersion()
if err != nil {
logger.Warningf("failed to report the K8s server version. %v", err)
log.NamespacedWarning(c.Namespace, logger, "failed to report the K8s server version. %v", err)
} else {
telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.K8sVersionKey, serverVersion.String())
}
@@ -699,13 +700,13 @@ func reportClusterTelemetry(c *cluster) {
}
func reportNodeTelemetry(c *cluster) {
logger.Info("reporting node telemetry")
log.NamespacedInfo(c.Namespace, logger, "reporting node telemetry")
operatorNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
// Report the K8sNodeCount
nodelist, err := c.context.Clientset.CoreV1().Nodes().List(c.ClusterInfo.Context, metav1.ListOptions{})
if err != nil {
logger.Warningf("failed to report the K8s node count. %v", err)
log.NamespacedWarning(c.Namespace, logger, "failed to report the K8s node count. %v", err)
} else {
telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.K8sNodeCount, strconv.Itoa(len(nodelist.Items)))
}
@@ -717,7 +718,7 @@ func reportNodeTelemetry(c *cluster) {
listoption := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, nodedaemon.CrashCollectorAppName)}
cephNodeList, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(c.ClusterInfo.Context, listoption)
if err != nil {
logger.Warningf("failed to report the ceph node count. %v", err)
log.NamespacedWarning(c.Namespace, logger, "failed to report the ceph node count. %v", err)
} else {
telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.CephNodeCount, strconv.Itoa(len(cephNodeList.Items)))
}
@@ -726,7 +727,7 @@ func reportNodeTelemetry(c *cluster) {
listoption := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, csi.CsiRBDPlugin)}
cephRbdNodelist, err := c.context.Clientset.CoreV1().Pods(operatorNamespace).List(c.ClusterInfo.Context, listoption)
if err != nil {
logger.Warningf("failed to report the ceph rbd node count. %v", err)
log.NamespacedWarning(c.Namespace, logger, "failed to report the ceph rbd node count. %v", err)
} else {
telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.RBDNodeCount, strconv.Itoa(len(cephRbdNodelist.Items)))
}
@@ -735,7 +736,7 @@ func reportNodeTelemetry(c *cluster) {
listoption = metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, csi.CsiCephFSPlugin)}
cephFSNodelist, err := c.context.Clientset.CoreV1().Pods(operatorNamespace).List(c.ClusterInfo.Context, listoption)
if err != nil {
logger.Warningf("failed to report the ceph cephfs node count. %v", err)
log.NamespacedWarning(c.Namespace, logger, "failed to report the ceph cephfs node count. %v", err)
} else {
telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.CephFSNodeCount, strconv.Itoa(len(cephFSNodelist.Items)))
}
@@ -744,7 +745,7 @@ func reportNodeTelemetry(c *cluster) {
listoption = metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, csi.CsiNFSPlugin)}
cephNFSNodelist, err := c.context.Clientset.CoreV1().Pods(operatorNamespace).List(c.ClusterInfo.Context, listoption)
if err != nil {
logger.Warningf("failed to report the ceph nfs node count. %v", err)
log.NamespacedWarning(c.Namespace, logger, "failed to report the ceph nfs node count. %v", err)
} else {
telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.NFSNodeCount, strconv.Itoa(len(cephNFSNodelist.Items)))
}
@@ -764,7 +765,7 @@ func (c *cluster) configureMsgr2() error {
encryptionEnabled := c.Spec.NetworkEncryptionEnabled()
if encryptionEnabled {
logger.Infof("setting msgr2 encryption mode to %q", encryptionSetting)
log.NamespacedInfo(c.Namespace, logger, "setting msgr2 encryption mode to %q", encryptionSetting)
if err := monStore.SetAll("global", encryptionGlobalConfigSettings); err != nil {
return err
}
@@ -797,7 +798,7 @@ func (c *cluster) configureMsgr2() error {
globalConfigSettings := map[string]string{
"ms_osd_compress_mode": "force",
}
logger.Infof("setting msgr2 compression mode to %q", "force")
log.NamespacedInfo(c.Namespace, logger, "setting msgr2 compression mode to %q", "force")
if err := monStore.SetAll("global", globalConfigSettings); err != nil {
return err
}
@@ -819,9 +820,9 @@ func (c *cluster) fetchCephConfigFromSecrets() (map[string]map[string]string, er
key, module, selector.LocalObjectReference.Name, err)
}
logger.Debugf("setting Ceph config key %q in module %q from secret %q",
log.NamespacedDebug(c.Namespace, logger, "setting Ceph config key %q in module %q from secret %q",
key, module, selector.LocalObjectReference.Name)
logger.Tracef("setting Ceph config key %q in module %q to value %q from secret %q",
log.NamespacedTrace(c.Namespace, logger, "setting Ceph config key %q in module %q to value %q from secret %q",
key, module, val, selector.LocalObjectReference.Name)
result[module][key] = val
}
@@ -851,7 +852,7 @@ func (c *cluster) fetchSecretValue(selector v1.SecretKeySelector) (string, error
func initClusterCephxStatus(c *cluster) error {
initErr := c.ClusterInfo.IsInitialized()
if initErr == nil {
logger.Debugf("not setting uninitialized cephx status on already initialized CephCluster in namespace %q", c.Namespace)
log.NamespacedDebug(c.Namespace, logger, "not setting uninitialized cephx status on already initialized CephCluster")
return nil
}
if c.ClusterInfo.Context.Err() != nil {
@@ -874,7 +875,7 @@ func initClusterCephxStatus(c *cluster) error {
}
uninitializedStatus := keyring.UninitializedCephxStatus()
logger.Infof("initializing cephx status for CephCluster in namespace %q", c.Namespace)
log.NamespacedInfo(c.Namespace, logger, "initializing cephx status")
clusterObj.Status.Cephx = cephv1.ClusterCephxStatus{
Mon: uninitializedStatus,
Mgr: uninitializedStatus,
@@ -901,9 +902,9 @@ func (c *cluster) deleteBootstrapKeys() {
for _, bootstrapkey := range bootstrapKeys {
err := client.AuthDelete(c.context, c.ClusterInfo, bootstrapkey)
if err != nil {
logger.Debugf("failed to delete bootstrap key %q in the namespace %q. Error: %v", bootstrapkey, c.Namespace, err)
log.NamespacedDebug(c.Namespace, logger, "failed to delete bootstrap key %q. Error: %v", bootstrapkey, err)
} else {
logger.Infof("successfully deleted bootstrap key %q in the namespace %q", bootstrapkey, c.Namespace)
log.NamespacedInfo(c.Namespace, logger, "successfully deleted bootstrap key %q", bootstrapkey)
}
}
}
+15 -14
View File
@@ -33,6 +33,7 @@ import (
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/ceph/csi"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
v1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -65,7 +66,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
if cluster.Spec.CephVersion.Image == "" {
err = mon.WriteConnectionConfig(c.context, cluster.ClusterInfo)
if err != nil {
logger.Errorf("failed to write config. attempting to continue. %v", err)
log.NamespacedError(cluster.Namespace, logger, "failed to write config. attempting to continue. %v", err)
}
}
@@ -81,13 +82,13 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
// If we don't do this, daemons will never start, waiting forever for this configmap to be present
//
// Only do this when doing a bit of management...
logger.Infof("creating %q configmap", k8sutil.ConfigOverrideName)
log.NamespacedInfo(cluster.Namespace, logger, "creating %q configmap", k8sutil.ConfigOverrideName)
err = populateConfigOverrideConfigMap(c.context, cluster.namespacedName.Namespace, cluster.ClusterInfo.OwnerInfo, cluster.clusterMetadata)
if err != nil {
return errors.Wrap(err, "failed to populate config override config map")
}
logger.Infof("creating %q secret", config.StoreName)
log.NamespacedInfo(cluster.Namespace, logger, "creating %q secret", config.StoreName)
err = config.GetStore(c.context, cluster.namespacedName.Namespace, cluster.ClusterInfo.OwnerInfo).CreateOrUpdate(cluster.ClusterInfo)
if err != nil {
return errors.Wrap(err, "failed to update the global config")
@@ -98,7 +99,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
if err := cluster.ClusterInfo.IsInitialized(); err != nil {
return errors.Wrap(err, "the cluster identity was not established")
}
logger.Info("external cluster identity established")
log.NamespacedInfo(cluster.Namespace, logger, "external cluster identity established")
// Create CSI Secrets only if the user has provided the admin key
if cluster.ClusterInfo.CephCred.Username == client.AdminUsername {
@@ -117,7 +118,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
cluster.Spec.Network.Connections = &cephv1.ConnectionsSpec{}
}
cluster.Spec.Network.Connections.RequireMsgr2 = true
logger.Debugf("a v2 port was found for a mon endpoint, so msgr2 is required")
log.NamespacedDebug(cluster.Namespace, logger, "a v2 port was found for a mon endpoint, so msgr2 is required")
break
}
}
@@ -136,7 +137,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
if err != nil {
return errors.Wrap(err, "failed to update csi cluster config")
}
logger.Info("successfully updated csi config map")
log.NamespacedInfo(cluster.Namespace, logger, "successfully updated csi config map")
// Create Crash Collector Secret
if !cluster.Spec.CrashCollector.Disable {
@@ -177,7 +178,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
}
if csi.EnableCSIOperator() {
logger.Info("create cephConnection and defaultClientProfile for external mode")
log.NamespacedInfo(cluster.Namespace, logger, "create cephConnection and defaultClientProfile for external mode")
err = csi.CreateUpdateCephConnection(c.context.Client, cluster.ClusterInfo, *cluster.Spec)
if err != nil {
return errors.Wrap(err, "failed to create/update cephConnection")
@@ -202,7 +203,7 @@ func purgeExternalCluster(clientset kubernetes.Interface, namespace string) {
for _, cm := range cmsToDelete {
err := clientset.CoreV1().ConfigMaps(namespace).Delete(ctx, cm, metav1.DeleteOptions{})
if err != nil && !kerrors.IsNotFound(err) {
logger.Errorf("failed to delete config map %q. %v", cm, err)
log.NamespacedError(namespace, logger, "failed to delete config map %q. %v", cm, err)
}
}
@@ -219,7 +220,7 @@ func purgeExternalCluster(clientset kubernetes.Interface, namespace string) {
for _, secret := range secretsToDelete {
err := clientset.CoreV1().Secrets(namespace).Delete(ctx, secret, metav1.DeleteOptions{})
if err != nil && !kerrors.IsNotFound(err) {
logger.Errorf("failed to delete secret %q. %v", secret, err)
log.NamespacedError(namespace, logger, "failed to delete secret %q. %v", secret, err)
}
}
}
@@ -255,12 +256,12 @@ func (c *ClusterController) configureExternalClusterMonitoring(context *clusterd
if err != nil {
return err
}
logger.Info("creating mgr external monitoring service")
log.NamespacedInfo(cluster.Namespace, logger, "creating mgr external monitoring service")
_, err = k8sutil.CreateOrUpdateService(cluster.ClusterInfo.Context, context.Clientset, cluster.Namespace, service)
if err != nil && !kerrors.IsAlreadyExists(err) {
return errors.Wrap(err, "failed to create or update mgr service")
}
logger.Info("mgr external metrics service created")
log.NamespacedInfo(cluster.Namespace, logger, "mgr external metrics service created")
// Configure external metrics endpoint
err = opcontroller.ConfigureExternalMetricsEndpoint(context, cluster.Spec.Monitoring, cluster.ClusterInfo, cluster.ownerInfo)
@@ -269,13 +270,13 @@ func (c *ClusterController) configureExternalClusterMonitoring(context *clusterd
}
// Deploy external ServiceMonitor
logger.Info("creating external service monitor")
log.NamespacedInfo(cluster.Namespace, logger, "creating external service monitor")
// servicemonitor takes some metadata from the service for easy mapping
err = manager.EnableServiceMonitor()
if err != nil {
logger.Errorf("failed to enable external service monitor. %v", err)
log.NamespacedError(cluster.Namespace, logger, "failed to enable external service monitor. %v", err)
} else {
logger.Info("external service monitor created")
log.NamespacedInfo(cluster.Namespace, logger, "external service monitor created")
}
return nil
}
+17 -16
View File
@@ -39,6 +39,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/csi"
"github.com/rook/rook/pkg/operator/ceph/reporting"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
@@ -284,7 +285,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
err := r.client.Get(r.opManagerContext, request.NamespacedName, cephCluster)
if err != nil {
if kerrors.IsNotFound(err) {
logger.Debug("cephCluster resource not found. Ignoring since object must be deleted.")
log.NamespacedDebug(request.Namespace, logger, "cephCluster resource not found. Ignoring since object must be deleted.")
return reconcile.Result{}, *cephCluster, nil
}
// Error reading the object - requeue the request.
@@ -297,7 +298,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
return reconcile.Result{}, *cephCluster, errors.Wrap(err, "failed to add finalizer")
}
if generationUpdated {
logger.Infof("reconciling the ceph cluster %q after adding finalizer", cephCluster.Name)
log.NamespacedInfo(request.Namespace, logger, "reconciling the ceph cluster %q after adding finalizer", cephCluster.Name)
return reconcile.Result{}, *cephCluster, nil
}
@@ -317,7 +318,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
// started). So we would get messages from the other go routine printing a message
// repeatedly that the context is cancelled.
if errors.Is(r.opManagerContext.Err(), context.Canceled) {
logger.Info("context cancelled, exiting reconcile")
log.NamespacedInfo(request.Namespace, logger, "context cancelled, exiting reconcile")
return reconcile.Result{}, *cephCluster, nil
}
@@ -357,7 +358,7 @@ func (r *ReconcileCephCluster) reconcileDelete(cephCluster *cephv1.CephCluster)
if cephCluster.Spec.CleanupPolicy.HasDataDirCleanPolicy() && !cephCluster.Spec.External.Enable {
monSecret, clusterFSID, err := r.clusterController.getCleanUpDetails(&cephCluster.Spec, cephCluster.Namespace)
if err != nil {
logger.Warningf("failed to get mon secret. skip cluster cleanup. remove finalizer. %v", err)
log.NamespacedWarning(cephCluster.Namespace, logger, "failed to get mon secret. skip cluster cleanup. remove finalizer. %v", err)
doCleanup = false
}
@@ -400,7 +401,7 @@ func NewClusterController(context *clusterd.Context, rookImage string) *ClusterC
func (c *ClusterController) reconcileCephCluster(clusterObj *cephv1.CephCluster, ownerInfo *k8sutil.OwnerInfo) error {
if clusterObj.Spec.CleanupPolicy.HasDataDirCleanPolicy() {
logger.Infof("skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set", clusterObj.Name, clusterObj.Namespace)
log.NamespacedInfo(clusterObj.Namespace, logger, "skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set", clusterObj.Name, clusterObj.Namespace)
return nil
}
var clustr *cluster
@@ -425,7 +426,7 @@ func (c *ClusterController) reconcileCephCluster(clusterObj *cephv1.CephCluster,
clustr.Spec = &clusterObj.Spec
c.clusterMap.Store(clustr.Namespace, clustr)
logger.Infof("reconciling ceph cluster in namespace %q", clustr.namespacedName.Namespace)
log.NamedInfo(clustr.namespacedName, logger, "reconciling ceph cluster")
// Start the main ceph cluster orchestration
return c.initializeCluster(clustr)
@@ -438,14 +439,14 @@ func (c *ClusterController) requestClusterDelete(clusterObj *cephv1.CephCluster)
if rawCluster, ok := c.clusterMap.Load(clusterObj.Namespace); ok {
existingCluster = rawCluster.(*cluster)
if existingCluster.namespacedName.Name != clusterObj.Name {
logger.Errorf("skipping deletion of CephCluster %q. CephCluster CR %q already exists in this namespace. only one cluster cr per namespace is supported.",
log.NamespacedError(nsName, logger, "skipping deletion of CephCluster %q. CephCluster CR %q already exists in this namespace. only one cluster cr per namespace is supported.",
nsName, existingCluster.namespacedName.Name)
return reconcile.Result{}, nil // do not requeue the delete
}
}
_, err := c.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(c.OpManagerCtx, "networkfences.csiaddons.openshift.io", metav1.GetOptions{})
if err == nil {
logger.Info("removing networkFence if matching cephCluster UID found")
log.NamespacedInfo(clusterObj.Namespace, logger, "removing networkFence if matching cephCluster UID found")
networkFenceList := &addonsv1alpha1.NetworkFenceList{}
labelSelector := labels.SelectorFromSet(map[string]string{
networkFenceLabel: string(clusterObj.GetUID()),
@@ -476,7 +477,7 @@ func (c *ClusterController) requestClusterDelete(clusterObj *cephv1.CephCluster)
}
}
logger.Infof("cleaning up CephCluster %q", nsName)
log.NamespacedInfo(clusterObj.Namespace, logger, "cleaning up CephCluster %q", nsName)
if existingCluster != nil {
// We used to stop the bucket controller here but when we get a DELETE event for the CephCluster
// we will reload the CRD manager anyway so the bucket controller go routine will be stopped
@@ -495,7 +496,7 @@ func (c *ClusterController) requestClusterDelete(clusterObj *cephv1.CephCluster)
}
if clusterObj.Spec.CleanupPolicy.AllowUninstallWithVolumes {
logger.Info("skipping check for existing PVs as allowUninstallWithVolumes is set to true")
log.NamespacedInfo(clusterObj.Namespace, logger, "skipping check for existing PVs as allowUninstallWithVolumes is set to true")
} else {
err := c.checkIfVolumesExist(clusterObj)
if err != nil {
@@ -510,7 +511,7 @@ func (c *ClusterController) requestClusterDelete(clusterObj *cephv1.CephCluster)
// Delete keys from KMS
err := c.deleteOSDEncryptionKeyFromKMS(clusterObj)
if err != nil {
logger.Errorf("failed to delete osd encryption keys for CephCluster %q from kms; deletion will continue. %v", nsName, err)
log.NamespacedError(clusterObj.Namespace, logger, "failed to delete osd encryption keys for CephCluster %q from kms; deletion will continue. %v", nsName, err)
}
}
@@ -534,7 +535,7 @@ func (c *ClusterController) checkIfVolumesExist(cluster *cephv1.CephCluster) err
func (c *ClusterController) csiVolumesAllowForDeletion(cluster *cephv1.CephCluster) error {
drivers := []string{csi.CephFSDriverName, csi.RBDDriverName}
logger.Infof("checking any PVC created by drivers %q and %q with clusterID %q", csi.CephFSDriverName, csi.RBDDriverName, cluster.Namespace)
log.NamespacedInfo(cluster.Namespace, logger, "checking any PVC created by drivers %q and %q with clusterID %q", csi.CephFSDriverName, csi.RBDDriverName, cluster.Namespace)
// check any PV is created in this cluster
attachmentsExist, err := c.checkPVPresentInCluster(drivers, cluster.Namespace)
if err != nil {
@@ -542,7 +543,7 @@ func (c *ClusterController) csiVolumesAllowForDeletion(cluster *cephv1.CephClust
}
// no PVC created in this cluster
if !attachmentsExist {
logger.Infof("no volume attachments for cluster %q", cluster.Namespace)
log.NamespacedInfo(cluster.Namespace, logger, "no volume attachments for cluster %q", cluster.Namespace)
return nil
}
@@ -557,7 +558,7 @@ func (c *ClusterController) checkPVPresentInCluster(drivers []string, clusterID
for _, p := range pv.Items {
if p.Spec.CSI == nil {
logger.Errorf("Spec.CSI is nil for PV %q", p.Name)
log.NamespacedError(clusterID, logger, "Spec.CSI is nil for PV %q", p.Name)
continue
}
if p.Spec.CSI.VolumeAttributes["clusterID"] == clusterID {
@@ -600,7 +601,7 @@ func (r *ReconcileCephCluster) removeFinalizer(client client.Client, name types.
err := client.Get(r.opManagerContext, name, obj)
if err != nil {
if kerrors.IsNotFound(err) {
logger.Debugf("%s resource not found. Ignoring since object must be deleted.", name.Name)
log.NamedDebug(name, logger, "%s resource not found. Ignoring since object must be deleted.", name.Name)
return nil
}
return errors.Wrapf(err, "failed to retrieve %q to remove finalizer", name.Name)
@@ -666,7 +667,7 @@ func (c *ClusterController) deleteOSDEncryptionKeyFromKMS(currentCluster *cephv1
// Generate and store the encrypted key in whatever KMS is configured
err = kmsConfig.DeleteSecret(osdPVC.Name)
if err != nil {
logger.Errorf("failed to delete secret. %v", err)
log.NamespacedError(currentCluster.Namespace, logger, "failed to delete secret. %v", err)
continue
}
}
+4 -3
View File
@@ -22,6 +22,7 @@ import (
"github.com/pkg/errors"
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
"github.com/rook/rook/pkg/util/log"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -89,7 +90,7 @@ func (c *Cluster) generateKeyring(m *mgrConfig) (string, error) {
}
if c.shouldRotateCephxKeys {
logger.Infof("rotating cephx key for mgr daemon %q in the namespace %q", m.ResourceName, c.clusterInfo.Namespace)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "rotating cephx key for mgr daemon %q in the namespace %q", m.ResourceName, c.clusterInfo.Namespace)
newKey, err := s.RotateKey(user)
if err != nil {
return "", errors.Wrapf(err, "failed to rotate cephx key for mgr daemon %q", m.ResourceName)
@@ -101,9 +102,9 @@ func (c *Cluster) generateKeyring(m *mgrConfig) (string, error) {
err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Delete(c.clusterInfo.Context, m.ResourceName, metav1.DeleteOptions{})
if err != nil {
if kerrors.IsNotFound(err) {
logger.Debugf("legacy mgr key %q is already removed", m.ResourceName)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "legacy mgr key %q is already removed", m.ResourceName)
} else {
logger.Warningf("legacy mgr key %q could not be removed. %v", m.ResourceName, err)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "legacy mgr key %q could not be removed. %v", m.ResourceName, err)
}
}
+16 -15
View File
@@ -32,6 +32,7 @@ import (
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util"
"github.com/rook/rook/pkg/util/exec"
"github.com/rook/rook/pkg/util/log"
v1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -91,7 +92,7 @@ func (c *Cluster) configureDashboardModules() error {
}
} else {
if err := client.MgrDisableModule(c.context, c.clusterInfo, dashboardModuleName); err != nil {
logger.Errorf("failed to disable mgr dashboard module. %v", err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to disable mgr dashboard module. %v", err)
}
return nil
}
@@ -106,7 +107,7 @@ func (c *Cluster) configureDashboardModules() error {
return err
}
if secureRequiresRestart || configChanged {
logger.Info("dashboard config has changed. restarting the dashboard module")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "dashboard config has changed. restarting the dashboard module")
return c.restartMgrModule(dashboardModuleName)
}
return nil
@@ -131,16 +132,16 @@ func (c *Cluster) deleteManagerDaemonConfiguration() bool {
for _, key := range mgrKeysToDelete {
err := monStore.Delete(mgrDaemonID, key)
if err != nil {
logger.Errorf("failed to delete configuration entry %q %q, err: %v", mgrDaemonID, key, err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to delete configuration entry %q %q, err: %v", mgrDaemonID, key, err)
success = false
}
}
}
if success {
logger.Info("All per-daemon mgr configuration has been deleted successfully.")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "All per-daemon mgr configuration has been deleted successfully.")
} else {
logger.Error("At least one delete operation failed while trying to delete per-daemon mgr configuration.")
log.NamespacedError(c.clusterInfo.Namespace, logger, "At least one delete operation failed while trying to delete per-daemon mgr configuration.")
}
return success
@@ -236,10 +237,10 @@ func (c *Cluster) createSelfSignedCert() (bool, error) {
args := []string{"config-key", "get", "mgr/dashboard/crt"}
output, err := client.NewCephCommand(c.context, c.clusterInfo, args).RunWithTimeout(exec.CephCommandsTimeout)
if err == nil && len(output) > 0 {
logger.Info("dashboard is already initialized with a cert")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "dashboard is already initialized with a cert")
return true, nil
}
logger.Debugf("dashboard cert does not appear to exist. err=%v", err)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "dashboard cert does not appear to exist. err=%v", err)
// create a self-signed cert for the https connections
args = []string{"dashboard", "create-self-signed-cert"}
@@ -248,14 +249,14 @@ func (c *Cluster) createSelfSignedCert() (bool, error) {
for i := 0; i < 5; i++ {
_, err := client.NewCephCommand(c.context, c.clusterInfo, args).RunWithTimeout(exec.CephCommandsTimeout)
if err == context.DeadlineExceeded {
logger.Warning("cert creation timed out. trying again")
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "cert creation timed out. trying again")
continue
}
if err != nil {
exitCode, parsed := c.exitCode(err)
if parsed {
if exitCode == invalidArgErrorCode {
logger.Info("dashboard module is not ready yet. trying again")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "dashboard module is not ready yet. trying again")
time.Sleep(dashboardInitWaitTime)
continue
}
@@ -263,16 +264,16 @@ func (c *Cluster) createSelfSignedCert() (bool, error) {
return false, errors.Wrap(err, "failed to create self signed cert on mgr")
}
}
logger.Info("dashboard cert created")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "dashboard cert created")
return false, nil
}
logger.Info("dashboard cert creation exceeded retries")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "dashboard cert creation exceeded retries")
return false, nil
}
func (c *Cluster) setLoginCredentials(password string) error {
// Set the login credentials. Write the command/args to the debug log so we don't write the password by default to the log.
logger.Infof("setting ceph dashboard %q login creds", dashboardUsername)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "setting ceph dashboard %q login creds", dashboardUsername)
var args []string
// for latest Ceph versions
@@ -284,7 +285,7 @@ func (c *Cluster) setLoginCredentials(password string) error {
defer func() {
if err := os.Remove(file.Name()); err != nil {
logger.Errorf("failed to clean up dashboard password file %q. %v", file.Name(), err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to clean up dashboard password file %q. %v", file.Name(), err)
}
}()
@@ -314,14 +315,14 @@ func (c *Cluster) setLoginCredentials(password string) error {
return errors.Wrapf(err, "failed to set password for user %q", dashboardUsername)
}
logger.Info("successfully set ceph dashboard creds")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "successfully set ceph dashboard creds")
return nil
}
func (c *Cluster) getOrGenerateDashboardPassword() (string, error) {
secret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(c.clusterInfo.Context, dashboardPasswordName, metav1.GetOptions{})
if err == nil {
logger.Info("the dashboard secret was already generated")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "the dashboard secret was already generated")
return decodeSecret(secret)
}
if !kerrors.IsNotFound(err) {
+4 -3
View File
@@ -19,6 +19,7 @@ package mgr
import (
"github.com/pkg/errors"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
policyv1 "k8s.io/api/policy/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -63,13 +64,13 @@ func (c *Cluster) deleteMgrPDB() {
err := c.context.Client.Get(c.clusterInfo.Context, pdbRequest, mgrPDB)
if err != nil {
if !kerrors.IsNotFound(err) {
logger.Errorf("failed to get mgr pdb %q. %v", mgrPDBName, err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get mgr pdb %q. %v", mgrPDBName, err)
}
return
}
logger.Debugf("ensuring the mgr pdb %q is deleted", mgrPDBName)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "ensuring the mgr pdb %q is deleted", mgrPDBName)
err = c.context.Client.Delete(c.clusterInfo.Context, mgrPDB)
if err != nil {
logger.Errorf("failed to delete mgr pdb %q. %v", mgrPDBName, err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to delete mgr pdb %q. %v", mgrPDBName, err)
}
}
+37 -36
View File
@@ -36,6 +36,7 @@ import (
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/exec"
"github.com/rook/rook/pkg/util/log"
v1 "k8s.io/api/apps/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -115,7 +116,7 @@ func (c *Cluster) Start() error {
return errors.Wrap(err, "error checking pod memory")
}
logger.Infof("start running mgr")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "start running mgr")
daemonIDs := c.getDaemonIDs()
var deploymentsToWaitFor []*v1.Deployment
@@ -130,7 +131,7 @@ func (c *Cluster) Start() error {
}
if c.shouldRotateCephxKeys {
logger.Infof("cephx keys for mgr daemons in the namespace %q will be rotated", c.clusterInfo.Namespace)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "cephx keys for mgr daemons in the namespace %q will be rotated", c.clusterInfo.Namespace)
}
for _, daemonID := range daemonIDs {
@@ -167,7 +168,7 @@ func (c *Cluster) Start() error {
}
if mgrsToSkipReconcile.Has(daemonID) {
logger.Warningf("Skipping reconcile of mgr %q labeled with %q", daemonID, cephv1.SkipReconcileLabelKey)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "Skipping reconcile of mgr %q labeled with %q", daemonID, cephv1.SkipReconcileLabelKey)
continue
}
@@ -176,13 +177,13 @@ func (c *Cluster) Start() error {
if !kerrors.IsAlreadyExists(err) {
return errors.Wrapf(err, "failed to create mgr deployment %s", resourceName)
}
logger.Infof("deployment for mgr %s already exists. updating if needed", resourceName)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "deployment for mgr %s already exists. updating if needed", resourceName)
if err := updateDeploymentAndWait(c.context, c.clusterInfo, d, config.MgrType, mgrConfig.DaemonID, c.spec.SkipUpgradeChecks, c.spec.ContinueUpgradeAfterChecksEvenIfNotHealthy); err != nil {
logger.Errorf("failed to update mgr deployment %q. %v", resourceName, err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to update mgr deployment %q. %v", resourceName, err)
if c.spec.ContinueUpgradeAfterChecksEvenIfNotHealthy {
logger.Infof("continuing reconcile of ceph cluster in namespace %s after error waiting for mgr because continueUpgradeAfterChecksEvenIfNotHealthy is true", c.clusterInfo.Namespace)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "continuing reconcile of ceph cluster in namespace %s after error waiting for mgr because continueUpgradeAfterChecksEvenIfNotHealthy is true", c.clusterInfo.Namespace)
} else {
logger.Infof("stopping reconcile of ceph cluster in namespace %s after error waiting for mgr because continueUpgradeAfterChecksEvenIfNotHealthy is false", c.clusterInfo.Namespace)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "stopping reconcile of ceph cluster in namespace %s after error waiting for mgr because continueUpgradeAfterChecksEvenIfNotHealthy is false", c.clusterInfo.Namespace)
return errors.Wrapf(err, "failed to update mgr deployment %q", resourceName)
}
}
@@ -237,11 +238,11 @@ func (c *Cluster) removeExtraMgrs(daemonIDs []string) {
options := metav1.ListOptions{LabelSelector: "app=" + AppName}
mgrDeployments, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(c.clusterInfo.Context, options)
if err != nil {
logger.Warningf("failed to check for extra mgrs. %v", err)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to check for extra mgrs. %v", err)
return
}
if len(mgrDeployments.Items) == len(daemonIDs) {
logger.Debugf("expected number %d of mgrs found", len(daemonIDs))
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "expected number %d of mgrs found", len(daemonIDs))
return
}
@@ -263,9 +264,9 @@ func (c *Cluster) removeExtraMgrs(daemonIDs []string) {
if !found {
err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(c.clusterInfo.Context, mgrDeployment.Name, metav1.DeleteOptions{})
if err == nil {
logger.Infof("removed extra mgr %q", mgrDeployment.Name)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "removed extra mgr %q", mgrDeployment.Name)
} else {
logger.Warningf("failed to remove extra mgr %q. %v", mgrDeployment.Name, err)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to remove extra mgr %q. %v", mgrDeployment.Name, err)
}
}
}
@@ -278,7 +279,7 @@ func (c *Cluster) SetMgrRoleLabel(daemonNameToUpdate string, isActive bool) erro
LabelSelector: fmt.Sprintf("%s=%s,%s=%s", k8sutil.AppAttr, AppName, controller.DaemonIDLabel, daemonNameToUpdate),
})
if err != nil {
logger.Infof("cannot get pod for mgr daemon %s", daemonNameToUpdate)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "cannot get pod for mgr daemon %s", daemonNameToUpdate)
return err // force mrg_role update in the next call
}
@@ -295,7 +296,7 @@ func (c *Cluster) SetMgrRoleLabel(daemonNameToUpdate string, isActive bool) erro
labels := pod.GetLabels()
currMgrRole, mgrHasLabel := labels[mgrRoleLabelName]
if !mgrHasLabel || currMgrRole != newMgrRole {
logger.Infof("updating mgr_role label value of daemon %s to '%s'. New active mgr is %s.", daemonNameToUpdate, newMgrRole, daemonNameToUpdate)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "updating mgr_role label value of daemon %s to '%s'. New active mgr is %s.", daemonNameToUpdate, newMgrRole, daemonNameToUpdate)
labels[mgrRoleLabelName] = newMgrRole
pod.SetLabels(labels)
_, err = c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).Update(c.clusterInfo.Context, &pods.Items[i], metav1.UpdateOptions{})
@@ -346,7 +347,7 @@ func (c *Cluster) reconcileServices() error {
if err := c.EnableServiceMonitor(); err != nil {
// We don't want to return an error to block the cluster reconcile
// since monitoring is an optional service.
logger.Errorf("failed to enable service monitor, prometheus may need to be installed. %v", err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to enable service monitor, prometheus may need to be installed. %v", err)
}
}
@@ -361,7 +362,7 @@ func (c *Cluster) updateServiceSelectors() {
selector := metav1.ListOptions{LabelSelector: "app=rook-ceph-mgr"}
services, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).List(c.clusterInfo.Context, selector)
if err != nil {
logger.Errorf("failed to query mgr services to update labels: %v", err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to query mgr services to update labels: %v", err)
return
}
for i, service := range services.Items {
@@ -374,7 +375,7 @@ func (c *Cluster) updateServiceSelectors() {
// Check if the service has a DaemonIDLabel (legacy mgr HA implementation) and remove it
_, hasDaemonLabel := service.Spec.Selector[controller.DaemonIDLabel]
if hasDaemonLabel {
logger.Infof("removing %s selector label on mgr service %q", controller.DaemonIDLabel, service.Name)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "removing %s selector label on mgr service %q", controller.DaemonIDLabel, service.Name)
delete(service.Spec.Selector, controller.DaemonIDLabel)
updateService = true
}
@@ -382,16 +383,16 @@ func (c *Cluster) updateServiceSelectors() {
// In case the service doesn't have the new mgr_role DaemonIDLabel we add it
_, hasMgrRoleLabel := service.Spec.Selector[mgrRoleLabelName]
if !hasMgrRoleLabel {
logger.Infof("adding %s selector label on mgr service %q", mgrRoleLabelName, service.Name)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "adding %s selector label on mgr service %q", mgrRoleLabelName, service.Name)
service.Spec.Selector[mgrRoleLabelName] = activeMgrStatus
updateService = true
}
if updateService {
if _, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Update(c.clusterInfo.Context, &services.Items[i], metav1.UpdateOptions{}); err != nil {
logger.Errorf("failed to update service %q. %v", service.Name, err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to update service %q. %v", service.Name, err)
} else {
logger.Infof("service %q successfully updated", service.Name)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "service %q successfully updated", service.Name)
}
}
}
@@ -399,22 +400,22 @@ func (c *Cluster) updateServiceSelectors() {
func (c *Cluster) configureModules(daemonIDs []string) {
// Configure the modules asynchronously so we can complete all the configuration much sooner.
startModuleConfiguration("prometheus", c.configurePrometheusModule)
startModuleConfiguration("dashboard", c.configureDashboardModules)
c.startModuleConfiguration("prometheus", c.configurePrometheusModule)
c.startModuleConfiguration("dashboard", c.configureDashboardModules)
// It is a bit confusing but modules that are in the "always_on_modules" list
// are "just" enabled, but still they must be configured to work properly
startModuleConfiguration("balancer", c.enableBalancerModule)
startModuleConfiguration("mgr module(s) from the spec", c.configureMgrModules)
c.startModuleConfiguration("balancer", c.enableBalancerModule)
c.startModuleConfiguration("mgr module(s) from the spec", c.configureMgrModules)
}
func startModuleConfiguration(description string, configureModules func() error) {
func (c *Cluster) startModuleConfiguration(description string, configureModules func() error) {
go func() {
err := configureModules()
if err != nil {
logger.Errorf("failed modules: %q. %v", description, err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed modules: %q. %v", description, err)
} else {
logger.Infof("successful modules: %s", description)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "successful modules: %s", description)
}
}()
}
@@ -427,7 +428,7 @@ func (c *Cluster) configurePrometheusModule() error {
}
} else {
if err := cephclient.MgrDisableModule(c.context, c.clusterInfo, PrometheusModuleName); err != nil {
logger.Errorf("failed to disable mgr prometheus module. %v", err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to disable mgr prometheus module. %v", err)
}
return nil
}
@@ -446,7 +447,7 @@ func (c *Cluster) configurePrometheusModule() error {
if err != nil {
return err
}
logger.Infof("prometheus config will change, port: %s", port)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "prometheus config will change, port: %s", port)
}
// scrape interval
if c.spec.Monitoring.Interval != nil {
@@ -455,18 +456,18 @@ func (c *Cluster) configurePrometheusModule() error {
if err != nil {
return err
}
logger.Infof("prometheus config will change, interval: %v", interval)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "prometheus config will change, interval: %v", interval)
}
if portHasChanged || intervalHasChanged {
logger.Info("prometheus config has changed. restarting the prometheus module")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "prometheus config has changed. restarting the prometheus module")
return c.restartMgrModule(PrometheusModuleName)
}
return nil
}
func (c *Cluster) restartMgrModule(name string) error {
logger.Infof("restarting the mgr module: %s", name)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "restarting the mgr module: %s", name)
if err := cephclient.MgrDisableModule(c.context, c.clusterInfo, name); err != nil {
return errors.Wrapf(err, "failed to disable mgr module %q.", name)
}
@@ -520,7 +521,7 @@ func (c *Cluster) configureMgrModules() error {
// Configure special settings for individual modules that are enabled
switch module.Name {
case rookModuleName:
startModuleConfiguration("orchestrator modules", c.configureOrchestratorModules)
c.startModuleConfiguration("orchestrator modules", c.configureOrchestratorModules)
}
} else {
@@ -605,10 +606,10 @@ func applyMonitoringLabels(c *Cluster, serviceMonitor *monitoringv1.ServiceMonit
serviceMonitor.Spec.Endpoints[0].RelabelConfigs = append(
serviceMonitor.Spec.Endpoints[0].RelabelConfigs, relabelConfig)
} else {
logger.Info("rook.io/managedBy not specified in monitoring labels")
log.NamespacedInfo(serviceMonitor.Namespace, logger, "rook.io/managedBy not specified in monitoring labels")
}
} else {
logger.Debug("monitoring labels not specified")
log.NamespacedDebug(serviceMonitor.Namespace, logger, "monitoring labels not specified")
}
}
}
@@ -638,11 +639,11 @@ func updateMgrCephxStatus(c *clusterd.Context, clusterInfo *cephclient.ClusterIn
}
updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, clusterInfo.CephVersion, cluster.Status.Cephx.Mgr)
cluster.Status.Cephx.Mgr = updatedStatus
logger.Debugf("updating mgr daemon cephx status to %+v", cluster.Status.Cephx.Mgr)
log.NamespacedDebug(clusterInfo.Namespace, logger, "updating mgr daemon cephx status to %+v", cluster.Status.Cephx.Mgr)
if err := reporting.UpdateStatus(c.Client, cluster); err != nil {
return errors.Wrap(err, "failed to update cluster cephx status for mgr daemon")
}
logger.Info("successfully updated the cephx status for mgr daemon")
log.NamespacedInfo(clusterInfo.Namespace, logger, "successfully updated the cephx status for mgr daemon")
return nil
})
+2 -1
View File
@@ -29,6 +29,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -41,7 +42,7 @@ const (
)
func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) {
logger.Debugf("mgrConfig: %+v", mgrConfig)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "mgrConfig: %+v", mgrConfig)
volumes := controller.DaemonVolumes(mgrConfig.DataPathMap, mgrConfig.ResourceName, c.spec.DataDirHostPath)
if c.spec.Network.IsMultus() {
+5 -4
View File
@@ -19,6 +19,7 @@ package mon
import (
"github.com/pkg/errors"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
policyv1 "k8s.io/api/policy/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
@@ -38,7 +39,7 @@ func (c *Cluster) reconcileMonPDB() error {
monCount := c.spec.Mon.Count
if monCount <= 2 {
logger.Debug("managePodBudgets is set, but mon-count <= 2. Not creating a disruptionbudget for Mons")
log.NamespacedDebug(c.Namespace, logger, "managePodBudgets is set, but mon-count <= 2. Not creating a disruptionbudget for Mons")
return nil
}
@@ -76,7 +77,7 @@ func (c *Cluster) blockMonDrain(request types.NamespacedName) error {
if !c.spec.DisruptionManagement.ManagePodBudgets {
return nil
}
logger.Info("prevent voluntary mon drain while failing over")
log.NamespacedInfo(c.Namespace, logger, "prevent voluntary mon drain while failing over")
// change MaxUnavailable mon PDB to 0
_, err := c.createOrUpdateMonPDB(0)
if err != nil {
@@ -90,7 +91,7 @@ func (c *Cluster) allowMonDrain(request types.NamespacedName) error {
if !c.spec.DisruptionManagement.ManagePodBudgets {
return nil
}
logger.Info("allow voluntary mon drain after failover")
log.NamespacedInfo(c.Namespace, logger, "allow voluntary mon drain after failover")
_, err := c.createOrUpdateMonPDB(c.getMaxUnavailableMonPodCount())
if err != nil {
return errors.Wrapf(err, "failed to update MaxUnavailable for mon PDB %q", request.Name)
@@ -100,7 +101,7 @@ func (c *Cluster) allowMonDrain(request types.NamespacedName) error {
func (c *Cluster) getMaxUnavailableMonPodCount() int32 {
if c.spec.Mon.Count >= 5 {
logger.Debug("setting the mon pdb max unavailable count to 2 in case there are 5 or more mons")
log.NamespacedDebug(c.Namespace, logger, "setting the mon pdb max unavailable count to 2 in case there are 5 or more mons")
return 2
}
+103 -101
View File
@@ -31,6 +31,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
@@ -71,7 +72,7 @@ func updateMonTimeout(monCluster *Cluster) {
if monCRDTimeoutSetting != "" {
if monTimeout, err := time.ParseDuration(monCRDTimeoutSetting); err == nil {
if monTimeout == timeZero {
logger.Warning("monitor failover is disabled")
log.NamespacedWarning(monCluster.Namespace, logger, "monitor failover is disabled")
}
MonOutTimeout = monTimeout
}
@@ -95,7 +96,7 @@ func updateMonInterval(monCluster *Cluster, h *HealthChecker) {
checkInterval := monCluster.spec.HealthCheck.DaemonHealth.Monitor.Interval
// allow overriding the check interval
if checkInterval != nil {
logger.Debugf("ceph mon status in namespace %q check interval %q", monCluster.Namespace, checkInterval.Duration.String())
log.NamespacedDebug(monCluster.Namespace, logger, "ceph mon status check interval %q", checkInterval.Duration.String())
h.interval = checkInterval.Duration
}
}
@@ -113,6 +114,7 @@ func NewHealthChecker(monCluster *Cluster) *HealthChecker {
// Check periodically checks the health of the monitors
func (hc *HealthChecker) Check(monitoringRoutines *sync.Map, daemon string) {
namespace := hc.monCluster.Namespace
for {
// Update Mon Timeout with CR details
updateMonTimeout(hc.monCluster)
@@ -124,28 +126,28 @@ func (hc *HealthChecker) Check(monitoringRoutines *sync.Map, daemon string) {
// we will get an invalid pointer error and the go routine will panic
v, ok := monitoringRoutines.Load(daemon)
if !ok {
logger.Infof("ceph cluster %q has been deleted. stopping monitoring of mons", hc.monCluster.Namespace)
log.NamespacedInfo(namespace, logger, "ceph cluster has been deleted. stopping monitoring of mons")
return
}
health := v.(*controller.ClusterHealth)
select {
case <-health.InternalCtx.Done():
logger.Infof("stopping monitoring of mons in namespace %q (internal context canceled)", hc.monCluster.Namespace)
log.NamespacedInfo(namespace, logger, "stopping monitoring of mons (internal context canceled)")
monitoringRoutines.Delete(daemon)
return
// Since c.ClusterInfo.IsInitialized() below uses a different context, we need to check if the context is done
case <-hc.monCluster.ClusterInfo.Context.Done():
logger.Infof("stopping monitoring of mons in namespace %q (mon cluster info context canceled)", hc.monCluster.Namespace)
log.NamespacedInfo(namespace, logger, "stopping monitoring of mons (mon cluster info context canceled)")
monitoringRoutines.Delete(daemon)
return
case <-time.After(hc.interval):
logger.Debugf("checking health of mons")
log.NamespacedDebug(namespace, logger, "checking health of mons")
err := hc.monCluster.checkHealth(health.InternalCtx)
if err != nil {
logger.Warningf("failed to check mon health. %v", err)
log.NamespacedWarning(namespace, logger, "failed to check mon health. %v", err)
}
}
}
@@ -170,11 +172,11 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
return errors.Wrap(err, "failed to check for mons to skip reconcile")
}
if monsToSkipReconcile.Len() > 0 {
logger.Warningf("skipping mon health check since mons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(monsToSkipReconcile))
log.NamespacedWarning(c.Namespace, logger, "skipping mon health check since mons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(monsToSkipReconcile))
return nil
}
logger.Debugf("Checking health for mons in cluster %q", c.ClusterInfo.Namespace)
log.NamespacedDebug(c.Namespace, logger, "Checking health for mons in cluster %q", c.ClusterInfo.Namespace)
// For an external connection we use a special function to get the status
if c.spec.External.Enable {
@@ -203,7 +205,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
if err != nil {
return errors.Wrap(err, "failed to get mon quorum status")
}
logger.Debugf("Mon quorum status: %+v", quorumStatus)
log.NamespacedDebug(c.Namespace, logger, "Mon quorum status: %+v", quorumStatus)
// handle external Mons
quorumStatus, err = c.reconcileExternalMons(ctx, quorumStatus)
@@ -214,7 +216,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
// Use a local mon count in case the user updates the crd in another goroutine.
// We need to complete a health check with a consistent value.
desiredMonCount := c.spec.Mon.Count
logger.Debugf("targeting the mon count %d", desiredMonCount)
log.NamespacedDebug(c.Namespace, logger, "targeting the mon count %d", desiredMonCount)
// Source of truth of which mons should exist is our *clusterInfo*
monsNotFound := map[string]interface{}{}
@@ -235,14 +237,14 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
// when the mon isn't in the clusterInfo, but is in quorum and there are
// enough mons, remove it else remove it on the next run
if inQuorum && len(quorumStatus.Quorum) > desiredMonCount {
logger.Warningf("mon %q not in source of truth but in quorum, removing", mon.Name)
log.NamespacedWarning(c.Namespace, logger, "mon %q not in source of truth but in quorum, removing", mon.Name)
if err := c.removeMon(mon.Name); err != nil {
logger.Warningf("failed to remove mon %q. %v", mon.Name, err)
log.NamespacedWarning(c.Namespace, logger, "failed to remove mon %q. %v", mon.Name, err)
}
// only remove one extra mon per health check
return nil
}
logger.Warningf(
log.NamespacedWarning(c.Namespace, logger,
"mon %q not in source of truth and not in quorum, not enough mons to remove now (wanted: %d, current: %d)",
mon.Name,
desiredMonCount,
@@ -254,16 +256,16 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
if _, err := c.trackMonInOrOutOfQuorum(mon.Name, true); err != nil {
return errors.Wrapf(err, "failed to track out of quorum mon %q", mon.Name)
}
logger.Debugf("mon %q found in quorum", mon.Name)
log.NamespacedDebug(c.Namespace, logger, "mon %q found in quorum", mon.Name)
// delete the "timeout" for a mon if the pod is in quorum again
if _, ok := c.monTimeoutList[mon.Name]; ok {
delete(c.monTimeoutList, mon.Name)
logger.Infof("mon %q is back in quorum, removed from mon out timeout list", mon.Name)
log.NamespacedInfo(c.Namespace, logger, "mon %q is back in quorum, removed from mon out timeout list", mon.Name)
}
continue
}
logger.Debugf("mon %q NOT found in quorum. Mon quorum status: %+v", mon.Name, quorumStatus)
log.NamespacedDebug(c.Namespace, logger, "mon %q NOT found in quorum. Mon quorum status: %+v", mon.Name, quorumStatus)
allMonsInQuorum = false
if _, err := c.trackMonInOrOutOfQuorum(mon.Name, false); err != nil {
return errors.Wrapf(err, "failed to track out of quorum mon %q", mon.Name)
@@ -271,7 +273,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
// if the time out is set to 0 this indicate that we don't want to trigger mon failover
if MonOutTimeout == timeZero {
logger.Warningf("mon %q NOT found in quorum and health timeout is 0, mon will never fail over", mon.Name)
log.NamespacedWarning(c.Namespace, logger, "mon %q NOT found in quorum and health timeout is 0, mon will never fail over", mon.Name)
continue
}
@@ -290,7 +292,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
if !rescheduleImmediately {
if time.Since(c.monTimeoutList[mon.Name]) <= MonOutTimeout {
timeToFailover := int(MonOutTimeout.Seconds() - time.Since(c.monTimeoutList[mon.Name]).Seconds())
logger.Warningf("mon %q not found in quorum, waiting for timeout (%d seconds left) before failover", mon.Name, timeToFailover)
log.NamespacedWarning(c.Namespace, logger, "mon %q not found in quorum, waiting for timeout (%d seconds left) before failover", mon.Name, timeToFailover)
continue
}
@@ -298,15 +300,15 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
monLabelSelector := fmt.Sprintf("%s=%s,%s=%s", k8sutil.AppAttr, AppName, controller.DaemonIDLabel, mon.Name)
isScheduled, err := k8sutil.IsPodScheduled(ctx, c.context.Clientset, c.Namespace, monLabelSelector)
if err != nil {
logger.Warningf("failed to check if mon %q is assigned to a node, continuing with mon failover. %v", mon.Name, err)
log.NamespacedWarning(c.Namespace, logger, "failed to check if mon %q is assigned to a node, continuing with mon failover. %v", mon.Name, err)
} else if !isScheduled && retriesBeforeNodeDrainFailover > 0 {
logger.Warningf("mon %q NOT found in quorum after timeout. Mon pod is not scheduled. Retrying with a timeout of %.2f seconds before failover", mon.Name, MonOutTimeout.Seconds())
log.NamespacedWarning(c.Namespace, logger, "mon %q NOT found in quorum after timeout. Mon pod is not scheduled. Retrying with a timeout of %.2f seconds before failover", mon.Name, MonOutTimeout.Seconds())
delete(c.monTimeoutList, mon.Name)
retriesBeforeNodeDrainFailover = retriesBeforeNodeDrainFailover - 1
return nil
}
retriesBeforeNodeDrainFailover = 1
logger.Warningf("mon %q NOT found in quorum and timeout exceeded, mon will be failed over", mon.Name)
log.NamespacedWarning(c.Namespace, logger, "mon %q NOT found in quorum and timeout exceeded, mon will be failed over", mon.Name)
}
if !c.failMon(len(quorumStatus.MonMap.Mons), desiredMonCount, mon.Name) {
@@ -328,7 +330,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
// after all unhealthy mons have been removed or failed over
// handle all mons that haven't been in the Ceph mon map
for mon := range monsNotFound {
logger.Warningf("mon %s NOT found in ceph mon map, failover", mon)
log.NamespacedWarning(c.Namespace, logger, "mon %s NOT found in ceph mon map, failover", mon)
c.failMon(len(c.ClusterInfo.InternalMonitors), desiredMonCount, mon)
// only deal with one "not found in ceph mon map" mon per health check
return nil
@@ -336,23 +338,23 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
// create/start new mons when there are fewer mons than the desired count in the CRD
if len(quorumStatus.MonMap.Mons) < desiredMonCount {
logger.Infof("adding mons. currently %d mons are in quorum and the desired count is %d.", len(quorumStatus.MonMap.Mons), desiredMonCount)
log.NamespacedInfo(c.Namespace, logger, "adding mons. currently %d mons are in quorum and the desired count is %d.", len(quorumStatus.MonMap.Mons), desiredMonCount)
return c.startMons(desiredMonCount)
}
// remove extra mons if the desired count has decreased in the CRD and all the mons are currently healthy
if allMonsInQuorum && len(quorumStatus.MonMap.Mons) > desiredMonCount {
if desiredMonCount < 2 && len(quorumStatus.MonMap.Mons) == 2 {
logger.Warningf("cannot reduce mon quorum size from 2 to 1")
log.NamespacedWarning(c.Namespace, logger, "cannot reduce mon quorum size from 2 to 1")
} else {
logger.Infof("removing an extra mon. currently %d are in quorum and only %d are desired", len(quorumStatus.MonMap.Mons), desiredMonCount)
log.NamespacedInfo(c.Namespace, logger, "removing an extra mon. currently %d are in quorum and only %d are desired", len(quorumStatus.MonMap.Mons), desiredMonCount)
return c.removeMon(c.determineExtraMonToRemove())
}
}
if allMonsInQuorum && len(quorumStatus.MonMap.Mons) == desiredMonCount {
// remove any pending/not needed mon canary deployment if everything is ok
logger.Debug("mon cluster is healthy, removing any existing canary deployment")
log.NamespacedDebug(c.Namespace, logger, "mon cluster is healthy, removing any existing canary deployment")
c.removeCanaryDeployments(monCanaryLabelSelector)
// Check whether two healthy mons are on the same node when they should not be.
@@ -382,7 +384,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
// failover any mons present in the mon fail over list
for _, mon := range c.ClusterInfo.InternalMonitors {
if _, ok := c.monsToFailover[mon.Name]; ok {
logger.Infof("fail over mon %q from the mon fail over list", mon.Name)
log.NamespacedInfo(c.Namespace, logger, "fail over mon %q from the mon fail over list", mon.Name)
c.failMon(len(c.ClusterInfo.InternalMonitors), desiredMonCount, mon.Name)
delete(c.monsToFailover, mon.Name)
return nil
@@ -394,25 +396,25 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
func (c *Cluster) shouldFailoverMonImmediately(ctx context.Context, monName string) bool {
// If the assigned node does not exist, reschedule immediately
logger.Debugf("checking if mon %q is scheduled on a node", monName)
log.NamespacedDebug(c.Namespace, logger, "checking if mon %q is scheduled on a node", monName)
assignedNode := ""
if mapping, ok := c.mapping.Schedule[monName]; ok {
if mapping == nil {
// mon schedule information is not available in the rook-ceph-mon-endpoints configmap in case of mons on PVC
logger.Debugf("mon schedule information is not available for mon %q", monName)
log.NamespacedDebug(c.Namespace, logger, "mon schedule information is not available for mon %q", monName)
return false
}
assignedNode = mapping.Hostname
}
if assignedNode == "" {
logger.Debugf("mon %q is not assigned to a node", monName)
log.NamespacedDebug(c.Namespace, logger, "mon %q is not assigned to a node", monName)
return false
}
logger.Debugf("mon %q is scheduled on node %q", monName, assignedNode)
log.NamespacedDebug(c.Namespace, logger, "mon %q is scheduled on node %q", monName, assignedNode)
nodeExists, err := k8sutil.NodeWithHostnameExists(ctx, c.context.Clientset, assignedNode)
if err == nil && !nodeExists {
logger.Warningf("mon %q NOT found in quorum and its assigned node %q is not found, failing over immediately", monName, assignedNode)
log.NamespacedWarning(c.Namespace, logger, "mon %q NOT found in quorum and its assigned node %q is not found, failing over immediately", monName, assignedNode)
return true
}
@@ -423,11 +425,11 @@ func (c *Cluster) shouldFailoverMonImmediately(ctx context.Context, monName stri
func (c *Cluster) reconcileExternalMons(ctx context.Context, quorumStatus cephclient.MonStatusResponse) (cephclient.MonStatusResponse, error) {
if len(c.spec.Mon.ExternalMonIDs) != 0 {
if c.spec.External.Enable {
logger.Warning("ignore external mon IDs in cluster spec: cluster is running in external mode")
log.NamespacedWarning(c.Namespace, logger, "ignore external mon IDs in cluster spec: cluster is running in external mode")
return quorumStatus, nil
}
if c.spec.IsStretchCluster() {
logger.Warning("ignore external mon IDs in cluster spec: cluster is running in stretch mode")
log.NamespacedWarning(c.Namespace, logger, "ignore external mon IDs in cluster spec: cluster is running in stretch mode")
return quorumStatus, nil
}
}
@@ -444,19 +446,19 @@ func (c *Cluster) reconcileExternalMons(ctx context.Context, quorumStatus cephcl
}
// existing external mon was removed from Cluster CRD spec:
// remove it from CLusterInfo
logger.Debugf("existing external mon %q was removed from spec: removing it", extID)
log.NamespacedDebug(c.Namespace, logger, "existing external mon %q was removed from spec: removing it", extID)
delete(c.ClusterInfo.ExternalMons, extID)
extMonsChanged = true
}
// handle external monitors if configured in cluster CRD:
logger.Debugf("external mon IDs: %v", extMonIDs)
log.NamespacedDebug(c.Namespace, logger, "external mon IDs: %v", extMonIDs)
for _, extID := range extMonIDs {
monStatus, inQuorum := getMonByID(extID, quorumStatus)
if inQuorum {
logger.Debugf("external mon %q in quorum", extID)
log.NamespacedDebug(c.Namespace, logger, "external mon %q in quorum", extID)
} else {
logger.Debugf("external mon %q not in quorum %+v, %+v", extID, quorumStatus.Quorum, quorumStatus.MonMap.Mons)
log.NamespacedDebug(c.Namespace, logger, "external mon %q not in quorum %+v, %+v", extID, quorumStatus.Quorum, quorumStatus.MonMap.Mons)
}
_, inInfo := c.ClusterInfo.ExternalMons[extID]
if inQuorum && !inInfo {
@@ -464,12 +466,12 @@ func (c *Cluster) reconcileExternalMons(ctx context.Context, quorumStatus cephcl
monInfo := c.monStatusToInfo(monStatus)
c.ClusterInfo.ExternalMons[extID] = monInfo
extMonsChanged = true
logger.Infof("new external mon %q found: %s, adding it", extID, monInfo.Endpoint)
log.NamespacedInfo(c.Namespace, logger, "new external mon %q found: %s, adding it", extID, monInfo.Endpoint)
} else if !inQuorum && inInfo {
// remove external mon from cluster info if it is out of quorum:
delete(c.ClusterInfo.ExternalMons, extID)
extMonsChanged = true
logger.Infof("new external mon %q not in quorum: removing it", extID)
log.NamespacedInfo(c.Namespace, logger, "new external mon %q not in quorum: removing it", extID)
}
}
if extMonsChanged {
@@ -516,7 +518,7 @@ func (c *Cluster) trackMonInOrOutOfQuorum(monName string, inQuorum bool) (bool,
// All mons are in quorum, so make sure no mons are marked out of quorum
for monName, mon := range c.ClusterInfo.InternalMonitors {
if mon.OutOfQuorum {
logger.Infof("resetting mon %q to be back in quorum", monName)
log.NamespacedInfo(c.Namespace, logger, "resetting mon %q to be back in quorum", monName)
mon.OutOfQuorum = false
updateNeeded = true
}
@@ -524,18 +526,18 @@ func (c *Cluster) trackMonInOrOutOfQuorum(monName string, inQuorum bool) (bool,
} else {
mon, ok := c.ClusterInfo.InternalMonitors[monName]
if !ok {
logger.Infof("mon %q not found to keep track of being out of quorum", monName)
log.NamespacedInfo(c.Namespace, logger, "mon %q not found to keep track of being out of quorum", monName)
return false, nil
}
// Mark the mon in quorum
if inQuorum && mon.OutOfQuorum {
logger.Infof("marking mon %q back in quorum", monName)
log.NamespacedInfo(c.Namespace, logger, "marking mon %q back in quorum", monName)
mon.OutOfQuorum = false
updateNeeded = true
}
// Mark the mon out of quorum
if !inQuorum && !mon.OutOfQuorum {
logger.Infof("marking mon %q out of quorum", monName)
log.NamespacedInfo(c.Namespace, logger, "marking mon %q out of quorum", monName)
mon.OutOfQuorum = true
updateNeeded = true
}
@@ -570,7 +572,7 @@ func (c *Cluster) determineExtraMonToRemove() string {
if stretchMonToRemove != "" {
return stretchMonToRemove
}
logger.Infof("did not find an extra mon to remove from the stretch cluster")
log.NamespacedInfo(c.Namespace, logger, "did not find an extra mon to remove from the stretch cluster")
return ""
}
@@ -578,13 +580,13 @@ func (c *Cluster) determineExtraMonToRemove() string {
arbitraryMon := ""
for _, mon := range mons {
if mon.NodeName == "" {
logger.Debugf("mon %q is not scheduled to a specific host", mon.DaemonName)
log.NamespacedDebug(c.Namespace, logger, "mon %q is not scheduled to a specific host", mon.DaemonName)
arbitraryMon = mon.DaemonName
continue
}
// Check if there are multiple mons on the node
if existingMon, ok := nodesWithMons[mon.NodeName]; ok {
logger.Infof("found mons %q and %q on node %s, removing mon %q", existingMon, mon.DaemonName, mon.NodeName, mon.DaemonName)
log.NamespacedInfo(c.Namespace, logger, "found mons %q and %q on node %s, removing mon %q", existingMon, mon.DaemonName, mon.NodeName, mon.DaemonName)
return mon.DaemonName
}
nodesWithMons[mon.NodeName] = mon.DaemonName
@@ -593,7 +595,7 @@ func (c *Cluster) determineExtraMonToRemove() string {
arbitraryMon = mon.DaemonName
}
logger.Infof("removing arbitrary extra mon %q", arbitraryMon)
log.NamespacedInfo(c.Namespace, logger, "removing arbitrary extra mon %q", arbitraryMon)
return arbitraryMon
}
@@ -603,7 +605,7 @@ func (c *Cluster) findExtraMonToRemoveFromStretchCluster(mons []*monConfig) stri
monInZones := map[string]string{}
for _, m := range mons {
if m.Zone == "" {
logger.Warningf("zone not found on mon %q", m.DaemonName)
log.NamespacedWarning(c.Namespace, logger, "zone not found on mon %q", m.DaemonName)
continue
}
zoneCount[m.Zone]++
@@ -620,12 +622,12 @@ func (c *Cluster) findExtraMonToRemoveFromStretchCluster(mons []*monConfig) stri
}
if zone.Arbiter {
if count > 1 {
logger.Infof("removing extra mon %q in arbiter zone %q", monInZones[zone.Name], zone.Name)
log.NamespacedInfo(c.Namespace, logger, "removing extra mon %q in arbiter zone %q", monInZones[zone.Name], zone.Name)
return monInZones[zone.Name]
}
} else {
if count > 2 {
logger.Infof("removing extra mon %q in zone %q", monInZones[zone.Name], zone.Name)
log.NamespacedInfo(c.Namespace, logger, "removing extra mon %q in zone %q", monInZones[zone.Name], zone.Name)
return monInZones[zone.Name]
}
}
@@ -639,69 +641,69 @@ func (c *Cluster) findExtraMonToRemoveFromStretchCluster(mons []*monConfig) stri
func (c *Cluster) failMon(monCount, desiredMonCount int, name string) bool {
// make sure the failed mon is marked out of quorum
if _, err := c.trackMonInOrOutOfQuorum(name, false); err != nil {
logger.Errorf("failed to track failed mon %q. %v", name, err)
log.NamespacedError(c.Namespace, logger, "failed to track failed mon %q. %v", name, err)
}
if monCount > desiredMonCount {
// no need to create a new mon since we have an extra
if err := c.removeMon(name); err != nil {
logger.Errorf("failed to remove mon %q. %v", name, err)
log.NamespacedError(c.Namespace, logger, "failed to remove mon %q. %v", name, err)
}
return true
}
// prevent any voluntary mon drain while failing over
if err := c.blockMonDrain(types.NamespacedName{Name: monPDBName, Namespace: c.Namespace}); err != nil {
logger.Errorf("failed to block mon drain. %v", err)
log.NamespacedError(c.Namespace, logger, "failed to block mon drain. %v", err)
}
// bring up a new mon to replace the unhealthy mon
if err := c.failoverMon(name); err != nil {
logger.Errorf("failed to failover mon %q. %v", name, err)
log.NamespacedError(c.Namespace, logger, "failed to failover mon %q. %v", name, err)
}
// allow any voluntary mon drain after failover
if err := c.allowMonDrain(types.NamespacedName{Name: monPDBName, Namespace: c.Namespace}); err != nil {
logger.Errorf("failed to allow mon drain. %v", err)
log.NamespacedError(c.Namespace, logger, "failed to allow mon drain. %v", err)
}
return true
}
func (c *Cluster) removeOrphanMonResources() {
if c.spec.Mon.VolumeClaimTemplate == nil {
logger.Debug("skipping check for orphaned mon pvcs since using the host path")
log.NamespacedDebug(c.Namespace, logger, "skipping check for orphaned mon pvcs since using the host path")
return
}
logger.Info("checking for orphaned mon resources")
log.NamespacedInfo(c.Namespace, logger, "checking for orphaned mon resources")
opts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, AppName)}
pvcs, err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).List(c.ClusterInfo.Context, opts)
if err != nil {
logger.Infof("failed to check for orphaned mon pvcs. %v", err)
log.NamespacedInfo(c.Namespace, logger, "failed to check for orphaned mon pvcs. %v", err)
return
}
for _, pvc := range pvcs.Items {
logger.Debugf("checking if pvc %q is orphaned", pvc.Name)
log.NamespacedDebug(c.Namespace, logger, "checking if pvc %q is orphaned", pvc.Name)
_, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, pvc.Name, metav1.GetOptions{})
if err == nil {
logger.Debugf("skipping pvc removal since the mon daemon %q still requires it", pvc.Name)
log.NamespacedDebug(c.Namespace, logger, "skipping pvc removal since the mon daemon %q still requires it", pvc.Name)
continue
}
if !kerrors.IsNotFound(err) {
logger.Infof("skipping pvc removal since the mon daemon %q might still require it. %v", pvc.Name, err)
log.NamespacedInfo(c.Namespace, logger, "skipping pvc removal since the mon daemon %q might still require it. %v", pvc.Name, err)
continue
}
logger.Infof("removing pvc %q since it is no longer needed for the mon daemon", pvc.Name)
log.NamespacedInfo(c.Namespace, logger, "removing pvc %q since it is no longer needed for the mon daemon", pvc.Name)
var gracePeriod int64 // delete immediately
propagation := metav1.DeletePropagationForeground
options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(c.ClusterInfo.Context, pvc.Name, *options)
if err != nil {
logger.Warningf("failed to delete orphaned monitor pvc %q. %v", pvc.Name, err)
log.NamespacedWarning(c.Namespace, logger, "failed to delete orphaned monitor pvc %q. %v", pvc.Name, err)
}
}
}
@@ -722,7 +724,7 @@ func (c *Cluster) updateMonDeploymentReplica(name string, enabled bool) error {
d.Spec.Replicas = &desiredReplicas
// update the deployment
logger.Infof("scaling the mon %q deployment to replica %d", name, desiredReplicas)
log.NamespacedInfo(c.Namespace, logger, "scaling the mon %q deployment to replica %d", name, desiredReplicas)
_, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Update(c.ClusterInfo.Context, d, metav1.UpdateOptions{})
if err != nil {
return errors.Wrapf(err, "failed to update mon %q replicas from %d to %d", name, originalReplicas, desiredReplicas)
@@ -731,7 +733,7 @@ func (c *Cluster) updateMonDeploymentReplica(name string, enabled bool) error {
}
func (c *Cluster) failoverMon(name string) error {
logger.Infof("Failing over monitor %q", name)
log.NamespacedInfo(c.Namespace, logger, "Failing over monitor %q", name)
// remove the failed mon from a local list of the existing mons for finding a stretch zone
existingMons := c.clusterInfoToMonConfigWithExclude(name)
@@ -743,13 +745,13 @@ func (c *Cluster) failoverMon(name string) error {
// Start a new monitor
m := c.newMonConfig(c.maxMonID+1, zone)
logger.Infof("starting new mon: %+v", m)
log.NamespacedInfo(c.Namespace, logger, "starting new mon: %+v", m)
// Scale down the failed mon to allow a new one to start
if c.stopMonDuringFailover(name) {
if err := c.updateMonDeploymentReplica(name, false); err != nil {
// attempt to continue with the failover even if the bad mon could not be stopped
logger.Warningf("failed to stop mon %q for failover. %v", name, err)
log.NamespacedWarning(c.Namespace, logger, "failed to stop mon %q for failover. %v", name, err)
}
}
@@ -761,21 +763,21 @@ func (c *Cluster) failoverMon(name string) error {
// do nothing if the new mon was started successfully, the deployment will anyway be deleted
return
}
logger.Warningf("failover of mon %q unsuccessful, cleaning up replacement mon %q", name, m.DaemonName)
log.NamespacedWarning(c.Namespace, logger, "failover of mon %q unsuccessful, cleaning up replacement mon %q", name, m.DaemonName)
if err := c.updateMonDeploymentReplica(name, true); err != nil {
// attempt to continue even if the bad mon could not be restarted
logger.Warningf("failed to restart failed mon %q after new mon wouldn't start. %v", name, err)
log.NamespacedWarning(c.Namespace, logger, "failed to restart failed mon %q after new mon wouldn't start. %v", name, err)
}
if err := c.removeMonWithOptionalQuorum(m.DaemonName, newMonMightBeInQuorum); err != nil {
logger.Infof("failed to remove mon %q from quorum. %v", m.DaemonName, err)
log.NamespacedInfo(c.Namespace, logger, "failed to remove mon %q from quorum. %v", m.DaemonName, err)
}
// Make sure the maxMonID is reverted to its previous value
// The maxMonId is committed to a configmap immediately after the mon deployment
// is started, even though c.maxMonID is not incremented until the mon failover is successful
logger.Infof("reverting maxMonId to %d", c.maxMonID)
log.NamespacedInfo(c.Namespace, logger, "reverting maxMonId to %d", c.maxMonID)
if err := c.commitMaxMonIDRequireIncrementing(c.maxMonID, false); err != nil {
logger.Errorf("failed to revert maxMonId after starting mon %q", m.DaemonName)
log.NamespacedError(c.Namespace, logger, "failed to revert maxMonId after starting mon %q", m.DaemonName)
}
}()
@@ -803,7 +805,7 @@ func (c *Cluster) failoverMon(name string) error {
if err != nil {
return errors.Wrapf(err, "failed to export service %q", monService.Name)
}
logger.Infof("mon %q exported IP is %s", m.DaemonName, exportedIP)
log.NamespacedInfo(c.Namespace, logger, "mon %q exported IP is %s", m.DaemonName, exportedIP)
m.PublicIP = exportedIP
} else {
m.PublicIP = monService.Spec.ClusterIP
@@ -846,7 +848,7 @@ func (c *Cluster) stopMonDuringFailover(name string) bool {
if mon, ok := c.monsToFailover[name]; ok && !mon.UseHostNetwork {
return true
}
logger.Infof("skipping stopping mon %q during failover, since with host networking a new mon cannot be started on the same node", name)
log.NamespacedInfo(c.Namespace, logger, "skipping stopping mon %q during failover, since with host networking a new mon cannot be started on the same node", name)
return false
}
@@ -858,15 +860,15 @@ func (c *Cluster) removeMon(daemonName string) error {
// make a best effort to remove the mon and all its resources
func (c *Cluster) removeMonWithOptionalQuorum(daemonName string, shouldRemoveFromQuorum bool) error {
if daemonName == "" {
logger.Info("did not identify a mon to remove")
log.NamespacedInfo(c.Namespace, logger, "did not identify a mon to remove")
return nil
}
logger.Infof("ensuring removal of unhealthy monitor %s", daemonName)
log.NamespacedInfo(c.Namespace, logger, "ensuring removal of unhealthy monitor %s", daemonName)
// Remove the bad monitor from quorum
if shouldRemoveFromQuorum {
if err := c.removeMonitorFromQuorum(daemonName); err != nil {
logger.Errorf("failed to remove mon %q from quorum. %v", daemonName, err)
log.NamespacedError(c.Namespace, logger, "failed to remove mon %q from quorum. %v", daemonName, err)
}
}
delete(c.ClusterInfo.InternalMonitors, daemonName)
@@ -899,39 +901,39 @@ func (c *Cluster) removeMonResources(daemonName string) {
options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
if err := c.context.Clientset.CoreV1().Services(c.Namespace).Delete(c.ClusterInfo.Context, resourceName, *options); err != nil {
if kerrors.IsNotFound(err) {
logger.Infof("dead mon service %s was already gone", resourceName)
log.NamespacedInfo(c.Namespace, logger, "dead mon service %s was already gone", resourceName)
} else {
logger.Errorf("failed to remove dead mon service %q. %v", resourceName, err)
log.NamespacedError(c.Namespace, logger, "failed to remove dead mon service %q. %v", resourceName, err)
}
}
// Remove the PVC backing the mon if it existed
if err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(c.ClusterInfo.Context, resourceName, metav1.DeleteOptions{}); err != nil {
if kerrors.IsNotFound(err) {
logger.Infof("mon pvc did not exist %q", resourceName)
log.NamespacedInfo(c.Namespace, logger, "mon pvc did not exist %q", resourceName)
} else {
logger.Errorf("failed to remove dead mon pvc %q. %v", resourceName, err)
log.NamespacedError(c.Namespace, logger, "failed to remove dead mon pvc %q. %v", resourceName, err)
}
}
// Remove the mon pod if it is still there
if err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Delete(c.ClusterInfo.Context, resourceName, *options); err != nil {
if kerrors.IsNotFound(err) {
logger.Infof("dead mon %s was already gone", resourceName)
log.NamespacedInfo(c.Namespace, logger, "dead mon %s was already gone", resourceName)
} else {
logger.Errorf("failed to remove dead mon deployment %q. %v", resourceName, err)
log.NamespacedError(c.Namespace, logger, "failed to remove dead mon deployment %q. %v", resourceName, err)
}
}
}
func (c *Cluster) removeMonitorFromQuorum(name string) error {
logger.Debugf("removing monitor %s", name)
log.NamespacedDebug(c.Namespace, logger, "removing monitor %s", name)
args := []string{"mon", "remove", name}
if _, err := cephclient.NewCephCommand(c.context, c.ClusterInfo, args).Run(); err != nil {
return errors.Wrapf(err, "mon %s remove failed", name)
}
logger.Infof("removed monitor %s", name)
log.NamespacedInfo(c.Namespace, logger, "removed monitor %s", name)
return nil
}
@@ -969,11 +971,11 @@ func (c *Cluster) addOrRemoveExternalMonitor(status cephclient.MonStatusResponse
oldClusterInfoMonitors[mon.Name] = mon
delete(c.ClusterInfo.InternalMonitors, monName)
}
logger.Debugf("ClusterInfo is now Empty, refilling it from status.MonMap.Mons")
log.NamespacedDebug(c.Namespace, logger, "ClusterInfo is now Empty, refilling it from status.MonMap.Mons")
monCount := len(status.MonMap.Mons)
if monCount == 1 {
logger.Warning("external cluster mon count is 1, consider adding new monitors.")
log.NamespacedWarning(c.Namespace, logger, "external cluster mon count is 1, consider adding new monitors.")
}
// Iterate over the mons first and compare it with ClusterInfo
@@ -984,25 +986,25 @@ func (c *Cluster) addOrRemoveExternalMonitor(status cephclient.MonStatusResponse
// If the mon is part of the quorum
if inQuorum {
info := c.monStatusToInfo(mon)
logger.Infof("new external mon %q found: %s, adding it", mon.Name, info.Endpoint)
log.NamespacedInfo(c.Namespace, logger, "new external mon %q found: %s, adding it", mon.Name, info.Endpoint)
c.ClusterInfo.InternalMonitors[mon.Name] = info
} else {
logger.Debugf("mon %q is not in quorum and not in ClusterInfo", mon.Name)
log.NamespacedDebug(c.Namespace, logger, "mon %q is not in quorum and not in ClusterInfo", mon.Name)
}
changed = true
} else {
// mon is in ClusterInfo
logger.Debugf("mon %q is in ClusterInfo, let's test if it's in quorum", mon.Name)
log.NamespacedDebug(c.Namespace, logger, "mon %q is in ClusterInfo, let's test if it's in quorum", mon.Name)
if !inQuorum {
// this mon was in clusterInfo but is not part of the quorum anymore
// thus don't add it again to ClusterInfo
logger.Infof("monitor %q is not part of the external cluster monitor quorum, removing it", mon.Name)
log.NamespacedInfo(c.Namespace, logger, "monitor %q is not part of the external cluster monitor quorum, removing it", mon.Name)
changed = true
} else {
// this mon was in clusterInfo and is still in the quorum
// add it again
c.ClusterInfo.InternalMonitors[mon.Name] = oldClusterInfoMonitors[mon.Name]
logger.Debugf("everything is fine mon %q in the clusterInfo and its quorum status is %v", mon.Name, inQuorum)
log.NamespacedDebug(c.Namespace, logger, "everything is fine mon %q in the clusterInfo and its quorum status is %v", mon.Name, inQuorum)
}
}
}
@@ -1019,7 +1021,7 @@ func (c *Cluster) addOrRemoveExternalMonitor(status cephclient.MonStatusResponse
}
}
logger.Debugf("ClusterInfo.Monitors is %+v", c.ClusterInfo.InternalMonitors)
log.NamespacedDebug(c.Namespace, logger, "ClusterInfo.Monitors is %+v", c.ClusterInfo.InternalMonitors)
return changed, nil
}
@@ -1041,11 +1043,11 @@ func (c *Cluster) monStatusToInfo(mon cephclient.MonMapEntry) *cephclient.MonInf
func (c *Cluster) evictMonIfMultipleOnSameNode() error {
if c.spec.Mon.AllowMultiplePerNode {
logger.Debug("skipping check for multiple mons on same node since multiple mons are allowed")
log.NamespacedDebug(c.Namespace, logger, "skipping check for multiple mons on same node since multiple mons are allowed")
return nil
}
logger.Info("checking if multiple mons are on the same node")
log.NamespacedInfo(c.Namespace, logger, "checking if multiple mons are on the same node")
// Get all the mon pods
label := fmt.Sprintf("app=%s", AppName)
@@ -1056,13 +1058,13 @@ func (c *Cluster) evictMonIfMultipleOnSameNode() error {
nodesToMons := map[string]string{}
for _, pod := range pods.Items {
logger.Debugf("analyzing mon pod %q on node %q", pod.Name, pod.Spec.NodeName)
log.NamespacedDebug(c.Namespace, logger, "analyzing mon pod %q on node %q", pod.Name, pod.Spec.NodeName)
if _, ok := pod.Labels["mon_canary"]; ok {
logger.Debugf("skipping mon canary pod %q", pod.Name)
log.NamespacedDebug(c.Namespace, logger, "skipping mon canary pod %q", pod.Name)
continue
}
if pod.Spec.NodeName == "" {
logger.Warningf("mon %q is not assigned to a node", pod.Name)
log.NamespacedWarning(c.Namespace, logger, "mon %q is not assigned to a node", pod.Name)
continue
}
monName := pod.Labels["mon"]
@@ -1073,7 +1075,7 @@ func (c *Cluster) evictMonIfMultipleOnSameNode() error {
continue
}
logger.Warningf("Both mons %q and %q are on node %q. Evicting mon %q", monName, previousMonName, pod.Spec.NodeName, monName)
log.NamespacedWarning(c.Namespace, logger, "Both mons %q and %q are on node %q. Evicting mon %q", monName, previousMonName, pod.Spec.NodeName, monName)
return c.failoverMon(monName)
}
+90 -88
View File
@@ -44,6 +44,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/reporting"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
apps "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
@@ -217,21 +218,21 @@ func (c *Cluster) Start(clusterInfo *cephclient.ClusterInfo, rookImage string, c
return nil, errors.Wrap(err, "failed to check pod memory")
}
logger.Infof("start running mons")
log.NamespacedInfo(c.Namespace, logger, "start running mons")
logger.Debugf("establishing ceph cluster info")
log.NamespacedDebug(c.Namespace, logger, "establishing ceph cluster info")
if err := c.initClusterInfo(cephVersion, c.ClusterInfo.NamespacedName().Name); err != nil {
return nil, errors.Wrap(err, "failed to initialize ceph cluster info")
}
logger.Infof("targeting the mon count %d", c.spec.Mon.Count)
log.NamespacedInfo(c.Namespace, logger, "targeting the mon count %d", c.spec.Mon.Count)
monsToSkipReconcile, err := controller.GetDaemonsToSkipReconcile(c.ClusterInfo.Context, c.context, c.Namespace, config.MonType, AppName)
if err != nil {
return nil, errors.Wrap(err, "failed to check for mons to skip reconcile")
}
if monsToSkipReconcile.Len() > 0 {
logger.Warningf("skipping mon reconcile since mons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(monsToSkipReconcile))
log.NamespacedWarning(c.Namespace, logger, "skipping mon reconcile since mons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(monsToSkipReconcile))
return c.ClusterInfo, nil
}
@@ -263,7 +264,7 @@ func (c *Cluster) startMons(targetCount int) error {
// fixed by updating the mon deployments. Instead of returning error here, log a
// warning, and retry setting this later.
setConfigsNeedsRetry = true
logger.Warningf("failed to set Rook and/or user-defined Ceph config options before starting mons; will retry after starting mons. %v", err)
log.NamespacedWarning(c.Namespace, logger, "failed to set Rook and/or user-defined Ceph config options before starting mons; will retry after starting mons. %v", err)
}
}
@@ -321,7 +322,7 @@ func (c *Cluster) startMons(targetCount int) error {
}
}
logger.Debugf("mon endpoints used are: %s", flattenMonEndpoints(c.ClusterInfo.AllMonitors()))
log.NamespacedDebug(c.Namespace, logger, "mon endpoints used are: %s", flattenMonEndpoints(c.ClusterInfo.AllMonitors()))
// reconcile mon PDB
if err := c.reconcileMonPDB(); err != nil {
@@ -376,14 +377,14 @@ func (c *Cluster) ConfigureArbiter() error {
monDump, err := cephclient.GetMonDump(c.context, c.ClusterInfo)
if err != nil {
logger.Warningf("attempting to enable arbiter after failed to detect if already enabled. %v", err)
log.NamespacedWarning(c.Namespace, logger, "attempting to enable arbiter after failed to detect if already enabled. %v", err)
} else if monDump.StretchMode {
if monDump.TiebreakerMon == c.arbiterMon {
logger.Infof("stretch mode is already enabled with tiebreaker %q", c.arbiterMon)
log.NamespacedInfo(c.Namespace, logger, "stretch mode is already enabled with tiebreaker %q", c.arbiterMon)
return nil
}
// Set the new mon tiebreaker
logger.Infof("updating tiebreaker mon from %q to %q", monDump.TiebreakerMon, c.arbiterMon)
log.NamespacedInfo(c.Namespace, logger, "updating tiebreaker mon from %q to %q", monDump.TiebreakerMon, c.arbiterMon)
if err := cephclient.SetNewTiebreaker(c.context, c.ClusterInfo, c.arbiterMon); err != nil {
return errors.Wrap(err, "failed to set new mon tiebreaker")
}
@@ -394,7 +395,7 @@ func (c *Cluster) ConfigureArbiter() error {
// The timeout is relatively short since the operator will requeue the reconcile
// and try again at a higher level if not yet found
failureDomain := c.getFailureDomainName()
logger.Infof("enabling stretch mode... waiting for two failure domains of type %q to be found in the CRUSH map after OSD initialization", failureDomain)
log.NamespacedInfo(c.Namespace, logger, "enabling stretch mode... waiting for two failure domains of type %q to be found in the CRUSH map after OSD initialization", failureDomain)
pollInterval := 5 * time.Second
totalWaitTime := 2 * time.Minute
err = wait.PollUntilContextTimeout(c.ClusterInfo.Context, pollInterval, totalWaitTime, true, func(ctx context.Context) (bool, error) {
@@ -408,7 +409,7 @@ func (c *Cluster) ConfigureArbiter() error {
// Wait for the .mgr pool to be created, which we expect is defined as a CephBlockPool
// We may be able to remove this code waiting for the pool once this is in the ceph release:
// https://github.com/ceph/ceph/pull/61371
logger.Info("enabling stretch mode... waiting for the builtin .mgr pool to be created")
log.NamespacedInfo(c.Namespace, logger, "enabling stretch mode... waiting for the builtin .mgr pool to be created")
err = wait.PollUntilContextTimeout(c.ClusterInfo.Context, pollInterval, totalWaitTime, true, func(ctx context.Context) (bool, error) {
return c.builtinMgrPoolExists(), nil
})
@@ -443,7 +444,7 @@ func (c *Cluster) readyToConfigureArbiter(checkOSDPods bool) (bool, error) {
return false, errors.Wrap(err, "failed to check whether all osds are running before enabling the arbiter")
}
if !allRunning {
logger.Infof("waiting for all OSD pods to be in running state")
log.NamespacedInfo(c.Namespace, logger, "waiting for all OSD pods to be in running state")
return false, nil
}
}
@@ -460,10 +461,10 @@ func (c *Cluster) readyToConfigureArbiter(checkOSDPods bool) (bool, error) {
if bucket.TypeName == failureDomain {
// skip zones specific to device classes
if strings.Index(bucket.Name, "~") > 0 {
logger.Debugf("skipping device class bucket %q", bucket.Name)
log.NamespacedDebug(c.Namespace, logger, "skipping device class bucket %q", bucket.Name)
continue
}
logger.Infof("found %s %q in CRUSH map with weight %d", failureDomain, bucket.Name, bucket.Weight)
log.NamespacedInfo(c.Namespace, logger, "found %s %q in CRUSH map with weight %d", failureDomain, bucket.Name, bucket.Weight)
zoneCount++
// check that the weights of the failure domains are all the same
@@ -471,7 +472,7 @@ func (c *Cluster) readyToConfigureArbiter(checkOSDPods bool) (bool, error) {
// found the first matching bucket
zoneWeight = bucket.Weight
} else if zoneWeight != bucket.Weight {
logger.Infof("found failure domains that have different weights")
log.NamespacedInfo(c.Namespace, logger, "found failure domains that have different weights")
return false, nil
}
}
@@ -483,7 +484,7 @@ func (c *Cluster) readyToConfigureArbiter(checkOSDPods bool) (bool, error) {
if zoneCount > 2 {
return false, fmt.Errorf("cannot configure stretch cluster with more than 2 failure domains, and found %d of type %q", zoneCount, failureDomain)
}
logger.Infof("found two expected failure domains %q for the stretch cluster", failureDomain)
log.NamespacedInfo(c.Namespace, logger, "found two expected failure domains %q for the stretch cluster", failureDomain)
return true, nil
}
@@ -494,9 +495,9 @@ func (c *Cluster) readyToConfigureArbiter(checkOSDPods bool) (bool, error) {
// This is needed when the operator is restarted and all mons may not be up or in quorum.
func (c *Cluster) ensureMonsRunning(mons []*monConfig, i, targetCount int, requireAllInQuorum bool) error {
if requireAllInQuorum {
logger.Infof("creating mon %s", mons[i].DaemonName)
log.NamespacedInfo(c.Namespace, logger, "creating mon %s", mons[i].DaemonName)
} else {
logger.Info("checking for basic quorum with existing mons")
log.NamespacedInfo(c.Namespace, logger, "checking for basic quorum with existing mons")
}
// Calculate how many mons we expected to exist after this method is completed.
@@ -609,7 +610,7 @@ func (c *Cluster) clusterInfoToMonConfigWithExclude(excludedMon string) []*monCo
isHostNetwork = true
}
}
logger.Debugf("Host network for mon %q is %t", monitor.Name, isHostNetwork)
log.NamespacedDebug(c.Namespace, logger, "Host network for mon %q is %t", monitor.Name, isHostNetwork)
mons = append(mons, &monConfig{
ResourceName: resourceName(monitor.Name),
@@ -739,10 +740,10 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) {
_, err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Create(c.ClusterInfo.Context, pvc, metav1.CreateOptions{})
if err == nil {
logger.Infof("created canary monitor %s pvc %s", d.Name, pvc.Name)
log.NamespacedInfo(c.Namespace, logger, "created canary monitor %s pvc %s", d.Name, pvc.Name)
} else {
if kerrors.IsAlreadyExists(err) {
logger.Debugf("creating mon %s pvc %s: already exists.", d.Name, pvc.Name)
log.NamespacedDebug(c.Namespace, logger, "creating mon %s pvc %s: already exists.", d.Name, pvc.Name)
} else {
return nil, errors.Wrapf(err, "failed to create mon %s pvc %s", d.Name, pvc.Name)
}
@@ -763,13 +764,13 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) {
_, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(c.ClusterInfo.Context, d, metav1.CreateOptions{})
if err == nil {
createdDeployment = true
logger.Infof("created canary deployment %s", d.Name)
log.NamespacedInfo(c.Namespace, logger, "created canary deployment %s", d.Name)
break
} else if kerrors.IsAlreadyExists(err) {
if err := k8sutil.DeleteDeployment(c.ClusterInfo.Context, c.context.Clientset, c.Namespace, d.Name); err != nil {
return nil, errors.Wrapf(err, "failed to delete canary deployment %s", d.Name)
}
logger.Infof("deleted existing canary deployment %s", d.Name)
log.NamespacedInfo(c.Namespace, logger, "deleted existing canary deployment %s", d.Name)
time.Sleep(time.Second * canaryRetryDelaySeconds)
} else {
return nil, errors.Wrapf(err, "failed to create canary monitor deployment %s", d.Name)
@@ -824,13 +825,13 @@ func realWaitForMonitorScheduling(c *Cluster, d *apps.Deployment) (SchedulingRes
}
if len(pods.Items) == 0 {
logger.Infof("waiting for canary pod creation %s", d.Name)
log.NamespacedInfo(c.Namespace, logger, "waiting for canary pod creation %s", d.Name)
continue
}
pod := pods.Items[0]
if pod.Spec.NodeName == "" {
logger.Debugf("monitor %s canary pod %s not yet scheduled", d.Name, pod.Name)
log.NamespacedDebug(c.Namespace, logger, "monitor %s canary pod %s not yet scheduled", d.Name, pod.Name)
continue
}
@@ -840,7 +841,7 @@ func realWaitForMonitorScheduling(c *Cluster, d *apps.Deployment) (SchedulingRes
}
result.Node = node
logger.Infof("canary monitor deployment %s scheduled to %s", d.Name, node.Name)
log.NamespacedInfo(c.Namespace, logger, "canary monitor deployment %s scheduled to %s", d.Name, node.Name)
return result, nil
}
@@ -853,7 +854,7 @@ func (c *Cluster) initMonIPs(mons []*monConfig) error {
return c.ClusterInfo.Context.Err()
}
if m.UseHostNetwork {
logger.Infof("setting mon %q endpoints for hostnetwork mode", m.DaemonName)
log.NamespacedInfo(c.Namespace, logger, "setting mon %q endpoints for hostnetwork mode", m.DaemonName)
node, ok := c.mapping.Schedule[m.DaemonName]
if !ok || node == nil {
return errors.Errorf("failed to find node for mon %q in assignment map", m.DaemonName)
@@ -871,7 +872,7 @@ func (c *Cluster) initMonIPs(mons []*monConfig) error {
if err != nil {
return errors.Wrapf(err, "failed to export service %q", monService.Name)
}
logger.Infof("mon %q exported IP is %s", m.DaemonName, exportedIP)
log.NamespacedInfo(c.Namespace, logger, "mon %q exported IP is %s", m.DaemonName, exportedIP)
m.PublicIP = exportedIP
} else {
m.PublicIP = monService.Spec.ClusterIP
@@ -889,15 +890,15 @@ func (c *Cluster) initMonIPs(mons []*monConfig) error {
func (c *Cluster) removeCanaryDeployments(labelSelector string) {
canaryDeployments, err := k8sutil.GetDeployments(c.ClusterInfo.Context, c.context.Clientset, c.Namespace, labelSelector)
if err != nil {
logger.Warningf("failed to get the list of monitor canary deployments. %v", err)
log.NamespacedWarning(c.Namespace, logger, "failed to get the list of monitor canary deployments. %v", err)
return
}
// Delete the canary mons
for _, canary := range canaryDeployments.Items {
logger.Infof("cleaning up canary monitor deployment %q", canary.Name)
log.NamespacedInfo(c.Namespace, logger, "cleaning up canary monitor deployment %q", canary.Name)
if err := k8sutil.DeleteDeployment(c.ClusterInfo.Context, c.context.Clientset, c.Namespace, canary.Name); err != nil {
logger.Warningf("failed to delete canary monitor deployment %q. %v", canary.Name, err)
log.NamespacedWarning(c.Namespace, logger, "failed to delete canary monitor deployment %q. %v", canary.Name, err)
}
}
}
@@ -929,7 +930,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error {
}
// scheduling for this monitor has already been completed
if _, ok := c.mapping.Schedule[mon.DaemonName]; ok {
logger.Debugf("mon %s already scheduled", mon.DaemonName)
log.NamespacedDebug(c.Namespace, logger, "mon %s already scheduled", mon.DaemonName)
continue
}
@@ -951,14 +952,14 @@ func (c *Cluster) assignMons(mons []*monConfig) error {
result, err := waitForMonitorScheduling(c, deployment)
if err != nil {
logger.Errorf("failed to schedule mon %q. %v", mon.DaemonName, err)
log.NamespacedError(c.Namespace, logger, "failed to schedule mon %q. %v", mon.DaemonName, err)
failedMonSchedule = true
return
}
nodeChoice := result.Node
if nodeChoice == nil {
logger.Errorf("failed to schedule monitor %q", mon.DaemonName)
log.NamespacedError(c.Namespace, logger, "failed to schedule monitor %q", mon.DaemonName)
failedMonSchedule = true
return
}
@@ -968,21 +969,21 @@ func (c *Cluster) assignMons(mons []*monConfig) error {
// directly to a node selector on the monitor pod.
var schedule *controller.MonScheduleInfo
if c.spec.Network.IsHost() || c.monVolumeClaimTemplate(mon) == nil {
logger.Infof("mon %s assigned to node %s", mon.DaemonName, nodeChoice.Name)
log.NamespacedInfo(c.Namespace, logger, "mon %s assigned to node %s", mon.DaemonName, nodeChoice.Name)
schedule, err = getNodeInfoFromNode(*nodeChoice)
if err != nil {
logger.Errorf("failed to get node info for node %q. %v", nodeChoice.Name, err)
log.NamespacedError(c.Namespace, logger, "failed to get node info for node %q. %v", nodeChoice.Name, err)
failedMonSchedule = true
return
}
} else {
logger.Infof("mon %q placement using native scheduler", mon.DaemonName)
log.NamespacedInfo(c.Namespace, logger, "mon %q placement using native scheduler", mon.DaemonName)
}
if c.spec.ZonesRequired() {
if schedule == nil {
schedule = &controller.MonScheduleInfo{}
}
logger.Infof("mon %q is assigned to zone %q", mon.DaemonName, mon.Zone)
log.NamespacedInfo(c.Namespace, logger, "mon %q is assigned to zone %q", mon.DaemonName, mon.Zone)
schedule.Zone = mon.Zone
}
@@ -998,7 +999,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error {
return errors.New("failed to schedule mons")
}
logger.Debug("mons have been scheduled")
log.NamespacedDebug(c.Namespace, logger, "mons have been scheduled")
return nil
}
@@ -1040,10 +1041,10 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e
deployments, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).List(c.ClusterInfo.Context, metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", AppName)})
if err != nil {
if kerrors.IsNotFound(err) {
logger.Infof("0 of %d expected mon deployments exist. creating new deployment(s).", len(mons))
log.NamespacedInfo(c.Namespace, logger, "0 of %d expected mon deployments exist. creating new deployment(s).", len(mons))
onlyCheckQuorumOnce = true
} else {
logger.Warningf("failed to list mon deployments. attempting to continue. %v", err)
log.NamespacedWarning(c.Namespace, logger, "failed to list mon deployments. attempting to continue. %v", err)
}
}
@@ -1055,10 +1056,10 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e
}
}
if len(deployments.Items) < len(mons) {
logger.Infof("%d of %d expected mon deployments exist. creating new deployment(s).", len(deployments.Items), len(mons))
log.NamespacedInfo(c.Namespace, logger, "%d of %d expected mon deployments exist. creating new deployment(s).", len(deployments.Items), len(mons))
onlyCheckQuorumOnce = true
} else if readyReplicas == 0 {
logger.Infof("%d of %d expected mons are ready. creating or updating deployments without checking quorum in attempt to achieve a healthy mon cluster", readyReplicas, len(mons))
log.NamespacedInfo(c.Namespace, logger, "%d of %d expected mons are ready. creating or updating deployments without checking quorum in attempt to achieve a healthy mon cluster", readyReplicas, len(mons))
onlyCheckQuorumOnce = true
}
@@ -1076,7 +1077,7 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e
// we need to do everything possible to verify the basic health of a cluster, complete the first orchestration,
// and start watching for all the CRs. If mons still have quorum we can continue with the orchestration even
// if they aren't all up.
logger.Errorf("attempting to continue after failing to start mon %q. %v", mons[i].DaemonName, err)
log.NamespacedError(c.Namespace, logger, "attempting to continue after failing to start mon %q. %v", mons[i].DaemonName, err)
}
// For the initial deployment (first creation) it's expected to not have all the monitors in quorum
@@ -1091,13 +1092,13 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e
}
}
logger.Infof("mons created: %d", len(mons))
log.NamespacedInfo(c.Namespace, logger, "mons created: %d", len(mons))
// Final verification that **all** mons are in quorum
// Do not proceed if one monitor is still syncing
// Only do this when monitors versions are different so we don't block the orchestration if a mon is down.
versions, err := cephclient.GetAllCephDaemonVersions(c.context, c.ClusterInfo)
if err != nil {
logger.Warningf("failed to get ceph daemons versions; this likely means there is no cluster yet. %v", err)
log.NamespacedWarning(c.Namespace, logger, "failed to get ceph daemons versions; this likely means there is no cluster yet. %v", err)
} else {
if len(versions.Mon) != 1 {
requireAllInQuorum = true
@@ -1113,20 +1114,20 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e
func (c *Cluster) checkForExtraMonResources(mons []*monConfig, deployments []apps.Deployment) string {
// If there are fewer mon deployments than the desired count, no need to remove an extra.
if len(deployments) <= c.spec.Mon.Count || len(deployments) <= len(mons) {
logger.Debug("no extra mon deployments to remove")
log.NamespacedDebug(c.Namespace, logger, "no extra mon deployments to remove")
return ""
}
// If there are fewer mons in the list than expected, either new mons are being created for
// a new cluster, or a mon failover is in progress and the list of mons only
// includes the single mon that was just started
if len(mons) < c.spec.Mon.Count {
logger.Debug("new cluster or mon failover in progress, not checking for extra mon deployments")
log.NamespacedDebug(c.Namespace, logger, "new cluster or mon failover in progress, not checking for extra mon deployments")
return ""
}
// If there are more deployments than expected mons from the ceph quorum,
// find the extra mon deployment and clean it up.
logger.Infof("there is an extra mon deployment that is not needed and not in quorum")
log.NamespacedInfo(c.Namespace, logger, "there is an extra mon deployment that is not needed and not in quorum")
for _, deploy := range deployments {
monName := deploy.Labels[controller.DaemonIDLabel]
found := false
@@ -1138,7 +1139,7 @@ func (c *Cluster) checkForExtraMonResources(mons []*monConfig, deployments []app
}
}
if !found {
logger.Infof("deleting extra mon deployment %q", deploy.Name)
log.NamespacedInfo(c.Namespace, logger, "deleting extra mon deployment %q", deploy.Name)
c.removeMonResources(monName)
return monName
}
@@ -1227,7 +1228,7 @@ func (c *Cluster) persistExpectedMonDaemonsAsEndpointSlice() error {
// Theoretically, we could now go ahead with the normal code path to
// delete (both IPv4 and IPv6) EndpointSlices, but 0 mons is certainly
// an error state, so it's better to do nothing destructive right now.
logger.Debug("no mon addresses found, skipping endpointslice resource reconciliation")
log.NamespacedDebug(c.Namespace, logger, "no mon addresses found, skipping endpointslice resource reconciliation")
return nil
}
@@ -1241,7 +1242,7 @@ func (c *Cluster) persistExpectedMonDaemonsAsEndpointSlice() error {
}
ip := net.ParseIP(host)
if ip == nil {
logger.Warningf("invalid IP parsed from mon endpoint: %s", mon.Endpoint)
log.NamespacedWarning(c.Namespace, logger, "invalid IP parsed from mon endpoint: %s", mon.Endpoint)
continue
}
if ip.To4() != nil {
@@ -1271,12 +1272,12 @@ func (c *Cluster) createEndpointSliceForAddresses(addresses []string, addressTyp
}
if len(addresses) == 0 {
logger.Debugf("no %s addresses found, deleting existing %q endpointslice if exists", addressType, sliceName)
log.NamespacedDebug(c.Namespace, logger, "no %s addresses found, deleting existing %q endpointslice if exists", addressType, sliceName)
if err := client.Delete(c.ClusterInfo.Context, sliceName, metav1.DeleteOptions{}); err != nil {
if kerrors.IsNotFound(err) {
logger.Debugf("endpointslice %q not found, nothing to delete", sliceName)
log.NamespacedDebug(c.Namespace, logger, "endpointslice %q not found, nothing to delete", sliceName)
} else {
logger.Errorf("failed to delete endpointslice %q: %v", sliceName, err)
log.NamespacedError(c.Namespace, logger, "failed to delete endpointslice %q: %v", sliceName, err)
}
}
return nil
@@ -1324,13 +1325,13 @@ func (c *Cluster) createEndpointSliceForAddresses(addresses []string, addressTyp
return errors.Wrapf(err, "failed to create %s endpointslice", addressType)
}
logger.Debugf("updating existing %s endpointslice %s", addressType, sliceName)
log.NamespacedDebug(c.Namespace, logger, "updating existing %s endpointslice %s", addressType, sliceName)
if _, err = client.Update(c.ClusterInfo.Context, endpointSlice, metav1.UpdateOptions{}); err != nil {
return errors.Wrapf(err, "failed to update %s endpointslice", addressType)
}
}
logger.Infof("created/updated %s endpointslice with addresses: %+v", addressType, addresses)
log.NamespacedInfo(c.Namespace, logger, "created/updated %s endpointslice with addresses: %+v", addressType, addresses)
return nil
}
@@ -1397,12 +1398,12 @@ func (c *Cluster) persistExpectedMonDaemonsInConfigMap() error {
return errors.Wrap(err, "failed to create mon endpoint config map")
}
logger.Debugf("updating config map %s that already exists", configMap.Name)
log.NamespacedDebug(c.Namespace, logger, "updating config map %s that already exists", configMap.Name)
if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(c.ClusterInfo.Context, configMap, metav1.UpdateOptions{}); err != nil {
return errors.Wrap(err, "failed to update mon endpoint config map")
}
}
logger.Infof("saved mon endpoints to config map %+v", configMap.Data)
log.NamespacedInfo(c.Namespace, logger, "saved mon endpoints to config map %+v", configMap.Data)
return nil
}
@@ -1420,7 +1421,7 @@ func (c *Cluster) getStoredMaxMonID() (string, error) {
// if the configmap cannot be loaded, assume a new cluster. If the mons have previously
// been created, the maxMonID will anyway analyze them to ensure the index is correct
// even if this error occurs.
logger.Infof("existing maxMonID not found or failed to load. %v", err)
log.NamespacedInfo(c.Namespace, logger, "existing maxMonID not found or failed to load. %v", err)
return "-1", nil
}
@@ -1446,11 +1447,11 @@ func (c *Cluster) commitMaxMonIDRequireIncrementing(desiredMaxMonID int, require
}
if requireIncrementing && existingMax >= desiredMaxMonID {
logger.Infof("no need to commit maxMonID %d since it is not greater than existing maxMonID %d", desiredMaxMonID, existingMax)
log.NamespacedInfo(c.Namespace, logger, "no need to commit maxMonID %d since it is not greater than existing maxMonID %d", desiredMaxMonID, existingMax)
return nil
}
logger.Infof("updating maxMonID from %d to %d", existingMax, desiredMaxMonID)
log.NamespacedInfo(c.Namespace, logger, "updating maxMonID from %d to %d", existingMax, desiredMaxMonID)
configmap.Data[controller.MaxMonIDKey] = strconv.Itoa(desiredMaxMonID)
if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(c.ClusterInfo.Context, configmap, metav1.UpdateOptions{}); err != nil {
@@ -1476,7 +1477,7 @@ func (c *Cluster) updateMon(m *monConfig, d *apps.Deployment) error {
_ = k8sutil.ExpandPVCIfRequired(c.ClusterInfo.Context, c.context.Client, desiredPvc, existingPvc)
}
logger.Infof("deployment for mon %s already exists. updating if needed",
log.NamespacedInfo(c.Namespace, logger, "deployment for mon %s already exists. updating if needed",
d.Name)
err := updateDeploymentAndWait(c.context, c.ClusterInfo, d, config.MonType, m.DaemonName, c.spec.SkipUpgradeChecks, false)
@@ -1519,6 +1520,7 @@ func (c *Cluster) updateMon(m *monConfig, d *apps.Deployment) error {
func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) error {
// check if the monitor deployment already exists. if the deployment does
// exist, also determine if it using pvc storage.
log.NamespacedInfo(c.Namespace, logger, "starting mon %q", m.DaemonName)
pvcExists := false
deploymentExists := false
@@ -1554,10 +1556,10 @@ func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) e
pvcName := m.ResourceName
d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, controller.DaemonVolumesDataPVC(pvcName))
controller.AddVolumeMountSubPath(&d.Spec.Template.Spec, "ceph-daemon-data")
logger.Debugf("adding pvc volume source %s to mon deployment %s", pvcName, d.Name)
log.NamespacedDebug(c.Namespace, logger, "adding pvc volume source %s to mon deployment %s", pvcName, d.Name)
} else {
d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, controller.DaemonVolumesDataHostPath(m.DataPathMap)...)
logger.Debugf("adding host path volume source to mon deployment %s", d.Name)
log.NamespacedDebug(c.Namespace, logger, "adding host path volume source to mon deployment %s", d.Name)
}
// placement settings from the CRD
@@ -1611,7 +1613,7 @@ func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) e
_, err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Create(c.ClusterInfo.Context, pvc, metav1.CreateOptions{})
if err != nil {
if kerrors.IsAlreadyExists(err) {
logger.Debugf("cannot create mon %s pvc %s: already exists.", d.Name, pvc.Name)
log.NamespacedDebug(c.Namespace, logger, "cannot create mon %s pvc %s: already exists.", d.Name, pvc.Name)
} else {
return errors.Wrapf(err, "failed to create mon %s pvc %s", d.Name, pvc.Name)
}
@@ -1631,7 +1633,7 @@ func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) e
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), k8sutil.LabelHostname(),
map[string]string{k8sutil.AppAttr: AppName}, nodeSelector)
logger.Debugf("Starting mon: %+v", d.Name)
log.NamespacedDebug(c.Namespace, logger, "Starting mon: %+v", d.Name)
_, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(c.ClusterInfo.Context, d, metav1.CreateOptions{})
if err != nil {
return errors.Wrapf(err, "failed to create mon deployment %s", d.Name)
@@ -1667,10 +1669,10 @@ func isMonIPUpdateRequiredForHostNetwork(mon string, isMonUsingHostNetwork bool,
func hasMonPathChanged(d *apps.Deployment, claim *corev1.PersistentVolumeClaim) bool {
if d.Labels["pvc_name"] == "" && claim != nil {
logger.Infof("skipping update for mon %q where path has changed from hostPath to pvc", d.Name)
log.NamespacedInfo(d.Namespace, logger, "skipping update for mon %q where path has changed from hostPath to pvc", d.Name)
return true
} else if d.Labels["pvc_name"] != "" && claim == nil {
logger.Infof("skipping update for mon %q where path has changed from pvc to hostPath", d.Name)
log.NamespacedInfo(d.Namespace, logger, "skipping update for mon %q where path has changed from pvc to hostPath", d.Name)
return true
}
@@ -1678,7 +1680,7 @@ func hasMonPathChanged(d *apps.Deployment, claim *corev1.PersistentVolumeClaim)
}
func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, mons []string, sleepTime int, requireAllInQuorum bool) error {
logger.Infof("waiting for mon quorum with %v", mons)
log.NamespacedInfo(clusterInfo.Namespace, logger, "waiting for mon quorum with %v", mons)
// wait for monitors to establish quorum
retryCount := 0
@@ -1705,18 +1707,18 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.Cl
for _, m := range mons {
running, err := k8sutil.PodsRunningWithLabel(clusterInfo.Context, context.Clientset, clusterInfo.Namespace, fmt.Sprintf("app=%s,mon=%s", AppName, m))
if err != nil {
logger.Infof("failed to query mon pod status, trying again. %v", err)
log.NamespacedInfo(clusterInfo.Namespace, logger, "failed to query mon pod status, trying again. %v", err)
continue
}
if running > 0 {
runningMonNames = append(runningMonNames, m)
} else {
allPodsRunning = false
logger.Infof("mon %s is not yet running", m)
log.NamespacedInfo(clusterInfo.Namespace, logger, "mon %s is not yet running", m)
}
}
logger.Infof("mons running: %v", runningMonNames)
log.NamespacedInfo(clusterInfo.Namespace, logger, "mons running: %v", runningMonNames)
if !allPodsRunning && requireAllInQuorum {
continue
}
@@ -1725,12 +1727,12 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.Cl
// their quorum status
monQuorumStatusResp, err := cephclient.GetMonQuorumStatus(context, clusterInfo)
if err != nil {
logger.Debugf("failed to get quorum_status. %v", err)
log.NamespacedDebug(clusterInfo.Namespace, logger, "failed to get quorum_status. %v", err)
continue
}
if !requireAllInQuorum {
logQuorumMembers(monQuorumStatusResp)
logQuorumMembers(clusterInfo.Namespace, monQuorumStatusResp)
break
}
@@ -1739,14 +1741,14 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.Cl
for _, name := range mons {
if !monFoundInQuorum(name, monQuorumStatusResp) {
// found an initial monitor that is not in quorum, bail out of this retry
logger.Warningf("monitor %s is not in quorum list", name)
log.NamespacedWarning(clusterInfo.Namespace, logger, "monitor %s is not in quorum list", name)
allInQuorum = false
break
}
}
if allInQuorum {
logQuorumMembers(monQuorumStatusResp)
logQuorumMembers(clusterInfo.Namespace, monQuorumStatusResp)
break
}
}
@@ -1754,14 +1756,14 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.Cl
return nil
}
func logQuorumMembers(monQuorumStatusResp cephclient.MonStatusResponse) {
func logQuorumMembers(namespace string, monQuorumStatusResp cephclient.MonStatusResponse) {
var monsInQuorum []string
for _, m := range monQuorumStatusResp.MonMap.Mons {
if monFoundInQuorum(m.Name, monQuorumStatusResp) {
monsInQuorum = append(monsInQuorum, m.Name)
}
}
logger.Infof("Monitors in quorum: %v", monsInQuorum)
log.NamespacedInfo(namespace, logger, "Monitors in quorum: %v", monsInQuorum)
}
func monFoundInQuorum(name string, monQuorumStatusResp cephclient.MonStatusResponse) bool {
@@ -1796,14 +1798,14 @@ func requiredDuringScheduling(spec *cephv1.ClusterSpec) bool {
}
func (c *Cluster) acquireOrchestrationLock() {
logger.Debugf("Acquiring lock for mon orchestration")
log.NamespacedDebug(c.Namespace, logger, "Acquiring lock for mon orchestration")
c.orchestrationMutex.Lock()
logger.Debugf("Acquired lock for mon orchestration")
log.NamespacedDebug(c.Namespace, logger, "Acquired lock for mon orchestration")
}
func (c *Cluster) releaseOrchestrationLock() {
c.orchestrationMutex.Unlock()
logger.Debugf("Released lock for mon orchestration")
log.NamespacedDebug(c.Namespace, logger, "Released lock for mon orchestration")
}
func (c *Cluster) RotateMonCephxKeys(clusterObj *cephv1.CephCluster) (bool, error) {
@@ -1817,16 +1819,16 @@ func (c *Cluster) RotateMonCephxKeys(clusterObj *cephv1.CephCluster) (bool, erro
}
if !shouldRotateMonKeys {
logger.Debugf("cephx key rotation for mon daemon in the namespace %q is not required", c.ClusterInfo.Namespace)
log.NamespacedDebug(c.Namespace, logger, "cephx key rotation for mon daemon in the namespace %q is not required", c.ClusterInfo.Namespace)
return shouldRotateMonKeys, nil
}
if !c.ClusterInfo.CephVersion.IsAtLeast(keyring.CephAuthMonRotateSupportedVersion) {
logger.Debugf("cephx key rotation for mons in namespace %q is indicated, but ceph version %#v does not support mon key rotation", c.Namespace, c.ClusterInfo.CephVersion)
log.NamespacedDebug(c.Namespace, logger, "cephx key rotation for mons in namespace %q is indicated, but ceph version %#v does not support mon key rotation", c.Namespace, c.ClusterInfo.CephVersion)
return false, nil
}
logger.Infof("cephx keys for mon daemons in the namespace %q will be rotated", c.ClusterInfo.Namespace)
log.NamespacedInfo(c.Namespace, logger, "cephx keys for mon daemons in the namespace %q will be rotated", c.ClusterInfo.Namespace)
k := keyring.GetSecretStore(c.context, c.ClusterInfo, c.ClusterInfo.OwnerInfo)
newKey, err := k.RotateKey(controller.MonCephxUser)
@@ -1847,7 +1849,7 @@ func (c *Cluster) RotateMonCephxKeys(clusterObj *cephv1.CephCluster) (bool, erro
return shouldRotateMonKeys, errors.Wrapf(err, "failed to save mon keyring secret after rotating the mon cephx keys in the namespace %q", c.ClusterInfo.Namespace)
}
logger.Infof("successfully rotated cephx keys for mon daemons in the namespace %q", c.ClusterInfo.Namespace)
log.NamespacedInfo(c.Namespace, logger, "successfully rotated cephx keys for mon daemons in the namespace %q", c.ClusterInfo.Namespace)
return shouldRotateMonKeys, nil
}
@@ -1860,11 +1862,11 @@ func (c *Cluster) UpdateMonCephxStatus(didRotate bool) error {
}
updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, c.ClusterInfo.CephVersion, cluster.Status.Cephx.Mon)
cluster.Status.Cephx.Mon = updatedStatus
logger.Debugf("updating mon daemon cephx status to %+v", cluster.Status.Cephx.Mgr)
log.NamespacedDebug(c.Namespace, logger, "updating mon daemon cephx status to %+v", cluster.Status.Cephx.Mgr)
if err := reporting.UpdateStatus(c.context.Client, cluster); err != nil {
return errors.Wrapf(err, "failed to update cluster cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace)
}
logger.Infof("successfully updated the cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace)
log.NamespacedInfo(c.Namespace, logger, "successfully updated the cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace)
return nil
})
+8 -7
View File
@@ -22,6 +22,7 @@ import (
"sort"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/util/log"
corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -42,8 +43,8 @@ func PredicateMonEndpointChanges[T *corev1.ConfigMap]() predicate.TypedFuncs[T]
cmOld := (*corev1.ConfigMap)(e.ObjectOld)
cmNew := (*corev1.ConfigMap)(e.ObjectNew)
if cmNew.GetName() == EndpointConfigMapName {
if wereMonEndpointsUpdated(cmOld.Data, cmNew.Data) {
logger.Info("monitor endpoints changed, updating the bootstrap peer token")
if wereMonEndpointsUpdated(cmNew.Namespace, cmOld.Data, cmNew.Data) {
log.NamespacedInfo(cmNew.Namespace, logger, "monitor endpoints changed, updating the bootstrap peer token")
return true
}
}
@@ -52,7 +53,7 @@ func PredicateMonEndpointChanges[T *corev1.ConfigMap]() predicate.TypedFuncs[T]
}
}
func wereMonEndpointsUpdated(oldCMData, newCMData map[string]string) bool {
func wereMonEndpointsUpdated(namespace string, oldCMData, newCMData map[string]string) bool {
// Check the mapping key first
if oldMapping, ok := oldCMData["mapping"]; ok {
if newMapping, ok := newCMData["mapping"]; ok {
@@ -60,20 +61,20 @@ func wereMonEndpointsUpdated(oldCMData, newCMData map[string]string) bool {
var oldMappingToGo opcontroller.Mapping
err := json.Unmarshal([]byte(oldMapping), &oldMappingToGo)
if err != nil {
logger.Debugf("failed to unmarshal new. %v", err)
log.NamespacedDebug(namespace, logger, "failed to unmarshal new. %v", err)
return false
}
var newMappingToGo opcontroller.Mapping
err = json.Unmarshal([]byte(newMapping), &newMappingToGo)
if err != nil {
logger.Debugf("failed to unmarshal new. %v", err)
log.NamespacedDebug(namespace, logger, "failed to unmarshal new. %v", err)
return false
}
// If length is different, monitors are different
if len(oldMappingToGo.Schedule) != len(newMappingToGo.Schedule) {
logger.Debugf("mons were added or removed from the endpoints cm")
log.NamespacedDebug(namespace, logger, "mons were added or removed from the endpoints cm")
return true
}
// Since Schedule is map, it's unordered, so let's order it
@@ -92,7 +93,7 @@ func wereMonEndpointsUpdated(oldCMData, newCMData map[string]string) bool {
// Iterate over the map and compare the values
for _, v := range oldKeys {
if !reflect.DeepEqual(oldMappingToGo.Schedule[v], newMappingToGo.Schedule[v]) {
logger.Debugf("oldMappingToGo.Schedule[v] AND newMappingToGo.Schedule[v]: %v | %v", oldMappingToGo.Schedule[v], newMappingToGo.Schedule[v])
log.NamespacedDebug(namespace, logger, "oldMappingToGo.Schedule[v] AND newMappingToGo.Schedule[v]: %v | %v", oldMappingToGo.Schedule[v], newMappingToGo.Schedule[v])
return true
}
}
@@ -37,7 +37,7 @@ func TestWereMonEndpointsUpdated(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := wereMonEndpointsUpdated(tt.args.oldCMData, tt.args.newCMData); got != tt.want {
if got := wereMonEndpointsUpdated("ns", tt.args.oldCMData, tt.args.newCMData); got != tt.want {
t.Errorf("whereMonEndpointsUpdated() = %v, want %v", got, tt.want)
}
})
+5 -4
View File
@@ -21,6 +21,7 @@ import (
"github.com/pkg/errors"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
v1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -54,7 +55,7 @@ func (c *Cluster) createService(mon *monConfig) (*v1.Service, error) {
if mon.PublicIP != "" {
_, err := c.context.Clientset.CoreV1().Services(c.Namespace).Get(c.ClusterInfo.Context, svcDef.Name, metav1.GetOptions{})
if err != nil && kerrors.IsNotFound(err) {
logger.Infof("ensuring the clusterIP for mon %q is %q", mon.DaemonName, mon.PublicIP)
log.NamespacedInfo(c.Namespace, logger, "ensuring the clusterIP for mon %q is %q", mon.DaemonName, mon.PublicIP)
svcDef.Spec.ClusterIP = mon.PublicIP
}
}
@@ -69,7 +70,7 @@ func (c *Cluster) createService(mon *monConfig) (*v1.Service, error) {
return nil, nil
}
logger.Infof("mon %q cluster IP is %s", mon.DaemonName, s.Spec.ClusterIP)
log.NamespacedInfo(c.Namespace, logger, "mon %q cluster IP is %s", mon.DaemonName, s.Spec.ClusterIP)
return s, nil
}
@@ -78,12 +79,12 @@ func (c *Cluster) exportService(service *v1.Service, monDaemon string) (string,
// query on <service>.<ns>.svc.clusterset.local requires the mon canary pod to be running
defer c.removeCanaryDeployments(monCanaryLabelSelector + fmt.Sprintf(",mon=%s", monDaemon))
logger.Infof("exporting service %q", service.Name)
log.NamespacedInfo(c.Namespace, logger, "exporting service %q", service.Name)
exportedIP, err := k8sutil.ExportService(c.ClusterInfo.Context, c.context, service, c.spec.Network.MultiClusterService.ClusterID)
if err != nil {
return "", errors.Wrapf(err, "failed to export service %q", service.Name)
}
logger.Infof("mon %q exported IP is %s", service.Name, exportedIP)
log.NamespacedInfo(c.Namespace, logger, "mon %q exported IP is %s", service.Name, exportedIP)
return exportedIP, nil
}
+6 -6
View File
@@ -28,6 +28,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
apps "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
@@ -187,7 +188,7 @@ func makeMonSecurityContext() *corev1.PodSecurityContext {
}
func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*corev1.Pod, error) {
logger.Debugf("monConfig: %+v", monConfig)
log.NamespacedDebug(c.Namespace, logger, "monConfig: %+v", monConfig)
podSpec := corev1.PodSpec{
InitContainers: []corev1.Container{
c.makeChownInitContainer(monConfig),
@@ -393,17 +394,16 @@ func UpdateCephDeploymentAndWait(context *clusterd.Context, clusterInfo *client.
callback := func(action string) error {
// At this point, we are in an upgrade
if skipUpgradeChecks {
logger.Warningf("this is a Ceph upgrade, not performing upgrade checks because skipUpgradeChecks is %t", skipUpgradeChecks)
log.NamespacedWarning(clusterInfo.Namespace, logger, "this is a Ceph upgrade, not performing upgrade checks because skipUpgradeChecks is %t", skipUpgradeChecks)
return nil
}
logger.Infof("checking if we can %s the deployment %s", action, deployment.Name)
log.NamespacedInfo(clusterInfo.Namespace, logger, "checking if we can %s the deployment %s", action, deployment.Name)
if action == "stop" {
err := client.OkToStop(context, clusterInfo, deployment.Name, daemonType, daemonName)
if err != nil {
if continueUpgradeAfterChecksEvenIfNotHealthy {
logger.Infof("The %s daemon %s is not ok-to-stop but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so proceeding to stop...", daemonType, daemonName)
log.NamespacedInfo(clusterInfo.Namespace, logger, "The %s daemon %s is not ok-to-stop but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so proceeding to stop...", daemonType, daemonName)
return nil
}
return errors.Wrapf(err, "failed to check if we can %s the deployment %s", action, deployment.Name)
@@ -414,7 +414,7 @@ func UpdateCephDeploymentAndWait(context *clusterd.Context, clusterInfo *client.
err := client.OkToContinue(context, clusterInfo, deployment.Name, daemonType, daemonName)
if err != nil {
if continueUpgradeAfterChecksEvenIfNotHealthy {
logger.Infof("The %s daemon %s is not ok-to-continue but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so continuing...", daemonType, daemonName)
log.NamespacedInfo(clusterInfo.Namespace, logger, "The %s daemon %s is not ok-to-continue but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so continuing...", daemonType, daemonName)
return nil
}
return errors.Wrapf(err, "failed to check if we can %s the deployment %s", action, deployment.Name)
+5 -4
View File
@@ -25,6 +25,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/util/log"
)
var monitorDaemonList = []string{"mon", "osd", "status"}
@@ -37,7 +38,7 @@ func (c *ClusterController) configureCephMonitoring(cluster *cluster, clusterInf
if health, ok := cluster.monitoringRoutines.Load(daemon); ok {
// If the context Err() is nil this means it hasn't been cancelled yet
if health.(*opcontroller.ClusterHealth).InternalCtx.Err() == nil {
logger.Debugf("monitoring routine for %q is already running", daemon)
log.NamespacedDebug(cluster.Namespace, logger, "monitoring routine for %q is already running", daemon)
if !isEnabled {
health.(*opcontroller.ClusterHealth).InternalCancel()
}
@@ -79,19 +80,19 @@ func (c *ClusterController) startMonitoringCheck(cluster *cluster, clusterInfo *
switch daemon {
case "mon":
healthChecker := mon.NewHealthChecker(cluster.mons)
logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
log.NamespacedInfo(cluster.Namespace, logger, "enabling ceph %s monitoring goroutine", daemon)
go healthChecker.Check(&cluster.monitoringRoutines, daemon)
case "osd":
if !cluster.Spec.External.Enable {
osdChecker := osd.NewOSDHealthMonitor(c.context, clusterInfo, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Spec.HealthCheck)
logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
log.NamespacedInfo(cluster.Namespace, logger, "enabling ceph %s monitoring goroutine", daemon)
go osdChecker.Start(&cluster.monitoringRoutines, daemon)
}
case "status":
cephChecker := newCephStatusChecker(c.context, clusterInfo, cluster.Spec)
logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
log.NamespacedInfo(cluster.Namespace, logger, "enabling ceph %s monitoring goroutine", daemon)
go cephChecker.checkCephStatus(&cluster.monitoringRoutines, daemon)
}
}
@@ -28,6 +28,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/util/log"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
@@ -57,11 +58,11 @@ func (r *ReconcileNode) createOrUpdateCephExporter(node corev1.Node, tolerations
// CephVersion change is done temporarily, as some regression was detected in Ceph version 17.2.6 which is summarised here https://github.com/ceph/ceph/pull/50718#issuecomment-1505608312.
// Thus, disabling ceph-exporter for now until all the regression are fixed.
if !cephVersion.IsAtLeast(MinVersionForCephExporter) {
logger.Infof("Skipping exporter reconcile on ceph version %q", cephVersion.String())
log.NamespacedInfo(cephCluster.Namespace, logger, "Skipping exporter reconcile on ceph version %q", cephVersion.String())
return controllerutil.OperationResultNone, nil
}
if cephCluster.Spec.Monitoring.MetricsDisabled {
logger.Info("Skipping exporter reconcile since monitoring.metricsDisabled is true")
log.NamespacedInfo(cephCluster.Namespace, logger, "Skipping exporter reconcile since monitoring.metricsDisabled is true")
return controllerutil.OperationResultNone, nil
}
@@ -283,10 +284,10 @@ func applyCephExporterLabels(cephCluster cephv1.CephCluster, serviceMonitor *mon
serviceMonitor.Spec.Endpoints[0].RelabelConfigs = append(
serviceMonitor.Spec.Endpoints[0].RelabelConfigs, relabelConfig)
} else {
logger.Info("rook.io/managedBy not specified in ceph-exporter labels")
log.NamespacedInfo(cephCluster.Namespace, logger, "rook.io/managedBy not specified in ceph-exporter labels")
}
} else {
logger.Debug("ceph-exporter labels not specified")
log.NamespacedDebug(cephCluster.Namespace, logger, "ceph-exporter labels not specified")
}
}
}
@@ -26,6 +26,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
"github.com/rook/rook/pkg/operator/ceph/reporting"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/util/retry"
@@ -96,7 +97,7 @@ func createCrashCollectorKeyring(s *keyring.SecretStore, context *clusterd.Conte
}
if shouldRotateCephxKeys {
logger.Infof("rotating cephx key for crash collector %q", crashCollectorKeyringUsername)
log.NamespacedInfo(clusterInfo.Namespace, logger, "rotating cephx key for crash collector %q", crashCollectorKeyringUsername)
newKey, err := s.RotateKey(crashCollectorKeyringUsername)
if err != nil {
return "", errors.Wrapf(err, "failed to rotate cephx key for crash collector %q", crashCollectorKeyringUsername)
@@ -124,7 +125,7 @@ func updateCrashCollectorCephxStatus(context *clusterd.Context, clusterInfo *cli
if err := reporting.UpdateStatus(context.Client, cluster); err != nil {
return errors.Wrap(err, "failed to update cluster cephx status for crash collector daemon")
}
logger.Debugf("successfully updated the crash collector cephx status for cluster in namespace %q to %+v", cluster.Namespace, cluster.Status.Cephx.CrashCollector)
log.NamespacedDebug(clusterInfo.Namespace, logger, "successfully updated the crash collector cephx status for cluster in namespace %q to %+v", cluster.Namespace, cluster.Status.Cephx.CrashCollector)
return nil
})
@@ -161,7 +162,7 @@ func createOrUpdateCrashCollectorSecret(clusterInfo *client.ClusterInfo, crashCo
return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", s.Name, clusterInfo.Namespace)
}
logger.Infof("created kubernetes crash collector secret for cluster %q", clusterInfo.Namespace)
log.NamespacedInfo(clusterInfo.Namespace, logger, "created kubernetes crash collector secret")
return nil
}
@@ -212,7 +213,7 @@ func createExporterKeyring(s *keyring.SecretStore, context *clusterd.Context, cl
}
if shouldRotateCephxKeys {
logger.Infof("rotating cephx key for ceph exporter %q", exporterKeyringUsername)
log.NamespacedInfo(clusterInfo.Namespace, logger, "rotating cephx key for ceph exporter %q", exporterKeyringUsername)
newKey, err := s.RotateKey(exporterKeyringUsername)
if err != nil {
return "", errors.Wrapf(err, "failed to rotate cephx key for ceph exporter %q", exporterKeyringUsername)
@@ -239,7 +240,7 @@ func updateCephExporterCephxStatus(context *clusterd.Context, clusterInfo *clien
if err := reporting.UpdateStatus(context.Client, cluster); err != nil {
return errors.Wrap(err, "failed to update cluster cephx status for ceph exporter daemon")
}
logger.Debugf("successfully updated the ceph exporter cephx status for cluster in namespace %q to %+v", cluster.Namespace, cluster.Status.Cephx.CephExporter)
log.NamespacedDebug(clusterInfo.Namespace, logger, "successfully updated the ceph exporter cephx status for cluster in namespace %q to %+v", cluster.Namespace, cluster.Status.Cephx.CephExporter)
return nil
})
@@ -276,6 +277,6 @@ func createOrUpdateExporterSecret(clusterInfo *client.ClusterInfo, exporterSecre
return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", s.Name, clusterInfo.Namespace)
}
logger.Infof("created kubernetes exporter secret for cluster %q", clusterInfo.Namespace)
log.NamespacedInfo(clusterInfo.Namespace, logger, "created kubernetes exporter secret for cluster %q", clusterInfo.Namespace)
return nil
}
@@ -25,6 +25,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
v1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
@@ -34,7 +35,7 @@ import (
func (r *ReconcileNode) reconcileCrashPruner(namespace string, cephCluster cephv1.CephCluster, tolerations []corev1.Toleration) error {
if cephCluster.Spec.CrashCollector.Disable {
logger.Debugf("crash collector is disabled in namespace %q so skipping crash retention reconcile", namespace)
log.NamespacedDebug(cephCluster.Namespace, logger, "crash collector is disabled in namespace %q so skipping crash retention reconcile", namespace)
return nil
}
@@ -44,27 +45,27 @@ func (r *ReconcileNode) reconcileCrashPruner(namespace string, cephCluster cephv
}
if cephCluster.Spec.CrashCollector.DaysToRetain == 0 {
logger.Debug("deleting cronjob if it exists...")
log.NamespacedDebug(cephCluster.Namespace, logger, "deleting cronjob if it exists...")
cronJob := &v1.CronJob{ObjectMeta: objectMeta}
err := r.client.Delete(r.opManagerContext, cronJob)
if err != nil {
if kerrors.IsNotFound(err) {
logger.Debug("cronJob resource not found. Ignoring since object must be deleted.")
log.NamespacedDebug(cephCluster.Namespace, logger, "cronJob resource not found. Ignoring since object must be deleted.")
} else {
return err
}
} else {
logger.Debug("successfully deleted crash pruner cronjob.")
log.NamespacedDebug(cephCluster.Namespace, logger, "successfully deleted crash pruner cronjob.")
}
} else {
logger.Debugf("daysToRetain set to: %d", cephCluster.Spec.CrashCollector.DaysToRetain)
log.NamespacedDebug(cephCluster.Namespace, logger, "daysToRetain set to: %d", cephCluster.Spec.CrashCollector.DaysToRetain)
op, err := r.createOrUpdateCephCron(cephCluster, tolerations)
if err != nil {
return errors.Wrapf(err, "node reconcile failed on op %q", op)
}
logger.Debugf("cronjob successfully reconciled. operation: %q", op)
log.NamespacedDebug(cephCluster.Namespace, logger, "cronjob successfully reconciled. operation: %q", op)
}
return nil
}
@@ -26,6 +26,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
"github.com/rook/rook/pkg/operator/ceph/cluster/rbd"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/util/log"
appsv1 "k8s.io/api/apps/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
@@ -77,7 +78,7 @@ func (r *ReconcileNode) Reconcile(context context.Context, request reconcile.Req
// workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface
result, err := r.reconcile(request)
if err != nil {
logger.Error(err)
log.NamespacedError(request.Namespace, logger, "%s", err.Error())
}
return result, err
}
@@ -95,7 +96,7 @@ func (r *ReconcileNode) cleanupExporterResources(ns string, nodeName string) (re
}
func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result, error) {
logger.Debugf("reconciling node: %q", request.Name)
log.NamespacedDebug(request.Namespace, logger, "reconciling node: %q", request.Name)
// get the node object
node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: request.Name}}
@@ -105,7 +106,7 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result,
// if a node is not present, check if there are any node daemons to remove
err := r.listNodeDaemonsAndDelete(request.Name, "")
if err != nil {
logger.Errorf("failed to list and delete deployment on node %q; user should delete them manually. %v", request.Name, err)
log.NamespacedError(request.Namespace, logger, "failed to list and delete deployment on node %q; user should delete them manually. %v", request.Name, err)
}
} else {
return reconcile.Result{}, errors.Wrapf(err, "could not get node %q", request.Name)
@@ -144,13 +145,13 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result,
return reconcile.Result{}, errors.Wrapf(err, "could not get cephcluster in namespaces %q", namespace)
}
if len(cephClusters.Items) < 1 {
logger.Debugf("no CephCluster found in the namespace %q", namespace)
log.NamespacedDebug(namespace, logger, "no CephCluster found in the namespace %q", namespace)
return reconcile.Result{}, nil
}
cephCluster := cephClusters.Items[0]
if len(cephClusters.Items) > 1 {
logger.Errorf("more than one CephCluster found in the namespace %q, choosing the first one %q", namespace, cephCluster.GetName())
log.NamespacedError(request.Namespace, logger, "more than one CephCluster found in the namespace %q, choosing the first one %q", namespace, cephCluster.GetName())
}
allDisabled := r.removeDisabledCrashCollectorDaemons(cephCluster.Spec, namespace) && r.removeDisabledCephExporterDaemons(cephCluster.Spec, namespace)
@@ -164,7 +165,7 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result,
err = r.client.Get(r.opManagerContext, types.NamespacedName{Name: crashCollectorKeyName, Namespace: namespace}, secret)
if err != nil {
if kerrors.IsNotFound(err) {
logger.Debugf("secret %q in namespace %q not found. retrying in %q. %v", crashCollectorKeyName, namespace, waitForRequeueIfSecretNotCreated.RequeueAfter.String(), err)
log.NamespacedDebug(namespace, logger, "secret %q in namespace %q not found. retrying in %q. %v", crashCollectorKeyName, namespace, waitForRequeueIfSecretNotCreated.RequeueAfter.String(), err)
return waitForRequeueIfSecretNotCreated, nil
}
@@ -174,7 +175,7 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result,
clusterImage := cephCluster.Spec.CephVersion.Image
cephVersion, err := opcontroller.GetImageVersion(cephCluster)
if err != nil {
logger.Errorf("ceph version not found for image %q used by cluster %q in namespace %q. %v", clusterImage, cephCluster.Name, cephCluster.Namespace, err)
log.NamespacedError(request.Namespace, logger, "ceph version not found for image %q used by cluster %q in namespace %q. %v", clusterImage, cephCluster.Name, cephCluster.Namespace, err)
return reconcile.Result{}, nil
}
@@ -233,19 +234,19 @@ func (r *ReconcileNode) createOrUpdateNodeDaemons(node corev1.Node, tolerations
op, err := r.createOrUpdateCephCrash(node, tolerations, cephCluster, cephVersion)
if err != nil {
if op == "unchanged" {
logger.Debugf("crash collector unchanged on node %q", node.Name)
log.NamespacedDebug(cephCluster.Namespace, logger, "crash collector unchanged on node %q", node.Name)
} else {
return errors.Wrapf(err, "crash collector reconcile failed on op %q", op)
}
} else {
logger.Debugf("crash collector successfully reconciled for node %q. operation: %q", node.Name, op)
log.NamespacedDebug(cephCluster.Namespace, logger, "crash collector successfully reconciled for node %q. operation: %q", node.Name, op)
}
}
if cephVersion.IsAtLeast(MinVersionForCephExporter) && !cephCluster.Spec.Monitoring.MetricsDisabled {
op, err := r.createOrUpdateCephExporter(node, tolerations, cephCluster, cephVersion)
if err != nil {
if op == "unchanged" {
logger.Debugf("ceph exporter unchanged on node %q", node.Name)
log.NamespacedDebug(cephCluster.Namespace, logger, "ceph exporter unchanged on node %q", node.Name)
} else {
return errors.Wrapf(err, "ceph exporter reconcile failed on op %q", op)
}
@@ -253,7 +254,7 @@ func (r *ReconcileNode) createOrUpdateNodeDaemons(node corev1.Node, tolerations
// CephVersion change is done temporarily, as some regression was detected in Ceph version 17.2.6 which is summarised here https://github.com/ceph/ceph/pull/50718#issuecomment-1505608312.
// Thus, disabling ceph-exporter for now until all the regression are fixed.
if cephVersion.IsAtLeast(MinVersionForCephExporter) {
logger.Debugf("ceph exporter successfully reconciled for node %q. operation: %q", node.Name, op)
log.NamespacedDebug(cephCluster.Namespace, logger, "ceph exporter successfully reconciled for node %q. operation: %q", node.Name, op)
// create the metrics service
service, err := MakeCephExporterMetricsService(cephCluster, exporterServiceMetricName, r.scheme)
if err != nil {
@@ -267,7 +268,7 @@ func (r *ReconcileNode) createOrUpdateNodeDaemons(node corev1.Node, tolerations
if err := EnableCephExporterServiceMonitor(r.context, cephCluster, r.scheme, r.opManagerContext, exporterServiceMetricName); err != nil {
return errors.Wrap(err, "failed to enable service monitor")
}
logger.Debug("service monitor for ceph exporter was enabled successfully")
log.NamespacedDebug(cephCluster.Namespace, logger, "service monitor for ceph exporter was enabled successfully")
}
}
}
@@ -302,12 +303,12 @@ func (r *ReconcileNode) listDeploymentAndDelete(appName, nodeName, ns string) er
return errors.Wrapf(err, "failed to list deployments in namespace %q", ns)
}
for _, d := range deploymentList.Items {
logger.Infof("deleting deployment %q for node %q", d.ObjectMeta.Name, nodeName)
log.NamespacedInfo(d.Namespace, logger, "deleting deployment %q for node %q", d.ObjectMeta.Name, nodeName)
err := r.deleteDeployment(d)
if err != nil {
return errors.Wrapf(err, "failed to delete deployment %q in namespace %q", d.Name, d.Namespace)
}
logger.Infof("successfully removed deployment %q in namespace %q from node %q", d.Name, d.Namespace, nodeName)
log.NamespacedInfo(d.Namespace, logger, "successfully removed deployment %q in namespace %q from node %q", d.Name, d.Namespace, nodeName)
}
return nil
@@ -320,7 +321,7 @@ func (r *ReconcileNode) deleteNodeDaemon(appName, namespace string) {
// Try to fetch the list of existing deployment and remove them
err := r.client.List(r.opManagerContext, deploymentList, client.MatchingLabels{k8sutil.AppAttr: appName}, namespaceListOpts)
if err != nil {
logger.Errorf("failed to list deployments in namespace %q, delete it/them manually. %v", namespace, err)
log.NamespacedError(namespace, logger, "failed to list deployments in namespace %q, delete it/them manually. %v", namespace, err)
return
}
@@ -328,10 +329,10 @@ func (r *ReconcileNode) deleteNodeDaemon(appName, namespace string) {
for _, d := range deploymentList.Items {
err := r.deleteDeployment(d)
if err != nil {
logger.Errorf("failed to delete deployment %q in namespace %q, delete it manually. %v", d.Name, d.Namespace, err)
log.NamespacedError(namespace, logger, "failed to delete deployment %q in namespace %q, delete it manually. %v", d.Name, d.Namespace, err)
continue
}
logger.Infof("Deployments %q in namespace %q successfully removed", d.Name, d.Namespace)
log.NamespacedInfo(d.Namespace, logger, "Deployments %q in namespace %q successfully removed", d.Name, d.Namespace)
}
}
+20 -19
View File
@@ -27,6 +27,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/sets"
)
@@ -79,20 +80,20 @@ func (c *createConfig) createNewOSDsFromStatus(
) {
if !c.awaitingStatusConfigMaps.Has(statusConfigMapName(nodeOrPVCName)) {
// If there is a dangling OSD prepare configmap from another reconcile, don't process it
logger.Infof("not creating deployments for OSD prepare results found in ConfigMap %q which was not created for the latest storage spec", statusConfigMapName(nodeOrPVCName))
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "not creating deployments for OSD prepare results found in ConfigMap %q which was not created for the latest storage spec", statusConfigMapName(nodeOrPVCName))
return
}
if c.finishedStatusConfigMaps.Has(statusConfigMapName(nodeOrPVCName)) {
// If we have already processed this configmap, don't process it again
logger.Infof("not creating deployments for OSD prepare results found in ConfigMap %q which was already processed", statusConfigMapName(nodeOrPVCName))
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "not creating deployments for OSD prepare results found in ConfigMap %q which was already processed", statusConfigMapName(nodeOrPVCName))
return
}
for i, osd := range status.OSDs {
if c.deployments.Exists(osd.ID) {
// This OSD will be handled by the updater
logger.Debugf("not creating deployment for OSD %d which already exists", osd.ID)
log.NamespacedDebug(c.cluster.clusterInfo.Namespace, logger, "not creating deployment for OSD %d which already exists", osd.ID)
continue
}
@@ -102,13 +103,13 @@ func (c *createConfig) createNewOSDsFromStatus(
c.cluster.clusterInfo.CephVersion, keyring.UninitializedCephxStatus())
if status.PvcBackedOSD {
logger.Infof("creating OSD %d on PVC %q", osd.ID, nodeOrPVCName)
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "creating OSD %d on PVC %q", osd.ID, nodeOrPVCName)
err := createDaemonOnPVCFunc(c.cluster, &status.OSDs[i], nodeOrPVCName, c.provisionConfig)
if err != nil {
errs.addError("%v", errors.Wrapf(err, "failed to create OSD %d on PVC %q", osd.ID, nodeOrPVCName))
}
} else {
logger.Infof("creating OSD %d on node %q", osd.ID, nodeOrPVCName)
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "creating OSD %d on node %q", osd.ID, nodeOrPVCName)
err := createDaemonOnNodeFunc(c.cluster, &status.OSDs[i], nodeOrPVCName, c.provisionConfig)
if err != nil {
errs.addError("%v", errors.Wrapf(err, "failed to create OSD %d on node %q", osd.ID, nodeOrPVCName))
@@ -136,7 +137,7 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig, errs *provi
// no valid VolumeSource is ready to run an osd
if len(c.deviceSets) == 0 {
logger.Info("no storageClassDeviceSets defined to configure OSDs on PVCs")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "no storageClassDeviceSets defined to configure OSDs on PVCs")
return sets.New[string](), nil
}
@@ -161,14 +162,14 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig, errs *provi
metadataSource, metadataOK := volume.PVCSources[bluestorePVCMetadata]
if metadataOK {
logger.Infof("OSD will have its main bluestore block on %q and its metadata device on %q", dataSource.ClaimName, metadataSource.ClaimName)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its main bluestore block on %q and its metadata device on %q", dataSource.ClaimName, metadataSource.ClaimName)
} else {
logger.Infof("OSD will have its main bluestore block on %q", dataSource.ClaimName)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its main bluestore block on %q", dataSource.ClaimName)
}
walSource, walOK := volume.PVCSources[bluestorePVCWal]
if walOK {
logger.Infof("OSD will have its wal device on %q", walSource.ClaimName)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its wal device on %q", walSource.ClaimName)
}
osdProps := osdProperties{
@@ -196,13 +197,13 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig, errs *provi
// Allow updating OSD prepare pod if the OSD needs migration
if c.migrateOSD != nil {
if strings.Contains(c.migrateOSD.BlockPath, dataSource.ClaimName) {
logger.Infof("updating OSD prepare pod to replace OSD.%d", c.migrateOSD.ID)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "updating OSD prepare pod to replace OSD.%d", c.migrateOSD.ID)
skipPreparePod = false
}
}
if skipPreparePod {
logger.Infof("skipping OSD prepare job creation for PVC %q because OSD daemon using the PVC already exists", osdProps.crushHostname)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "skipping OSD prepare job creation for PVC %q because OSD daemon using the PVC already exists", osdProps.crushHostname)
continue
}
@@ -264,13 +265,13 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig, errs *provi
// usage of awaitingStatusConfigMaps in this file.
func (c *Cluster) startProvisioningOverNodes(config *provisionConfig, errs *provisionErrors) (sets.Set[string], error) {
if !c.spec.Storage.UseAllNodes && len(c.spec.Storage.Nodes) == 0 {
logger.Info("no nodes are defined for configuring OSDs on raw devices")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "no nodes are defined for configuring OSDs on raw devices")
return sets.New[string](), nil
}
if c.spec.Storage.UseAllNodes {
if len(c.spec.Storage.Nodes) > 0 {
logger.Warningf("useAllNodes is TRUE, but nodes are specified. NODES in the cluster CR will be IGNORED unless useAllNodes is FALSE.")
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "useAllNodes is TRUE, but nodes are specified. NODES in the cluster CR will be IGNORED unless useAllNodes is FALSE.")
}
// Get the list of all nodes in the cluster. The placement settings will be applied below.
@@ -286,19 +287,19 @@ func (c *Cluster) startProvisioningOverNodes(config *provisionConfig, errs *prov
}
c.spec.Storage.Nodes = append(c.spec.Storage.Nodes, storageNode)
}
logger.Debugf("storage nodes: %+v", c.spec.Storage.Nodes)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "storage nodes: %+v", c.spec.Storage.Nodes)
}
// generally speaking, this finds nodes which are capable of running new osds
validNodes := k8sutil.GetValidNodes(c.clusterInfo.Context, c.spec.Storage, c.context.Clientset, cephv1.GetOSDPlacement(c.spec.Placement))
logger.Infof("%d of the %d storage nodes are valid", len(validNodes), len(c.spec.Storage.Nodes))
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "%d of the %d storage nodes are valid", len(validNodes), len(c.spec.Storage.Nodes))
c.ValidStorage = *c.spec.Storage.DeepCopy()
c.ValidStorage.Nodes = validNodes
// no valid node is ready to run an osd
if len(validNodes) == 0 {
logger.Warningf("no valid nodes available to run osds on nodes in namespace %q", c.clusterInfo.Namespace)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "no valid nodes available to run osds on nodes in namespace %q", c.clusterInfo.Namespace)
return sets.New[string](), nil
}
@@ -316,12 +317,12 @@ func (c *Cluster) startProvisioningOverNodes(config *provisionConfig, errs *prov
// don't care about osd device class resources since it will be overwritten later for prepareosd resources
n := c.resolveNode(node.Name, "")
if n == nil {
logger.Warningf("node %q did not resolve", node.Name)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "node %q did not resolve", node.Name)
continue
}
if n.Name == "" {
logger.Warningf("skipping node with a blank name! %+v", n)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "skipping node with a blank name! %+v", n)
continue
}
@@ -372,7 +373,7 @@ func (c *Cluster) runPrepareJob(osdProps *osdProperties, config *provisionConfig
return errors.Wrapf(err, "failed to run osd provisioning job for %s %q", nodeOrPVC, nodeOrPVCName)
}
logger.Infof("started OSD provisioning job for %s %q", nodeOrPVC, nodeOrPVCName)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "started OSD provisioning job for %s %q", nodeOrPVC, nodeOrPVCName)
return nil
}
+12 -11
View File
@@ -29,6 +29,7 @@ import (
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -109,7 +110,7 @@ func (c *Cluster) prepareStorageClassDeviceSets(errs *provisionErrors) {
highestExistingID := -1
countInDeviceSet := 0
if existingIDs, ok := uniqueOSDsPerDeviceSet[deviceSet.Name]; ok {
logger.Infof("verifying PVCs exist for %d OSDs in device set %q", existingIDs.Len(), deviceSet.Name)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "verifying PVCs exist for %d OSDs in device set %q", existingIDs.Len(), deviceSet.Name)
for existingID := range existingIDs {
pvcID, err := strconv.Atoi(existingID)
if err != nil {
@@ -129,7 +130,7 @@ func (c *Cluster) prepareStorageClassDeviceSets(errs *provisionErrors) {
// No new PVCs will be created if we have too many
pvcsToCreate := deviceSet.Count - countInDeviceSet
if pvcsToCreate > 0 {
logger.Infof("creating %d new PVCs for device set %q", pvcsToCreate, deviceSet.Name)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "creating %d new PVCs for device set %q", pvcsToCreate, deviceSet.Name)
}
for i := 0; i < pvcsToCreate; i++ {
pvcID := highestExistingID + i + 1
@@ -234,12 +235,12 @@ func (c *Cluster) createDeviceSetPVC(existingPVCs map[string]*v1.PersistentVolum
}
if existingPVC != nil {
logger.Infof("OSD PVC %q already exists", existingPVC.Name)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD PVC %q already exists", existingPVC.Name)
desiredPvcSize, desiredOK := pvc.Spec.Resources.Requests[v1.ResourceStorage]
actualPvcSize, currentOK := existingPVC.Spec.Resources.Requests[v1.ResourceStorage]
if !desiredOK || !currentOK {
logger.Debugf("desired or current size are not specified for PVC %q", existingPVC.Name)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "desired or current size are not specified for PVC %q", existingPVC.Name)
return existingPVC, nil
}
@@ -260,7 +261,7 @@ func (c *Cluster) createDeviceSetPVC(existingPVCs map[string]*v1.PersistentVolum
if err != nil {
return nil, errors.Wrapf(err, "failed to create PVC %q for device set %q", pvc.Name, deviceSetName)
}
logger.Infof("successfully provisioned PVC %q", deployedPVC.Name)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "successfully provisioned PVC %q", deployedPVC.Name)
return deployedPVC, nil
}
@@ -345,13 +346,13 @@ func waitForPvcToExpandWithTimeout(ctx context.Context, client client.Client, pv
timeout := currentTime.Add(pvcResizeWaitTimeOut)
for {
if time.Now().UTC().After(timeout) {
logger.Warning("timed out waiting for PVC size to be updated, osds may need to be restarted manually after resize")
log.NamespacedWarning(namespace, logger, "timed out waiting for PVC size to be updated, osds may need to be restarted manually after resize")
return
}
allPvcResized := checkAllPvcResize(ctx, client, namespace, pvcResizeMap)
if allPvcResized {
logger.Info("all osd pvcs resized")
log.NamespacedInfo(namespace, logger, "all osd pvcs resized")
return
}
@@ -373,7 +374,7 @@ func checkAllPvcResize(ctx context.Context, client client.Client, namespace stri
currentPVC := &v1.PersistentVolumeClaim{}
err := client.Get(ctx, types.NamespacedName{Name: pvcName, Namespace: namespace}, currentPVC)
if err != nil {
logger.Errorf("failed to get PVC %q. %v", currentPVC.Name, err)
log.NamespacedError(namespace, logger, "failed to get PVC %q. %v", currentPVC.Name, err)
continue
}
@@ -385,7 +386,7 @@ func checkAllPvcResize(ctx context.Context, client client.Client, namespace stri
if currentPVC.Status.Capacity != nil {
if currentPVC.Status.Capacity[v1.ResourceStorage] == pvcSize {
logger.Infof("PVC %q size is updated to %s", currentPVC.Name, pvcSize.String())
log.NamespacedInfo(namespace, logger, "PVC %q size is updated to %s", currentPVC.Name, pvcSize.String())
numberOfPvcResized++
pvcResizeMap[pvcName] = pvcResize{
desiredSize: pvc.desiredSize,
@@ -393,11 +394,11 @@ func checkAllPvcResize(ctx context.Context, client client.Client, namespace stri
resizeConfirmed: true,
}
} else {
logger.Debugf("PVC %q size is not updated to %v", currentPVC.Name, pvcSize.String())
log.NamespacedDebug(namespace, logger, "PVC %q size is not updated to %v", currentPVC.Name, pvcSize.String())
}
}
}
logger.Infof("%d of osd pvcs resized and %d remain to be resized", numberOfPvcResized, len(pvcResizeMap)-numberOfPvcResized)
log.NamespacedInfo(namespace, logger, "%d of osd pvcs resized and %d remain to be resized", numberOfPvcResized, len(pvcResizeMap)-numberOfPvcResized)
return numberOfPvcResized == len(pvcResizeMap)
}
+12 -11
View File
@@ -27,6 +27,7 @@ import (
"github.com/rook/rook/pkg/daemon/ceph/client"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
kerrors "k8s.io/apimachinery/pkg/api/errors"
)
@@ -58,7 +59,7 @@ func NewOSDHealthMonitor(context *clusterd.Context, clusterInfo *client.ClusterI
// allow overriding the check interval
checkInterval := healthCheck.DaemonHealth.ObjectStorageDaemon.Interval
if checkInterval != nil {
logger.Infof("ceph osd status in namespace %q check interval %q", h.clusterInfo.Namespace, checkInterval.Duration.String())
log.NamespacedInfo(h.clusterInfo.Namespace, logger, "ceph osd status in namespace %q check interval %q", h.clusterInfo.Namespace, checkInterval.Duration.String())
h.interval = &checkInterval.Duration
}
@@ -72,17 +73,17 @@ func (m *OSDHealthMonitor) Start(monitoringRoutines *sync.Map, daemon string) {
// we will get an invalid pointer error and the go routine will panic
v, ok := monitoringRoutines.Load(daemon)
if !ok {
logger.Infof("ceph cluster %q has been deleted. stopping monitoring of OSDs", m.clusterInfo.Namespace)
log.NamespacedInfo(m.clusterInfo.Namespace, logger, "ceph cluster %q has been deleted. stopping monitoring of OSDs", m.clusterInfo.Namespace)
return
}
health := v.(*opcontroller.ClusterHealth)
select {
case <-time.After(*m.interval):
logger.Debug("checking osd processes status.")
log.NamespacedDebug(m.clusterInfo.Namespace, logger, "checking osd processes status.")
m.checkOSDHealth()
case <-health.InternalCtx.Done():
logger.Infof("stopping monitoring of OSDs in namespace %q", m.clusterInfo.Namespace)
log.NamespacedInfo(m.clusterInfo.Namespace, logger, "stopping monitoring of OSDs in namespace %q", m.clusterInfo.Namespace)
monitoringRoutines.Delete(daemon)
return
}
@@ -98,7 +99,7 @@ func (m *OSDHealthMonitor) Update(removeOSDsIfOUTAndSafeToRemove bool) {
func (m *OSDHealthMonitor) checkOSDHealth() {
err := m.checkOSDDump()
if err != nil {
logger.Debugf("failed to check OSD Dump. %v", err)
log.NamespacedDebug(m.clusterInfo.Namespace, logger, "failed to check OSD Dump. %v", err)
}
}
@@ -115,7 +116,7 @@ func (m *OSDHealthMonitor) checkOSDDump() error {
}
id := int(id64)
logger.Debugf("validating status of osd.%d", id)
log.NamespacedDebug(m.clusterInfo.Namespace, logger, "validating status of osd.%d", id)
status, in, err := osdDump.StatusByID(int64(id))
if err != nil {
@@ -123,17 +124,17 @@ func (m *OSDHealthMonitor) checkOSDDump() error {
}
if status == upStatus {
logger.Debugf("osd.%d is healthy.", id)
log.NamespacedDebug(m.clusterInfo.Namespace, logger, "osd.%d is healthy.", id)
continue
}
logger.Debugf("osd.%d is marked 'DOWN'", id)
log.NamespacedDebug(m.clusterInfo.Namespace, logger, "osd.%d is marked 'DOWN'", id)
if in != inStatus {
logger.Debugf("osd.%d is marked 'OUT'", id)
log.NamespacedDebug(m.clusterInfo.Namespace, logger, "osd.%d is marked 'OUT'", id)
if m.removeOSDsIfOUTAndSafeToRemove {
if err := m.removeOSDDeploymentIfSafeToDestroy(id); err != nil {
logger.Errorf("error handling marked out osd osd.%d. %v", id, err)
log.NamespacedError(m.clusterInfo.Namespace, logger, "error handling marked out osd osd.%d. %v", id, err)
}
}
}
@@ -162,7 +163,7 @@ func (m *OSDHealthMonitor) removeOSDDeploymentIfSafeToDestroy(outOSDid int) erro
podDeletionTimeStamp := podCreationTimestamp.Add(graceTime)
currentTime := time.Now().UTC()
if podDeletionTimeStamp.Before(currentTime) {
logger.Infof("osd.%d is 'safe-to-destroy'. removing the osd deployment.", outOSDid)
log.NamespacedInfo(m.clusterInfo.Namespace, logger, "osd.%d is 'safe-to-destroy'. removing the osd deployment.", outOSDid)
if err := k8sutil.DeleteDeployment(m.clusterInfo.Context, m.context.Clientset, dp.Items[0].Namespace, dp.Items[0].Name); err != nil {
return errors.Wrapf(err, "failed to delete osd deployment %s", dp.Items[0].Name)
}
@@ -25,6 +25,7 @@ import (
kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms"
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
batch "k8s.io/api/batch/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -247,7 +248,7 @@ func (c *Cluster) reconcileKeyRotationCronJob() error {
}
return nil
}
logger.Debugf("successfully deleted key rotation cron jobs")
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "successfully deleted key rotation cron jobs")
return nil
}
@@ -259,7 +260,7 @@ func (c *Cluster) reconcileKeyRotationCronJob() error {
return errors.Wrap(err, "failed to query existing OSD deployments")
}
logger.Debugf("found %d osd deployments", len(deployments.Items))
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "found %d osd deployments", len(deployments.Items))
for i := range deployments.Items {
osdDep := deployments.Items[i]
osd, err := c.getOSDInfo(&osdDep)
@@ -279,7 +280,7 @@ func (c *Cluster) reconcileKeyRotationCronJob() error {
continue
}
logger.Infof("starting OSD key rotation cron job for osd %d", osd.ID)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "starting OSD key rotation cron job for osd %d", osd.ID)
cj, err := c.makeKeyRotationCronJob(pvcName, osd, osdProps)
if err != nil {
return errors.Wrap(err, "failed to make key rotation cron job")
@@ -294,9 +295,9 @@ func (c *Cluster) reconcileKeyRotationCronJob() error {
if err != nil {
return errors.Wrapf(err, "failed to create or update key rotation cron job %q", cj.Name)
}
logger.Infof("started OSD key rotation cron job %q", cj.Name)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "started OSD key rotation cron job %q", cj.Name)
}
logger.Infof("successfully started OSD key rotation cron jobs")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "successfully started OSD key rotation cron jobs")
return nil
}
+6 -5
View File
@@ -26,6 +26,7 @@ import (
"github.com/rook/rook/pkg/clusterd"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
@@ -94,7 +95,7 @@ func (m *migrationConfig) migrateForEncryption(c *Cluster, osdDeployments *appsv
if err != nil {
return errors.Wrapf(err, "failed to details about the OSD %q", osdDeployments.Items[i].Name)
}
logger.Infof("migration is required for OSD.%d due to change in encryption settings from %t to %t in storageClassDeviceSet %q", osdInfo.ID, actualEncryptedSetting, requestedEncryptionSetting, osdDeviceSetName)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "migration is required for OSD.%d due to change in encryption settings from %t to %t in storageClassDeviceSet %q", osdInfo.ID, actualEncryptedSetting, requestedEncryptionSetting, osdDeviceSetName)
if _, exists := m.osds[osdInfo.ID]; !exists {
m.osds[osdInfo.ID] = &osdInfo
}
@@ -113,7 +114,7 @@ func (m *migrationConfig) migrateForOSDStore(c *Cluster, osdDeployments *appsv1.
if err != nil {
return errors.Wrapf(err, "failed to details about the OSD %q", osdDeployments.Items[i].Name)
}
logger.Infof("migration is required for OSD.%d to update storeType from %q to %q", osdInfo.ID, osdStore, desiredOSDStore)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "migration is required for OSD.%d to update storeType from %q to %q", osdInfo.ID, osdStore, desiredOSDStore)
if _, exists := m.osds[osdInfo.ID]; !exists {
m.osds[osdInfo.ID] = &osdInfo
}
@@ -185,12 +186,12 @@ func isLastOSDMigrationComplete(c *Cluster) (bool, error) {
_, err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Get(c.clusterInfo.Context, deploymentName, metav1.GetOptions{})
if err != nil {
if kerrors.IsNotFound(err) {
logger.Infof("deployment for the last migrated OSD with ID - %d is not found.", migratedOSDId)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "deployment for the last migrated OSD with ID - %d is not found.", migratedOSDId)
return false, nil
}
}
logger.Infof("migration of OSD.%d was successful", migratedOSDId)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "migration of OSD.%d was successful", migratedOSDId)
return true, nil
}
@@ -205,7 +206,7 @@ func getLastMigratedOSDId(context *clusterd.Context, clusterInfo *cephclient.Clu
osdID, ok := cm.Data[OSDIdKey]
if !ok || osdID == "" {
logger.Debugf("empty config map %q", OSDMigrationConfigName)
log.NamespacedDebug(clusterInfo.Namespace, logger, "empty config map %q", OSDMigrationConfigName)
return -1, nil
}
+50 -49
View File
@@ -38,6 +38,7 @@ import (
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util"
"github.com/rook/rook/pkg/util/log"
"github.com/coreos/pkg/capnslog"
"github.com/pkg/errors"
@@ -206,47 +207,47 @@ func (c *Cluster) validateOSDSettings() error {
}
func (c *Cluster) validateTopologyAcrossNodes() error {
logger.Info("Starting validateTopologyAcrossNodes()")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Starting validateTopologyAcrossNodes()")
if os.Getenv("ROOK_SKIP_OSD_TOPOLOGY_CHECK") == "true" {
logger.Debugf("Skipping topology validation due to ROOK_SKIP_OSD_TOPOLOGY_CHECK=true")
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "Skipping topology validation due to ROOK_SKIP_OSD_TOPOLOGY_CHECK=true")
return nil
}
if topologyValidated {
logger.Debug("Skipping topology validation because it was already validated")
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "Skipping topology validation because it was already validated")
return nil
}
logger.Info("Validating node topology across all cluster nodes")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Validating node topology across all cluster nodes")
nodelist, err := c.context.Clientset.CoreV1().Nodes().List(c.clusterInfo.Context, metav1.ListOptions{})
if err != nil {
logger.Errorf("Failed to list nodes for topology validation: %v", err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "Failed to list nodes for topology validation: %v", err)
return errors.Wrap(err, "failed to list nodes for topology validation")
}
logger.Infof("Fetched nodelist with %d nodes", len(nodelist.Items))
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Fetched nodelist with %d nodes", len(nodelist.Items))
if err := topology.CheckTopologyConflicts(&nodelist.Items); err != nil {
logger.Infof("Topology conflict detected: %v", err)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Topology conflict detected: %v", err)
// Check if there are any existing OSDs
osdRunning, errPods := k8sutil.PodsRunningWithLabel(c.clusterInfo.Context, c.context.Clientset, c.clusterInfo.Namespace, "app=rook-ceph-osd")
if errPods != nil {
logger.Errorf("Failed to list OSD pods: %v", errPods)
log.NamespacedError(c.clusterInfo.Namespace, logger, "Failed to list OSD pods: %v", errPods)
return errPods // fail safe
}
logger.Infof("Found %d existing OSDs", osdRunning)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Found %d existing OSDs", osdRunning)
if osdRunning == 0 {
logger.Errorf("Topology conflict detected in new cluster: %v", err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "Topology conflict detected in new cluster: %v", err)
return errors.Wrap(err, "topology conflict detected in new cluster")
}
// Existing OSDs found — log and continue
logger.Warningf("Topology conflict detected, but skipping failure since OSDs already exist. %v", err)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "Topology conflict detected, but skipping failure since OSDs already exist. %v", err)
return nil
}
logger.Info("Node topology validation passed without conflicts")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Node topology validation passed without conflicts")
topologyValidated = true
logger.Info("validateTopologyAcrossNodes() completed successfully")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "validateTopologyAcrossNodes() completed successfully")
return nil
}
@@ -265,10 +266,10 @@ func (c *Cluster) Start() error {
if err := c.initializeNodeConfigmaps(); err != nil {
return err
}
logger.Infof("start running osds in namespace %q", namespace)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "start running osds in namespace %q", namespace)
if !c.spec.Storage.UseAllNodes && len(c.spec.Storage.Nodes) == 0 && len(c.spec.Storage.StorageClassDeviceSets) == 0 {
logger.Warningf("useAllNodes is set to false and no nodes, storageClassDevicesets or volumeSources are specified, no OSD pods are going to be created")
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "useAllNodes is set to false and no nodes, storageClassDevicesets or volumeSources are specified, no OSD pods are going to be created")
}
if c.spec.WaitTimeoutForHealthyOSDInMinutes != 0 {
@@ -276,11 +277,11 @@ func (c *Cluster) Start() error {
} else {
c.clusterInfo.OsdUpgradeTimeout = defaultWaitTimeoutForHealthyOSD
}
logger.Infof("wait timeout for healthy OSDs during upgrade or restart is %q", c.clusterInfo.OsdUpgradeTimeout)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "wait timeout for healthy OSDs during upgrade or restart is %q", c.clusterInfo.OsdUpgradeTimeout)
osdsToSkipReconcile, err := controller.GetDaemonsToSkipReconcile(c.clusterInfo.Context, c.context, c.clusterInfo.Namespace, OsdIdLabelKey, AppName)
if err != nil {
logger.Warningf("failed to get osds to skip reconcile. %v", err)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to get osds to skip reconcile. %v", err)
}
migrationConfig, err := c.startOSDMigration()
@@ -301,20 +302,20 @@ func (c *Cluster) Start() error {
}
}
logger.Debugf("%d of %d OSD Deployments need update", updateQueue.Len(), deployments.Len())
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "%d of %d OSD Deployments need update", updateQueue.Len(), deployments.Len())
updateConfig := c.newUpdateConfig(config, updateQueue, deployments, osdsToSkipReconcile)
// prepare for creating new OSDs
statusConfigMaps := sets.New[string]()
logger.Info("start provisioning the OSDs on PVCs, if needed")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "start provisioning the OSDs on PVCs, if needed")
pvcConfigMaps, err := c.startProvisioningOverPVCs(config, errs)
if err != nil {
return err
}
statusConfigMaps = statusConfigMaps.Union(pvcConfigMaps)
logger.Info("start provisioning the OSDs on nodes, if needed")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "start provisioning the OSDs on nodes, if needed")
nodeConfigMaps, err := c.startProvisioningOverNodes(config, errs)
if err != nil {
return err
@@ -360,26 +361,26 @@ func (c *Cluster) Start() error {
// remove the osdBootstrapKey as the osds are running
c.deleteOsdBootstrapKeyring()
logger.Infof("finished running OSDs in namespace %q", namespace)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "finished running OSDs in namespace %q", namespace)
return nil
}
func (c *Cluster) deleteOsdBootstrapKeyring() {
err := cephclient.AuthDelete(c.context, c.clusterInfo, "client.bootstrap-osd")
if err != nil {
logger.Debugf("failed to delete client.bootstrap-osd bootstrap key in the namespace %q. Error: %v", c.clusterInfo.Namespace, err)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "failed to delete client.bootstrap-osd bootstrap key in the namespace %q. Error: %v", c.clusterInfo.Namespace, err)
} else {
logger.Debugf("successfully deleted client.bootstrap-osd bootstrap key in the namespace %q", c.clusterInfo.Namespace)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "successfully deleted client.bootstrap-osd bootstrap key in the namespace %q", c.clusterInfo.Namespace)
}
}
func (c *Cluster) startOSDMigration() (*migrationConfig, error) {
if !c.isMigrationRequested() {
logger.Debug("no OSD migration is requested")
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "no OSD migration is requested")
return nil, nil
}
logger.Info("osd migration is requested")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "osd migration is requested")
// start migration only if PGs are active+clean
pgsHealhty, err := c.waitForHealthyPGs()
@@ -409,7 +410,7 @@ func (c *Cluster) startOSDMigration() (*migrationConfig, error) {
// delete deployment of the osd that needs migration
if migrationConfig != nil && len(migrationConfig.osds) > 0 {
osdToMigrate := migrationConfig.getOSDToMigrate()
logger.Infof("deleting OSD.%d deployment for migration ", osdToMigrate.ID)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "deleting OSD.%d deployment for migration ", osdToMigrate.ID)
err = c.deleteOSDDeployment(osdToMigrate.ID)
if err != nil {
return nil, errors.Wrapf(err, "failed to delete deployment for osd.%d that needs migration %q", osdToMigrate.ID, c.clusterInfo.Namespace)
@@ -437,13 +438,13 @@ func (c *Cluster) postReconcileUpdateOSDProperties(desiredOSDs map[int]*OSDInfo)
if err != nil {
return errors.Wrap(err, "failed to get osd usage")
}
logger.Debugf("post processing osd properties with %d actual osds from ceph osd df and %d existing osds found during reconcile", len(osdUsage.OSDNodes), len(desiredOSDs))
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "post processing osd properties with %d actual osds from ceph osd df and %d existing osds found during reconcile", len(osdUsage.OSDNodes), len(desiredOSDs))
for _, actualOSD := range osdUsage.OSDNodes {
if c.spec.Storage.AllowOsdCrushWeightUpdate {
_, err := cephclient.ResizeOsdCrushWeight(actualOSD, c.context, c.clusterInfo)
if err != nil {
// Log the error and allow other updates to continue
logger.Errorf("failed to resize osd crush weight on cluster in namespace %s: %v", c.clusterInfo.Namespace, err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to resize osd crush weight on cluster in namespace %s: %v", c.clusterInfo.Namespace, err)
}
}
@@ -453,7 +454,7 @@ func (c *Cluster) postReconcileUpdateOSDProperties(desiredOSDs map[int]*OSDInfo)
}
if err := c.updateDeviceClassIfChanged(actualOSD.ID, desiredOSD.DeviceClass, actualOSD.DeviceClass); err != nil {
// Log the error and allow other updates to continue
logger.Errorf("failed to update device class on cluster in namespace %s: %v", c.clusterInfo.Namespace, err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to update device class on cluster in namespace %s: %v", c.clusterInfo.Namespace, err)
}
}
@@ -466,14 +467,14 @@ func (c *Cluster) updateDeviceClassIfChanged(osdID int, desiredDeviceClass, actu
return nil
}
if desiredDeviceClass != "" && desiredDeviceClass != actualDeviceClass {
logger.Infof("updating osd.%d device class from %q to %q", osdID, actualDeviceClass, desiredDeviceClass)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "updating osd.%d device class from %q to %q", osdID, actualDeviceClass, desiredDeviceClass)
err := cephclient.SetDeviceClass(c.context, c.clusterInfo, osdID, desiredDeviceClass)
if err != nil {
return errors.Wrapf(err, "failed to set device class on osd %d", osdID)
}
return nil
}
logger.Debugf("no device class change needed for osd.%d. desired=%q, actual=%q", osdID, desiredDeviceClass, actualDeviceClass)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "no device class change needed for osd.%d. desired=%q, actual=%q", osdID, desiredDeviceClass, actualDeviceClass)
return nil
}
@@ -584,21 +585,21 @@ func (c *Cluster) getOSDPropsForPVC(pvcName string) (osdProperties, error) {
// The data PVC template is required.
dataSource, dataOK := deviceSet.PVCSources[bluestorePVCData]
if !dataOK {
logger.Warningf("failed to find data source daemon for device set %q, missing the data template", deviceSet.Name)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to find data source daemon for device set %q, missing the data template", deviceSet.Name)
continue
}
if pvcName == dataSource.ClaimName {
metadataSource, metadataOK := deviceSet.PVCSources[bluestorePVCMetadata]
if metadataOK {
logger.Infof("OSD will have its main bluestore block on %q and its metadata device on %q", dataSource.ClaimName, metadataSource.ClaimName)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its main bluestore block on %q and its metadata device on %q", dataSource.ClaimName, metadataSource.ClaimName)
} else {
logger.Infof("OSD will have its main bluestore block on %q", dataSource.ClaimName)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its main bluestore block on %q", dataSource.ClaimName)
}
walSource, walOK := deviceSet.PVCSources[bluestorePVCWal]
if walOK {
logger.Infof("OSD will have its wal device on %q", walSource.ClaimName)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its wal device on %q", walSource.ClaimName)
}
if deviceSet.Resources.Limits == nil && deviceSet.Resources.Requests == nil {
@@ -670,7 +671,7 @@ func (c *Cluster) getPVCHostName(pvcName string) (string, error) {
for _, pod := range pods.Items {
name, err := k8sutil.GetNodeHostName(c.clusterInfo.Context, c.context.Clientset, pod.Spec.NodeName)
if err != nil {
logger.Warningf("falling back to node name %s since hostname not found for node", pod.Spec.NodeName)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "falling back to node name %s since hostname not found for node", pod.Spec.NodeName)
name = pod.Spec.NodeName
}
if name == "" {
@@ -754,7 +755,7 @@ func (c *Cluster) getOSDInfo(d *appsv1.Deployment) (OSDInfo, error) {
// If CVMode is empty, this likely means we upgraded Rook
// This property did not exist before so we need to initialize it
if osd.CVMode == "" {
logger.Infof("required CVMode for OSD %d was not found. assuming this is an LVM OSD", osd.ID)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "required CVMode for OSD %d was not found. assuming this is an LVM OSD", osd.ID)
osd.CVMode = "lvm"
}
@@ -762,7 +763,7 @@ func (c *Cluster) getOSDInfo(d *appsv1.Deployment) (OSDInfo, error) {
if isPVC && osd.TopologyAffinity == "" {
osd.TopologyAffinity, err = getTopologyFromNode(c.clusterInfo.Context, c.context.Clientset, d, osd)
if err != nil {
logger.Errorf("failed to get topology affinity for osd %d. %v", osd.ID, err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get topology affinity for osd %d. %v", osd.ID, err)
}
}
@@ -772,7 +773,7 @@ func (c *Cluster) getOSDInfo(d *appsv1.Deployment) (OSDInfo, error) {
if !locationFound {
location, _, err := getLocationFromPod(c.clusterInfo.Context, c.context.Clientset, d, cephclient.GetCrushRootFromSpec(&c.spec))
if err != nil {
logger.Errorf("failed to get location. %v", err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get location. %v", err)
} else {
osd.Location = location
}
@@ -884,7 +885,7 @@ func getTopologyFromNode(ctx context.Context, clientset kubernetes.Interface, d
// osd is not portable, no need to load the topology affinity
return "", nil
}
logger.Infof("detecting topology affinity for osd %d after upgrade", osd.ID)
log.NamespacedInfo(d.Namespace, logger, "detecting topology affinity for osd %d after upgrade", osd.ID)
// Get the osd pod and its assigned node, then look up the node labels
pods, err := clientset.CoreV1().Pods(d.Namespace).List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", OsdIdLabelKey, d.Labels[OsdIdLabelKey])})
@@ -903,7 +904,7 @@ func getTopologyFromNode(ctx context.Context, clientset kubernetes.Interface, d
return "", errors.Wrap(err, "failed to get the node for topology affinity")
}
_, topologyAffinity := topology.ExtractOSDTopologyFromLabels(node.Labels)
logger.Infof("found osd %d topology affinity at %q", osd.ID, topologyAffinity)
log.NamespacedInfo(d.Namespace, logger, "found osd %d topology affinity at %q", osd.ID, topologyAffinity)
return topologyAffinity, nil
}
@@ -984,7 +985,7 @@ func (c *Cluster) applyUpgradeOSDFunctionality() {
// Get all the daemons versions
versions, err := cephclient.GetAllCephDaemonVersions(c.context, c.clusterInfo)
if err != nil {
logger.Warningf("failed to get ceph daemons versions; this likely means there are no osds yet. %v", err)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to get ceph daemons versions; this likely means there are no osds yet. %v", err)
return
}
@@ -997,7 +998,7 @@ func (c *Cluster) applyUpgradeOSDFunctionality() {
for v := range versions.Osd {
osdVersion, err = cephver.ExtractCephVersion(v)
if err != nil {
logger.Warningf("failed to extract ceph version. %v", err)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to extract ceph version. %v", err)
return
}
// Ensure the required version of OSDs is set to the current consistent version,
@@ -1005,7 +1006,7 @@ func (c *Cluster) applyUpgradeOSDFunctionality() {
// previous major ceph version.
err = cephclient.EnableReleaseOSDFunctionality(c.context, c.clusterInfo, osdVersion.ReleaseName())
if err != nil {
logger.Warningf("failed to enable new osd functionality. %v", err)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to enable new osd functionality. %v", err)
return
}
}
@@ -1016,12 +1017,12 @@ func (c *Cluster) applyUpgradeOSDFunctionality() {
func (c *Cluster) deleteOSDDeployment(osdID int) error {
// Delete the OSD deployment
deploymentName := fmt.Sprintf("rook-ceph-osd-%d", osdID)
logger.Infof("removing the OSD deployment %q", deploymentName)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "removing the OSD deployment %q", deploymentName)
if err := k8sutil.DeleteDeployment(c.clusterInfo.Context, c.context.Clientset, c.clusterInfo.Namespace, deploymentName); err != nil {
if !kerrors.IsNotFound(err) {
return errors.Wrapf(err, "failed to delete OSD deployment %q.", deploymentName)
}
logger.Debugf("osd deployment %q not found. Ignoring since object must be deleted.", deploymentName)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "osd deployment %q not found. Ignoring since object must be deleted.", deploymentName)
}
return nil
}
@@ -1035,7 +1036,7 @@ func (c *Cluster) waitForHealthyPGs() (bool, error) {
if pgClean {
return true, nil
}
logger.Infof("waiting for PGs to be healthy. PG status: %q", pgHealthMsg)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "waiting for PGs to be healthy. PG status: %q", pgHealthMsg)
return false, nil
}
@@ -1083,7 +1084,7 @@ func (c *Cluster) updateCephOsdStorageStatus() error {
err := c.context.Client.Get(c.clusterInfo.Context, c.clusterInfo.NamespacedName(), &cephCluster)
if err != nil {
if kerrors.IsNotFound(err) {
logger.Debug("CephCluster resource not found. Ignoring since object must be deleted.")
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "CephCluster resource not found. Ignoring since object must be deleted.")
return nil
}
return errors.Wrapf(err, "failed to retrieve ceph cluster %q to update ceph Storage", c.clusterInfo.NamespacedName().Name)
@@ -1177,7 +1178,7 @@ func (c *Cluster) initializeNodeConfigmaps() error {
for _, configmap := range configmaps.Items {
if strings.HasPrefix(configmap.Name, prefix) {
nodeName := strings.TrimPrefix(configmap.Name, prefix)
logger.Infof("found node %q configmap, will mount it for alternate ceph conf overrides for osds on this node", nodeName)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "found node %q configmap, will mount it for alternate ceph conf overrides for osds on this node", nodeName)
c.nodeConfigmaps[nodeName] = struct{}{}
}
}
@@ -31,6 +31,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
batch "k8s.io/api/batch/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -365,13 +366,13 @@ func (c *Cluster) deleteAllOrphanedPrepareJobs() {
}
jobs, err := c.context.Clientset.BatchV1().Jobs(c.clusterInfo.Namespace).List(c.clusterInfo.Context, listOpts)
if err != nil {
logger.Warningf("failed to clean up any orphaned OSD prepare jobs. failed to list OSD prepare jobs. %v", err)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to clean up any orphaned OSD prepare jobs. failed to list OSD prepare jobs. %v", err)
return
}
nodes, err := c.context.Clientset.CoreV1().Nodes().List(c.clusterInfo.Context, metav1.ListOptions{})
if err != nil {
logger.Warningf("failed to clean up any orphaned OSD prepare jobs. failed to list nodes. %v", err)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to clean up any orphaned OSD prepare jobs. failed to list nodes. %v", err)
return
}
@@ -387,9 +388,9 @@ func (c *Cluster) deleteAllOrphanedPrepareJobs() {
nodeSelector := job.Spec.Template.Spec.NodeSelector
if jobHostName, ok := nodeSelector[k8sutil.LabelHostname()]; ok {
if !hostNames.Has(jobHostName) {
logger.Infof("cleaning up orphaned OSD prepare job %q.", job.Name)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "cleaning up orphaned OSD prepare job %q.", job.Name)
if err := c.context.Clientset.BatchV1().Jobs(c.clusterInfo.Namespace).Delete(c.clusterInfo.Context, job.Name, metav1.DeleteOptions{}); err != nil {
logger.Warningf("failed to clean up OSD prepare job %q. %v", job.Name, err)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to clean up OSD prepare job %q. %v", job.Name, err)
}
}
}
+11 -10
View File
@@ -31,6 +31,7 @@ import (
cephkey "github.com/rook/rook/pkg/operator/ceph/config/keyring"
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -312,14 +313,14 @@ func deploymentName(osdID int) string {
func (c *Cluster) updateCephConfigVolume(volumes []v1.Volume, nodeName string) []v1.Volume {
if _, ok := c.nodeConfigmaps[nodeName]; !ok {
logger.Debugf("no configmap to override for node %q", nodeName)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "no configmap to override for node %q", nodeName)
return volumes
}
updatedVolumes := []v1.Volume{}
for _, volume := range volumes {
if volume.Name == k8sutil.ConfigOverrideName {
logger.Debugf("found configmap volume to override for node %q", nodeName)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "found configmap volume to override for node %q", nodeName)
name := fmt.Sprintf("%s-%s", k8sutil.ConfigOverrideName, nodeName)
volume.VolumeSource.Projected.Sources[0].ConfigMap.LocalObjectReference.Name = name
}
@@ -351,12 +352,12 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provision
}
if c.spec.Storage.AllowDeviceClassUpdate && osdProps.storeConfig.DeviceClass != "" && osdProps.storeConfig.DeviceClass != osd.DeviceClass {
logger.Infof("The device class for osd %d is changing from %q to %q", osd.ID, osd.DeviceClass, osdProps.storeConfig.DeviceClass)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "The device class for osd %d is changing from %q to %q", osd.ID, osd.DeviceClass, osdProps.storeConfig.DeviceClass)
osd.DeviceClass = osdProps.storeConfig.DeviceClass
}
// Assign the resources specific to this device class
if resources, ok := cephv1.GetOSDResourcesForDeviceClass(c.spec.Resources, osd.DeviceClass); ok {
logger.Debugf("assigning resources for device class %q to osd %d: %+v", osd.DeviceClass, osd.ID, resources)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "assigning resources for device class %q to osd %d: %+v", osd.DeviceClass, osd.ID, resources)
osdProps.resources = resources
}
@@ -560,7 +561,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provision
return nil, errors.Wrapf(err, "failed to export service %q", osdService.Name)
}
logger.Infof("osd.%d exported IP is %q", osd.ID, exportedIP)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "osd.%d exported IP is %q", osd.ID, exportedIP)
args = append(args, []string{
fmt.Sprintf("--public-addr=%s", exportedIP),
@@ -793,7 +794,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provision
}
if osdProps.portable {
// portable OSDs must have affinity to the topology where the osd prepare job was executed
if err := applyTopologyAffinity(&deployment.Spec.Template.Spec, *osd); err != nil {
if err := c.applyTopologyAffinity(&deployment.Spec.Template.Spec, *osd); err != nil {
return nil, err
}
} else {
@@ -848,7 +849,7 @@ func (c *Cluster) createOSDService(osd OSDInfo, labels map[string]string) (*v1.S
return nil, err
}
logger.Infof("osd.%d cluster IP is %q", osd.ID, svc.Spec.ClusterIP)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "osd.%d cluster IP is %q", osd.ID, svc.Spec.ClusterIP)
return svc, nil
}
@@ -869,12 +870,12 @@ func (c *Cluster) applyAllPlacementIfNeeded(d *v1.PodSpec) {
}
}
func applyTopologyAffinity(spec *v1.PodSpec, osd OSDInfo) error {
func (c *Cluster) applyTopologyAffinity(spec *v1.PodSpec, osd OSDInfo) error {
if osd.TopologyAffinity == "" {
logger.Debugf("no topology affinity to set for osd %d", osd.ID)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "no topology affinity to set for osd %d", osd.ID)
return nil
}
logger.Infof("assigning osd %d topology affinity to %q", osd.ID, osd.TopologyAffinity)
log.NamespacedInfo(osd.Cluster, logger, "assigning osd %d topology affinity to %q", osd.ID, osd.TopologyAffinity)
nodeAffinity, err := k8sutil.GenerateNodeAffinity(osd.TopologyAffinity)
if err != nil {
return errors.Wrapf(err, "failed to generate osd %d topology affinity", osd.ID)
+14 -13
View File
@@ -26,6 +26,7 @@ import (
"github.com/pkg/errors"
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -176,7 +177,7 @@ func (c *Cluster) updateAndCreateOSDs(
if !doLoop {
return err
}
logger.Errorf("%v", err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "%v", err)
}
}
@@ -248,20 +249,20 @@ func (c *Cluster) updateAndCreateOSDsLoop(
select {
case event, ok := <-watcher.ResultChan():
if !ok {
logger.Infof("restarting watcher for OSD provisioning status ConfigMaps. the watcher closed the channel")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "restarting watcher for OSD provisioning status ConfigMaps. the watcher closed the channel")
return true, nil
}
if !isAddOrModifyEvent(event.Type) {
// We don't want to process delete events when we delete configmaps after having
// processed them. We also don't want to process BOOKMARK or ERROR events.
logger.Debugf("not processing %s event for object %q", event.Type, eventObjectName(event))
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "not processing %s event for object %q", event.Type, eventObjectName(event))
break // case
}
configMap, ok := event.Object.(*corev1.ConfigMap)
if !ok {
logger.Errorf("recovering. %s. expected type ConfigMap but found %T", watchErrMsg, configMap)
log.NamespacedError(c.clusterInfo.Namespace, logger, "recovering. %s. expected type ConfigMap but found %T", watchErrMsg, configMap)
break // case
}
@@ -275,11 +276,11 @@ func (c *Cluster) updateAndCreateOSDsLoop(
// Log progress
c, cExp := createConfig.progress()
u, uExp := updateConfig.progress()
logger.Infof("waiting... %d of %d OSD prepare jobs have finished processing and %d of %d OSDs have been updated", c, cExp, u, uExp)
log.NamespacedInfo(namespace, logger, "waiting... %d of %d OSD prepare jobs have finished processing and %d of %d OSDs have been updated", c, cExp, u, uExp)
// keep track of each time progress is made
if c != prevCreatedCount || u != prevUpdatedCount {
logger.Debugf("cluster in namespace %q, progress was made processing OSDs: %d nodes/OSDs were processed for creation and %d updated", namespace, c-prevCreatedCount, u-prevUpdatedCount)
log.NamespacedDebug(namespace, logger, "cluster in namespace %q, progress was made processing OSDs: %d nodes/OSDs were processed for creation and %d updated", namespace, c-prevCreatedCount, u-prevUpdatedCount)
prevCreatedCount = c
prevUpdatedCount = u
prevChangeTime = time.Now().UTC()
@@ -292,7 +293,7 @@ func (c *Cluster) updateAndCreateOSDsLoop(
}
case <-c.clusterInfo.Context.Done():
logger.Infof("context cancelled, exiting OSD update and create loop")
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "context cancelled, exiting OSD update and create loop")
return false, c.clusterInfo.Context.Err()
}
}
@@ -327,7 +328,7 @@ func (c *Cluster) createOSDsForStatusMap(
) {
nodeOrPVCName, ok := configMap.Labels[nodeLabelKey]
if !ok {
logger.Warningf("missing node label on configmap %s", configMap.Name)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "missing node label on configmap %s", configMap.Name)
return
}
@@ -340,7 +341,7 @@ func (c *Cluster) createOSDsForStatusMap(
nodeOrPVC = "PVC"
}
logger.Infof("OSD orchestration status for %s %s is %q", nodeOrPVC, nodeOrPVCName, status.Status)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD orchestration status for %s %s is %q", nodeOrPVC, nodeOrPVCName, status.Status)
if status.Status == OrchestrationStatusCompleted {
createConfig.createNewOSDsFromStatus(status, nodeOrPVCName, errs)
@@ -362,7 +363,7 @@ func statusConfigMapName(nodeOrPVCName string) string {
func (c *Cluster) deleteStatusConfigMap(nodeOrPVCName string) {
if err := c.kv.ClearStore(c.clusterInfo.Context, statusConfigMapName(nodeOrPVCName)); err != nil {
logger.Errorf("failed to remove the status configmap %q. %v", statusConfigMapName(nodeOrPVCName), err)
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to remove the status configmap %q. %v", statusConfigMapName(nodeOrPVCName), err)
}
}
@@ -373,14 +374,14 @@ func (c *Cluster) deleteAllStatusConfigMaps() {
cmClientset := c.context.Clientset.CoreV1().ConfigMaps(c.clusterInfo.Namespace)
cms, err := cmClientset.List(c.clusterInfo.Context, listOpts)
if err != nil {
logger.Warningf("failed to clean up any dangling OSD prepare status configmaps. failed to list OSD prepare status configmaps. %v", err)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to clean up any dangling OSD prepare status configmaps. failed to list OSD prepare status configmaps. %v", err)
return
}
for _, cm := range cms.Items {
logger.Debugf("cleaning up dangling OSD prepare status configmap %q", cm.Name)
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "cleaning up dangling OSD prepare status configmap %q", cm.Name)
err := cmClientset.Delete(c.clusterInfo.Context, cm.Name, metav1.DeleteOptions{})
if err != nil {
logger.Warningf("failed to clean up dangling OSD prepare status configmap %q. %v", cm.Name, err)
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to clean up dangling OSD prepare status configmap %q. %v", cm.Name, err)
}
}
}
+17 -16
View File
@@ -26,6 +26,7 @@ import (
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -85,14 +86,14 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
if !c.cluster.spec.SkipUpgradeChecks && c.cluster.spec.UpgradeOSDRequiresHealthyPGs {
pgHealthMsg, pgClean, err := cephclient.IsClusterClean(c.cluster.context, c.cluster.clusterInfo, c.cluster.spec.DisruptionManagement.PGHealthyRegex)
if err != nil {
logger.Warningf("failed to check PGs status to update OSDs, will try updating it again later. %v", err)
log.NamespacedWarning(c.cluster.clusterInfo.Namespace, logger, "failed to check PGs status to update OSDs, will try updating it again later. %v", err)
return
}
if !pgClean {
logger.Infof("PGs are not healthy to update OSDs, will try updating it again later. PGs status: %q", pgHealthMsg)
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "PGs are not healthy to update OSDs, will try updating it again later. PGs status: %q", pgHealthMsg)
return
}
logger.Infof("PGs are healthy to proceed updating OSDs. %v", pgHealthMsg)
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "PGs are healthy to proceed updating OSDs. %v", pgHealthMsg)
}
osdIDQuery, _ := c.queue.Pop()
@@ -102,7 +103,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
if c.cluster.spec.SkipUpgradeChecks || !shouldCheckOkToStopFunc(c.cluster.context, c.cluster.clusterInfo) {
// If we should not check ok-to-stop, then only process one OSD at a time. There are likely
// less than 3 OSDs in the cluster or the cluster is on a single node. E.g., in CI :wink:.
logger.Infof("skipping osd checks for ok-to-stop")
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "skipping osd checks for ok-to-stop")
osdIDs = []int{osdIDQuery}
} else {
osdMaxUpdatesInParallel := c.cluster.spec.Storage.OSDMaxUpdatesInParallel
@@ -112,29 +113,29 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
osdIDs, err = cephclient.OSDOkToStop(c.cluster.context, c.cluster.clusterInfo, osdIDQuery, osdMaxUpdatesInParallel)
if err != nil {
if c.cluster.spec.ContinueUpgradeAfterChecksEvenIfNotHealthy {
logger.Infof("OSD %d is not ok-to-stop but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so continuing to update it", osdIDQuery)
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "OSD %d is not ok-to-stop but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so continuing to update it", osdIDQuery)
osdIDs = []int{osdIDQuery} // make sure to update the queried OSD
} else {
logger.Infof("OSD %d is not ok-to-stop. will try updating it again later", osdIDQuery)
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "OSD %d is not ok-to-stop. will try updating it again later", osdIDQuery)
c.queue.Push(osdIDQuery) // push back onto queue to make sure we retry it later
return
}
}
}
logger.Debugf("updating OSDs: %v", osdIDs)
log.NamespacedDebug(c.cluster.clusterInfo.Namespace, logger, "updating OSDs: %v", osdIDs)
updatedDeployments := make([]*appsv1.Deployment, 0, len(osdIDs))
listIDs := []string{} // use this to build the k8s api selector query
for _, osdID := range osdIDs {
if !c.deployments.Exists(osdID) {
logger.Debugf("not updating deployment for OSD %d that is newly created", osdID)
log.NamespacedDebug(c.cluster.clusterInfo.Namespace, logger, "not updating deployment for OSD %d that is newly created", osdID)
continue
}
// osdIDQuery which has been popped off the queue but it does need to be updated
if osdID != osdIDQuery && !c.queue.Exists(osdID) {
logger.Debugf("not updating deployment for OSD %d that is not in the update queue. the OSD has already been updated", osdID)
log.NamespacedDebug(c.cluster.clusterInfo.Namespace, logger, "not updating deployment for OSD %d that is not in the update queue. the OSD has already been updated", osdID)
continue
}
@@ -152,7 +153,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
c.osdDesiredState[osdID] = &osdInfo
if c.osdsToSkipReconcile.Has(strconv.Itoa(osdID)) {
logger.Warningf("Skipping update for OSD %d since labeled with %s", osdID, cephv1.SkipReconcileLabelKey)
log.NamespacedWarning(c.cluster.clusterInfo.Namespace, logger, "Skipping update for OSD %d since labeled with %s", osdID, cephv1.SkipReconcileLabelKey)
continue
}
@@ -162,7 +163,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
if osdInfo.DeviceClass == "" || osdInfo.DeviceClass == "None" {
deviceClassInfo, err := cephclient.OSDDeviceClasses(c.cluster.context, c.cluster.clusterInfo, []string{strconv.Itoa(osdID)})
if err != nil {
logger.Errorf("failed to get device class for existing deployment %q. %v", depName, err)
log.NamespacedError(c.cluster.clusterInfo.Namespace, logger, "failed to get device class for existing deployment %q. %v", depName, err)
} else {
osdInfo.DeviceClass = deviceClassInfo[0].DeviceClass
}
@@ -188,14 +189,14 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
}
if osdIsOnPVC(dep) {
logger.Infof("updating OSD %d on PVC %q", osdID, nodeOrPVCName)
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "updating OSD %d on PVC %q", osdID, nodeOrPVCName)
updatedDep, err = deploymentOnPVCFunc(c.cluster, &osdInfo, nodeOrPVCName, c.provisionConfig)
message := fmt.Sprintf("Processing OSD %d on PVC %q", osdID, nodeOrPVCName)
updateConditionFunc(c.cluster.clusterInfo.Context, c.cluster.context, c.cluster.clusterInfo.NamespacedName(), k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, message)
} else {
if !c.cluster.ValidStorage.NodeExists(nodeOrPVCName) {
// node will not reconcile, so don't update the deployment
logger.Warningf(
log.NamespacedWarning(c.cluster.clusterInfo.Namespace, logger,
"not updating OSD %d on node %q. node no longer exists in the storage spec. "+
"if the user wishes to remove OSDs from the node, they must do so manually. "+
"Rook will not remove OSDs from nodes that are removed from the storage spec in order to prevent accidental data loss",
@@ -203,7 +204,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
continue
}
logger.Infof("updating OSD %d on node %q", osdID, nodeOrPVCName)
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "updating OSD %d on node %q", osdID, nodeOrPVCName)
updatedDep, err = deploymentOnNodeFunc(c.cluster, &osdInfo, nodeOrPVCName, c.provisionConfig)
message := fmt.Sprintf("Processing OSD %d on node %q", osdID, nodeOrPVCName)
updateConditionFunc(c.cluster.clusterInfo.Context, c.cluster.context, c.cluster.clusterInfo.NamespacedName(), k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, message)
@@ -424,7 +425,7 @@ func (c *Cluster) rotateCephxKey(osdInfo OSDInfo) (cephv1.CephxStatus, error) {
return didNotRotateCephxStatus, nil
}
logger.Infof("rotating cephx key of OSD %d for CephCluster in namespace %q", osdInfo.ID, c.clusterInfo.Namespace)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "rotating cephx key of OSD %d for CephCluster in namespace %q", osdInfo.ID, c.clusterInfo.Namespace)
user := fmt.Sprintf("osd.%d", osdInfo.ID)
// Note: OSD key is not stored in k8s secret; rotated key is picked up by OSD init container
_, err = cephclient.AuthRotate(c.context, c.clusterInfo, user)
@@ -435,7 +436,7 @@ func (c *Cluster) rotateCephxKey(osdInfo OSDInfo) (cephv1.CephxStatus, error) {
// rotating the `client.osd-lockbox.$OSD_UUID` keys created for luks-encrypted OSDs by ceph-volume
if osdInfo.Encrypted {
osdLockBoxUser := fmt.Sprintf("client.osd-lockbox.%s", osdInfo.UUID)
logger.Infof("rotating osd-lockbox cephx key of encrypted OSD %d for CephCluster in namespace %q", osdInfo.ID, c.clusterInfo.Namespace)
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "rotating osd-lockbox cephx key of encrypted OSD %d for CephCluster in namespace %q", osdInfo.ID, c.clusterInfo.Namespace)
_, err = cephclient.AuthRotate(c.context, c.clusterInfo, osdLockBoxUser)
if err != nil {
return didNotRotateCephxStatus, errors.Wrapf(err, "failed to rotate osd-lockbox cephx key for the encrypted OSD %d", osdInfo.ID)
+2 -1
View File
@@ -23,6 +23,7 @@ import (
kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms"
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/util/log"
v1 "k8s.io/api/core/v1"
)
@@ -169,7 +170,7 @@ func getPVCOSDVolumes(osdProps *osdProperties, configDir string, namespace strin
volumes = append(volumes, walPVCVolume...)
}
logger.Debugf("volumes are %+v", volumes)
log.NamespacedDebug(namespace, logger, "volumes are %+v", volumes)
return volumes
}
+8 -7
View File
@@ -27,6 +27,7 @@ import (
discoverDaemon "github.com/rook/rook/pkg/daemon/discover"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -139,7 +140,7 @@ func predicateForOperatorConfigCMWatcher[T *corev1.ConfigMap]() predicate.TypedF
}
if objOld.Data["ROOK_USE_CSI_OPERATOR"] != objNew.Data["ROOK_USE_CSI_OPERATOR"] {
logger.Infof("ROOK_USE_CSI_OPERATOR changed from %q to %q", objOld.Data["ROOK_USE_CSI_OPERATOR"], objNew.Data["ROOK_USE_CSI_OPERATOR"])
log.NamespacedInfo(objNew.Namespace, logger, "ROOK_USE_CSI_OPERATOR changed from %q to %q", objOld.Data["ROOK_USE_CSI_OPERATOR"], objNew.Data["ROOK_USE_CSI_OPERATOR"])
return true
}
@@ -196,18 +197,18 @@ func watchControllerPredicate[T *cephv1.CephCluster](ctx context.Context, c clie
objOld := (*cephv1.CephCluster)(e.ObjectOld)
objNew := (*cephv1.CephCluster)(e.ObjectNew)
logger.Debug("update event on CephCluster CR")
log.NamespacedDebug(objNew.Namespace, logger, "update event on CephCluster CR")
// If the labels "do_not_reconcile" is set on the object, let's not reconcile that request
if opcontroller.IsDoNotReconcile(objNew.GetLabels()) {
logger.Debugf("object %q matched on update but %q label is set, doing nothing", opcontroller.DoNotReconcileLabelName, objNew.Name)
log.NamespacedDebug(objNew.Namespace, logger, "object %q matched on update but %q label is set, doing nothing", opcontroller.DoNotReconcileLabelName, objNew.Name)
return false
}
diff := cmp.Diff(objOld.Spec, objNew.Spec, resourceQtyComparer)
if diff != "" {
logger.Infof("CR has changed for %q. diff=%s", objNew.Name, diff)
log.NamespacedInfo(objNew.Namespace, logger, "CR has changed for %q. diff=%s", objNew.Name, diff)
if objNew.Spec.CleanupPolicy.HasDataDirCleanPolicy() {
logger.Infof("skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set. not reloading the manager", objNew.GetName(), objNew.GetNamespace())
log.NamespacedInfo(objNew.Namespace, logger, "skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set. not reloading the manager", objNew.GetName(), objNew.GetNamespace())
return false
}
@@ -217,7 +218,7 @@ func watchControllerPredicate[T *cephv1.CephCluster](ctx context.Context, c clie
return false
} else if !objOld.GetDeletionTimestamp().Equal(objNew.GetDeletionTimestamp()) {
logger.Infof("CR %q is going be deleted, cancelling any ongoing orchestration", objNew.Name)
log.NamespacedInfo(objNew.Namespace, logger, "CR %q is going be deleted, cancelling any ongoing orchestration", objNew.Name)
// Stop any ongoing orchestration
opcontroller.ReloadManager()
@@ -225,7 +226,7 @@ func watchControllerPredicate[T *cephv1.CephCluster](ctx context.Context, c clie
return false
} else if objOld.GetGeneration() != objNew.GetGeneration() {
logger.Debugf("reconciling CephCluster %q with changed generation", objNew.Name)
log.NamespacedDebug(objNew.Namespace, logger, "reconciling CephCluster %q with changed generation", objNew.Name)
return true
}
+4 -3
View File
@@ -26,6 +26,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -59,7 +60,7 @@ func (r *ReconcileCephRBDMirror) generateKeyring(clusterInfo *client.ClusterInfo
}
if r.shouldRotateCephxKeys {
logger.Infof("rotating cephx key for CephRBDMirror %q in the namespace %q", daemonConfig.ResourceName, clusterInfo.Namespace)
log.NamespacedInfo(clusterInfo.Namespace, logger, "rotating cephx key for CephRBDMirror %q in the namespace %q", daemonConfig.ResourceName, clusterInfo.Namespace)
newKey, err := s.RotateKey(user)
if err != nil {
return "", errors.Wrapf(err, "failed to rotate cephx key for CephRDBMirror %q in the namespace %q", daemonConfig.ResourceName, clusterInfo.Namespace)
@@ -79,9 +80,9 @@ func (r *ReconcileCephRBDMirror) reconcileAddBootstrapPeer(cephRBDMirror *cephv1
// List all the peers secret, we can have more than one peer we might want to configure
// For each, get the Kubernetes Secret and import the "peer token" so that we can configure the mirroring
logger.Warning("(DEPRECATED) use of peer secret names in CephRBDMirror is deprecated. Please use CephBlockPool CR to configure peer secret names and import peers.")
log.NamespacedWarning(cephRBDMirror.Namespace, logger, "(DEPRECATED) use of peer secret names in CephRBDMirror is deprecated. Please use CephBlockPool CR to configure peer secret names and import peers.")
for _, peerSecret := range cephRBDMirror.Spec.Peers.SecretNames {
logger.Debugf("fetching bootstrap peer kubernetes secret %q", peerSecret)
log.NamespacedDebug(cephRBDMirror.Namespace, logger, "fetching bootstrap peer kubernetes secret %q", peerSecret)
s, err := r.context.Clientset.CoreV1().Secrets(r.clusterInfo.Namespace).Get(r.opManagerContext, peerSecret, metav1.GetOptions{})
// We don't care about IsNotFound here, we still need to fail
if err != nil {
+11 -10
View File
@@ -31,6 +31,7 @@ import (
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/ceph/reporting"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
@@ -168,7 +169,7 @@ func (r *ReconcileCephRBDMirror) Reconcile(context context.Context, request reco
if err != nil {
return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed set ready failure status to the cephRBDMirror %q", request.NamespacedName)
}
logger.Errorf("failed to reconcile %v", err)
log.NamespacedError(request.Namespace, logger, "failed to reconcile %v", err)
}
return reporting.ReportReconcileResult(logger, r.recorder, request, &cephRBDMirror, reconcileResponse, err)
@@ -180,7 +181,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
err := r.client.Get(r.opManagerContext, request.NamespacedName, cephRBDMirror)
if err != nil {
if kerrors.IsNotFound(err) {
logger.Debug("cephRBDMirror resource not found. Ignoring since object must be deleted.")
log.NamespacedDebug(request.Namespace, logger, "cephRBDMirror resource not found. Ignoring since object must be deleted.")
return reconcile.Result{}, *cephRBDMirror, nil
}
// Error reading the object - requeue the request.
@@ -213,7 +214,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
// Make sure a CephCluster is present otherwise do nothing
cephCluster, isReadyToReconcile, _, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName)
if !isReadyToReconcile {
logger.Debugf("CephCluster resource not ready in namespace %q, retrying in %q.", request.NamespacedName.Namespace, reconcileResponse.RequeueAfter.String())
log.NamedDebug(request.NamespacedName, logger, "CephCluster resource not ready, retrying in %q.", reconcileResponse.RequeueAfter.String())
return reconcileResponse, *cephRBDMirror, nil
}
r.cephClusterSpec = &cephCluster.Spec
@@ -238,7 +239,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
)
if err != nil {
if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
logger.Info(opcontroller.OperatorNotInitializedMessage)
log.NamespacedInfo(request.Namespace, logger, opcontroller.OperatorNotInitializedMessage)
return opcontroller.WaitForRequeueIfOperatorNotInitialized, *cephRBDMirror, nil
}
return reconcile.Result{}, *cephRBDMirror, errors.Wrap(err, "failed to detect running and desired ceph version")
@@ -256,7 +257,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
r.clusterInfo.CephVersion = *runningCephVersion
// Add bootstrap peer if any
logger.Debug("reconciling ceph rbd mirror peers addition")
log.NamespacedDebug(request.Namespace, logger, "reconciling ceph rbd mirror peers addition")
reconcileResponse, err = r.reconcileAddBootstrapPeer(cephRBDMirror, request.NamespacedName)
if err != nil {
return opcontroller.ImmediateRetryResult, *cephRBDMirror, errors.Wrap(err, "failed to add ceph rbd mirror peer")
@@ -268,11 +269,11 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
return reconcile.Result{}, *cephRBDMirror, errors.Wrapf(err, "failed to determine if cephx keys should be rotated for the cephRBDMirror %q", request.NamespacedName)
}
if r.shouldRotateCephxKeys {
logger.Infof("cephx keys for CephRBDMirror %q will be rotated", request.NamespacedName)
log.NamedInfo(request.NamespacedName, logger, "cephx keys for CephRBDMirror will be rotated")
}
// CREATE/UPDATE
logger.Debug("reconciling ceph rbd mirror deployments")
log.NamespacedDebug(request.Namespace, logger, "reconciling ceph rbd mirror deployments")
reconcileResponse, err = r.reconcileCreateCephRBDMirror(cephRBDMirror)
if err != nil {
return opcontroller.ImmediateRetryResult, *cephRBDMirror, errors.Wrap(err, "failed to create ceph rbd mirror deployments")
@@ -288,7 +289,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
}
// Return and do not requeue
logger.Debug("done reconciling ceph rbd mirror")
log.NamespacedDebug(request.Namespace, logger, "done reconciling ceph rbd mirror")
return reconcile.Result{}, *cephRBDMirror, nil
}
@@ -318,7 +319,7 @@ func (r *ReconcileCephRBDMirror) updateStatus(observedGeneration int64, namespac
err := r.client.Get(r.opManagerContext, namespacedName, rbdMirror)
if err != nil {
if kerrors.IsNotFound(err) {
logger.Debug("CephRBDMirror resource not found. Ignoring since object must be deleted.")
log.NamespacedDebug(namespacedName.Namespace, logger, "CephRBDMirror resource not found. Ignoring since object must be deleted.")
return nil
}
return errors.Wrapf(err, "failed to retrieve CephRBDMirror %q to update status to %q", namespacedName, status)
@@ -347,6 +348,6 @@ func (r *ReconcileCephRBDMirror) updateStatus(observedGeneration int64, namespac
return err
}
logger.Debugf("successfully updated cephRBDMirror %q status updated to %+v", namespacedName, rbdMirror.Status)
log.NamedDebug(namespacedName, logger, "successfully updated cephRBDMirror status updated to %+v", rbdMirror.Status)
return nil
}
+6 -5
View File
@@ -28,6 +28,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
@@ -51,7 +52,7 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro
return errors.Wrap(err, "error checking pod memory")
}
logger.Infof("configure rbd-mirroring with %d workers", cephRBDMirror.Spec.Count)
log.NamespacedInfo(cephRBDMirror.Namespace, logger, "configure rbd-mirroring with %d workers", cephRBDMirror.Spec.Count)
ownerInfo := k8sutil.NewOwnerInfo(cephRBDMirror, r.scheme)
daemonID := k8sutil.IndexToName(0)
@@ -68,7 +69,7 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro
return errors.Wrap(err, "failed to check for RBD Mirror to skip reconcile")
}
if rbdMirrorToSkipReconcile.Len() > 0 {
logger.Warningf("skipping RBD mirror reconcile since RBD mirror daemons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(rbdMirrorToSkipReconcile))
log.NamespacedWarning(cephRBDMirror.Namespace, logger, "skipping RBD mirror reconcile since RBD mirror daemons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(rbdMirrorToSkipReconcile))
return nil
}
@@ -102,11 +103,11 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro
if !kerrors.IsAlreadyExists(err) {
return errors.Wrapf(err, "failed to create %q deployment", resourceName)
}
logger.Infof("deployment for rbd-mirror %q already exists. updating if needed", resourceName)
log.NamespacedInfo(cephRBDMirror.Namespace, logger, "deployment for rbd-mirror %q already exists. updating if needed", resourceName)
if err := updateDeploymentAndWait(r.context, r.clusterInfo, d, config.RbdMirrorType, daemonConf.DaemonID, r.cephClusterSpec.SkipUpgradeChecks, false); err != nil {
// fail could be an issue updating label selector (immutable), so try del and recreate
logger.Debugf("updateDeploymentAndWait failed for rbd-mirror %q. Attempting del-and-recreate. %v", resourceName, err)
log.NamespacedDebug(cephRBDMirror.Namespace, logger, "updateDeploymentAndWait failed for rbd-mirror %q. Attempting del-and-recreate. %v", resourceName, err)
err = r.context.Clientset.AppsV1().Deployments(cephRBDMirror.Namespace).Delete(r.opManagerContext, d.Name, metav1.DeleteOptions{})
if err != nil && !kerrors.IsNotFound(err) {
return errors.Wrapf(err, "failed to delete rbd-mirror %q during del-and-recreate update attempt", resourceName)
@@ -117,6 +118,6 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro
}
}
logger.Infof("%q deployment started", resourceName)
log.NamespacedInfo(cephRBDMirror.Namespace, logger, "%q deployment started", resourceName)
return nil
}
@@ -23,6 +23,7 @@ import (
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/util/log"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "telemetry")
@@ -53,8 +54,8 @@ var CSIVersion string
func ReportKeyValue(context *clusterd.Context, clusterInfo *client.ClusterInfo, key, value string) {
ms := config.GetMonStore(context, clusterInfo)
if err := ms.SetKeyValue(key, value); err != nil {
logger.Warningf("failed to set telemetry key %q. %v", key, err)
log.NamespacedWarning(clusterInfo.Namespace, logger, "failed to set telemetry key %q. %v", key, err)
return
}
logger.Debugf("set telemetry key: %s=%s", key, value)
log.NamespacedDebug(clusterInfo.Namespace, logger, "set telemetry key: %s=%s", key, value)
}
+8 -7
View File
@@ -23,6 +23,7 @@ import (
"github.com/pkg/errors"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/log"
v1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -36,7 +37,7 @@ func populateConfigOverrideConfigMap(clusterdContext *clusterd.Context, namespac
existingCM, err := clusterdContext.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, k8sutil.ConfigOverrideName, metav1.GetOptions{})
if err != nil {
if !kerrors.IsNotFound(err) {
logger.Warningf("failed to get cm %q to check labels and annotations", k8sutil.ConfigOverrideName)
log.NamespacedWarning(namespace, logger, "failed to get cm %q to check labels and annotations", k8sutil.ConfigOverrideName)
return nil
}
@@ -66,7 +67,7 @@ func populateConfigOverrideConfigMap(clusterdContext *clusterd.Context, namespac
if err != nil {
return errors.Wrapf(err, "failed to create override configmap %s", namespace)
}
logger.Infof("created placeholder configmap for ceph overrides %q", cm.Name)
log.NamespacedInfo(namespace, logger, "created placeholder configmap for ceph overrides %q", cm.Name)
return nil
}
@@ -83,9 +84,9 @@ func populateConfigOverrideConfigMap(clusterdContext *clusterd.Context, namespac
if updateRequired {
_, err = clusterdContext.Clientset.CoreV1().ConfigMaps(namespace).Update(ctx, existingCM, metav1.UpdateOptions{})
if err != nil {
logger.Warningf("failed to add recommended labels and annotations to configmap %q. %v", existingCM.Name, err)
log.NamespacedWarning(namespace, logger, "failed to add recommended labels and annotations to configmap %q. %v", existingCM.Name, err)
} else {
logger.Infof("added expected labels and annotations to configmap %q", existingCM.Name)
log.NamespacedInfo(namespace, logger, "added expected labels and annotations to configmap %q", existingCM.Name)
}
}
return nil
@@ -96,15 +97,15 @@ func initRequiredMetadata(metadata metav1.ObjectMeta, labels, annotations map[st
releaseNameAttr := "meta.helm.sh/release-name"
chartName, ok := metadata.Annotations[releaseNameAttr]
if !ok {
logger.Debug("cluster helm chart is not configured, not adding helm annotations to configmap")
log.NamespacedDebug(metadata.Namespace, logger, "cluster helm chart is not configured, not adding helm annotations to configmap")
return false
}
if _, ok := annotations[releaseNameAttr]; ok {
logger.Debug("cluster helm chart helm annotations already added to configmap")
log.NamespacedDebug(metadata.Namespace, logger, "cluster helm chart helm annotations already added to configmap")
return false
}
logger.Infof("adding helm chart name %q annotation to configmap", chartName)
log.NamespacedInfo(metadata.Namespace, logger, "adding helm chart name %q annotation to configmap", chartName)
labels["app.kubernetes.io/managed-by"] = "Helm"
annotations[releaseNameAttr] = chartName
annotations["meta.helm.sh/release-namespace"] = metadata.Namespace
+22 -21
View File
@@ -24,6 +24,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
"github.com/rook/rook/pkg/operator/ceph/controller"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/util/log"
)
func (c *ClusterController) detectAndValidateCephVersion(cluster *cluster) (*cephver.CephVersion, bool, error) {
@@ -40,7 +41,7 @@ func (c *ClusterController) detectAndValidateCephVersion(cluster *cluster) (*cep
return nil, false, err
}
logger.Info("validating ceph version from provided image")
log.NamespacedInfo(cluster.Namespace, logger, "validating ceph version from provided image")
if err := cluster.validateCephVersion(version); err != nil {
return nil, cluster.isUpgrade, err
}
@@ -54,7 +55,7 @@ func (c *ClusterController) detectAndValidateCephVersion(cluster *cluster) (*cep
func (c *cluster) printOverallCephVersion() {
versions, err := daemonclient.GetAllCephDaemonVersions(c.context, c.ClusterInfo)
if err != nil {
logger.Errorf("failed to get ceph daemons versions. %v", err)
log.NamespacedError(c.Namespace, logger, "failed to get ceph daemons versions. %v", err)
return
}
@@ -62,21 +63,21 @@ func (c *cluster) printOverallCephVersion() {
for v := range versions.Overall {
version, err := cephver.ExtractCephVersion(v)
if err != nil {
logger.Errorf("failed to extract ceph version. %v", err)
log.NamespacedError(c.Namespace, logger, "failed to extract ceph version. %v", err)
return
}
vv := *version
logger.Infof("successfully upgraded cluster to version: %q", vv.String())
log.NamespacedInfo(c.Namespace, logger, "successfully upgraded cluster to version: %q", vv.String())
}
} else {
// This shouldn't happen, but let's log just in case
logger.Warningf("upgrade orchestration completed but somehow we still have more than one Ceph version running. %v:", versions.Overall)
log.NamespacedWarning(c.Namespace, logger, "upgrade orchestration completed but somehow we still have more than one Ceph version running. %v:", versions.Overall)
}
}
// This function compare the Ceph spec image and the cluster running version
// It returns true if the image is different and false if identical
func diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion, runningVersions cephv1.CephDaemonsVersions) (bool, error) {
func (c *cluster) diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion, runningVersions cephv1.CephDaemonsVersions) (bool, error) {
numberOfCephVersions := len(runningVersions.Overall)
if numberOfCephVersions == 0 {
// let's return immediately
@@ -85,7 +86,7 @@ func diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion,
if numberOfCephVersions > 1 {
// let's return immediately
logger.Warningf("it looks like we have more than one ceph version running. triggering upgrade. %+v:", runningVersions.Overall)
log.NamespacedWarning(c.Namespace, logger, "it looks like we have more than one ceph version running. triggering upgrade. %+v:", runningVersions.Overall)
return true, nil
}
@@ -93,24 +94,24 @@ func diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion,
for v := range runningVersions.Overall {
version, err := cephver.ExtractCephVersion(v)
if err != nil {
logger.Errorf("failed to extract ceph version. %v", err)
log.NamespacedError(c.Namespace, logger, "failed to extract ceph version. %v", err)
return false, err
}
clusterRunningVersion := *version
// If this is the same version
if cephver.IsIdentical(clusterRunningVersion, imageSpecVersion) {
logger.Debugf("both cluster and image spec versions are identical, doing nothing %s", imageSpecVersion.String())
log.NamespacedDebug(c.Namespace, logger, "both cluster and image spec versions are identical, doing nothing %s", imageSpecVersion.String())
return false, nil
}
if cephver.IsSuperior(imageSpecVersion, clusterRunningVersion) {
logger.Infof("image spec version %s is higher than the running cluster version %s, upgrading", imageSpecVersion.String(), clusterRunningVersion.String())
log.NamespacedInfo(c.Namespace, logger, "image spec version %s is higher than the running cluster version %s, upgrading", imageSpecVersion.String(), clusterRunningVersion.String())
return true, nil
}
if cephver.IsInferior(imageSpecVersion, clusterRunningVersion) {
logger.Warningf("image spec version %s is lower than the running cluster version %s, downgrading is not supported", imageSpecVersion.String(), clusterRunningVersion.String())
log.NamespacedWarning(c.Namespace, logger, "image spec version %s is lower than the running cluster version %s, downgrading is not supported", imageSpecVersion.String(), clusterRunningVersion.String())
return true, nil
}
}
@@ -129,11 +130,11 @@ func (c *cluster) validateCephVersion(version *cephver.CephVersion) error {
if !c.Spec.CephVersion.AllowUnsupported {
return errors.Errorf("allowUnsupported must be set to true to run with this version %q", version.String())
}
logger.Warningf("unsupported ceph version detected: %q, pursuing", version)
log.NamespacedWarning(c.Namespace, logger, "unsupported ceph version detected: %q, pursuing", version)
}
if version.Unsupported() {
logger.Errorf("UNSUPPORTED: ceph version %q detected, it is recommended to rollback to the previous pin-point stable release, pursuing anyways", version)
log.NamespacedError(c.Namespace, logger, "UNSUPPORTED: ceph version %q detected, it is recommended to rollback to the previous pin-point stable release, pursuing anyways", version)
}
}
@@ -149,13 +150,13 @@ func (c *cluster) validateCephVersion(version *cephver.CephVersion) error {
// Write connection info (ceph config file and keyring) for ceph commands
err = mon.WriteConnectionConfig(c.context, clusterInfo)
if err != nil {
logger.Errorf("failed to write config. attempting to continue. %v", err)
log.NamespacedError(c.Namespace, logger, "failed to write config. attempting to continue. %v", err)
}
}
if err := clusterInfo.IsInitialized(); err != nil {
// If not initialized, this is likely a new cluster so there is nothing to do
logger.Debugf("cluster not initialized, nothing to validate. %v", err)
log.NamespacedDebug(c.Namespace, logger, "cluster not initialized, nothing to validate. %v", err)
return nil
}
@@ -170,21 +171,21 @@ func (c *cluster) validateCephVersion(version *cephver.CephVersion) error {
// On external cluster setup, if we don't bootstrap any resources in the Kubernetes cluster then
// there is no need to validate the Ceph image further
if c.Spec.External.Enable && c.Spec.CephVersion.Image == "" {
logger.Debug("no spec image specified on external cluster, not validating Ceph version.")
log.NamespacedDebug(c.Namespace, logger, "no spec image specified on external cluster, not validating Ceph version.")
return nil
}
// Get cluster running versions
versions, err := daemonclient.GetAllCephDaemonVersions(c.context, c.ClusterInfo)
if err != nil {
logger.Errorf("failed to get ceph daemons versions, this typically happens during the first cluster initialization. %v", err)
log.NamespacedError(c.Namespace, logger, "failed to get ceph daemons versions, this typically happens during the first cluster initialization. %v", err)
return nil
}
runningVersions := *versions
differentImages, err := diffImageSpecAndClusterRunningVersion(*version, runningVersions)
differentImages, err := c.diffImageSpecAndClusterRunningVersion(*version, runningVersions)
if err != nil {
logger.Errorf("failed to determine if we should upgrade or not. %v", err)
log.NamespacedError(c.Namespace, logger, "failed to determine if we should upgrade or not. %v", err)
// we shouldn't block the orchestration if we can't determine the version of the image spec, we proceed anyway in best effort
// we won't be able to check if there is an update or not and what to do, so we don't check the cluster status either
// This will happen if someone uses ceph/daemon:latest-master for instance
@@ -197,13 +198,13 @@ func (c *cluster) validateCephVersion(version *cephver.CephVersion) error {
cephHealthy := daemonclient.IsCephHealthy(c.context, c.ClusterInfo)
if !cephHealthy {
if c.Spec.SkipUpgradeChecks {
logger.Warning("ceph is not healthy but SkipUpgradeChecks is set, forcing upgrade.")
log.NamespacedWarning(c.Namespace, logger, "ceph is not healthy but SkipUpgradeChecks is set, forcing upgrade.")
} else {
return errors.Errorf("ceph status in namespace %s is not healthy, refusing to upgrade. Either fix the health issue or force an update by setting skipUpgradeChecks to true in the cluster CR", c.Namespace)
}
}
// This is an upgrade
logger.Infof("upgrading ceph cluster to %q", version.String())
log.NamespacedInfo(c.Namespace, logger, "upgrading ceph cluster to %q", version.String())
c.isUpgrade = true
}
+8 -7
View File
@@ -44,7 +44,8 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
err := json.Unmarshal(fakeRunningVersions, &dummyRunningVersions)
assert.NoError(t, err)
m, err := diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions)
c := testSpec(t)
m, err := c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions)
assert.Error(t, err) // Overall is absent
assert.False(t, m)
@@ -60,7 +61,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
err = json.Unmarshal(fakeRunningVersions, &dummyRunningVersions2)
assert.NoError(t, err)
m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions2)
m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions2)
assert.NoError(t, err)
assert.True(t, m)
@@ -76,7 +77,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
assert.NoError(t, err)
// Allow the downgrade
m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions3)
m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions3)
assert.NoError(t, err)
assert.True(t, m)
@@ -92,7 +93,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
err = json.Unmarshal(fakeRunningVersions, &dummyRunningVersions4)
assert.NoError(t, err)
m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions4)
m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions4)
assert.NoError(t, err)
assert.True(t, m)
@@ -111,7 +112,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
err = json.Unmarshal(fakeRunningVersions, &dummyRunningVersions5)
assert.NoError(t, err)
m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions5)
m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions5)
assert.NoError(t, err)
assert.False(t, m)
@@ -130,7 +131,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
err = json.Unmarshal(fakeRunningVersions, &dummyRunningVersions6)
assert.NoError(t, err)
m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions6)
m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions6)
assert.NoError(t, err)
assert.True(t, m)
@@ -149,7 +150,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
err = json.Unmarshal(fakeRunningVersions, &dummyRunningVersions7)
assert.NoError(t, err)
m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions7)
m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions7)
assert.NoError(t, err)
assert.False(t, m)
}
+63
View File
@@ -0,0 +1,63 @@
/*
Copyright 2025 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package log
import (
"fmt"
"github.com/coreos/pkg/capnslog"
"k8s.io/apimachinery/pkg/types"
)
func NamedInfo(name types.NamespacedName, logger *capnslog.PackageLogger, message string, args ...interface{}) {
NamespacedInfo(name.String(), logger, message, args...)
}
func NamedWarning(name types.NamespacedName, logger *capnslog.PackageLogger, message string, args ...interface{}) {
NamespacedWarning(name.String(), logger, message, args...)
}
func NamedError(name types.NamespacedName, logger *capnslog.PackageLogger, message string, args ...interface{}) {
NamespacedError(name.String(), logger, message, args...)
}
func NamedDebug(name types.NamespacedName, logger *capnslog.PackageLogger, message string, args ...interface{}) {
NamespacedDebug(name.String(), logger, message, args...)
}
func NamedTrace(name types.NamespacedName, logger *capnslog.PackageLogger, message string, args ...interface{}) {
NamespacedTrace(name.String(), logger, message, args...)
}
func NamespacedInfo(namespace string, logger *capnslog.PackageLogger, message string, args ...interface{}) {
logger.Infof("[%s] %s", namespace, fmt.Sprintf(message, args...))
}
func NamespacedWarning(namespace string, logger *capnslog.PackageLogger, message string, args ...interface{}) {
logger.Warningf("[%s] %s", namespace, fmt.Sprintf(message, args...))
}
func NamespacedError(namespace string, logger *capnslog.PackageLogger, message string, args ...interface{}) {
logger.Errorf("[%s] %s", namespace, fmt.Sprintf(message, args...))
}
func NamespacedDebug(namespace string, logger *capnslog.PackageLogger, message string, args ...interface{}) {
logger.Debugf("[%s] %s", namespace, fmt.Sprintf(message, args...))
}
func NamespacedTrace(namespace string, logger *capnslog.PackageLogger, message string, args ...interface{}) {
logger.Tracef("[%s] %s", namespace, fmt.Sprintf(message, args...))
}