core: include namespace more consistently in logging
When writing to the operator log, where possible the namespace of the resource on which the operator is working will be logged as a standard prefix to more easily troubleshoot what resource is being reconciled Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
43 files changed
+727
-622
No files matched your search
@@ -34,6 +34,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/reporting"
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -75,7 +76,7 @@ func newCephStatusChecker(context *clusterd.Context, clusterInfo *cephclient.Clu
|
||||
checkInterval = &checkIntervalCRSetting.Duration
|
||||
}
|
||||
if checkInterval != nil {
|
||||
logger.Infof("ceph status check interval is %s", checkInterval.String())
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "ceph status check interval is %s", checkInterval.String())
|
||||
c.interval = checkInterval
|
||||
}
|
||||
|
||||
@@ -87,7 +88,7 @@ func (c *cephStatusChecker) checkCephStatus(monitoringRoutines *sync.Map, daemon
|
||||
// check the status immediately before starting the loop
|
||||
v, ok := monitoringRoutines.Load(daemon)
|
||||
if !ok {
|
||||
logger.Infof("ceph cluster %q has been deleted. stopping ceph status check", c.clusterInfo.Namespace)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "ceph cluster %q has been deleted. stopping ceph status check", c.clusterInfo.Namespace)
|
||||
return
|
||||
}
|
||||
c.checkStatus(v.(*opcontroller.ClusterHealth).InternalCtx)
|
||||
@@ -97,13 +98,13 @@ func (c *cephStatusChecker) checkCephStatus(monitoringRoutines *sync.Map, daemon
|
||||
// we will get an invalid pointer error and the go routine will panic
|
||||
v, ok := monitoringRoutines.Load(daemon)
|
||||
if !ok {
|
||||
logger.Infof("ceph cluster %q has been deleted. stopping ceph status check", c.clusterInfo.Namespace)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "ceph cluster %q has been deleted. stopping ceph status check", c.clusterInfo.Namespace)
|
||||
return
|
||||
}
|
||||
health := v.(*opcontroller.ClusterHealth)
|
||||
select {
|
||||
case <-health.InternalCtx.Done():
|
||||
logger.Infof("stopping monitoring of ceph status")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "stopping monitoring of ceph status")
|
||||
monitoringRoutines.Delete(daemon)
|
||||
return
|
||||
|
||||
@@ -118,27 +119,27 @@ func (c *cephStatusChecker) checkStatus(ctx context.Context) {
|
||||
var status cephclient.CephStatus
|
||||
var err error
|
||||
|
||||
logger.Debugf("checking health of cluster")
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "checking health of cluster")
|
||||
// Check ceph's status
|
||||
status, err = cephclient.StatusWithUser(c.context, c.clusterInfo)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
|
||||
logger.Info("skipping ceph status since operator is still initializing")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "skipping ceph status since operator is still initializing")
|
||||
return
|
||||
}
|
||||
logger.Errorf("failed to get ceph status. %v", err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get ceph status. %v", err)
|
||||
status := cephStatusOnError(err.Error())
|
||||
c.updateCephStatus(status)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Debugf("cluster status: %+v", status)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "cluster status: %+v", status)
|
||||
c.updateCephStatus(&status)
|
||||
|
||||
if status.Health.Status != "HEALTH_OK" {
|
||||
logger.Debug("checking for stuck pods on not ready nodes")
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "checking for stuck pods on not ready nodes")
|
||||
if err := c.forceDeleteStuckRookPodsOnNotReadyNodes(ctx); err != nil {
|
||||
logger.Errorf("failed to delete pod on not ready nodes. %v", err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to delete pod on not ready nodes. %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,16 +149,16 @@ func (c *cephStatusChecker) checkStatus(ctx context.Context) {
|
||||
func (c *cephStatusChecker) configureHealthSettings(status cephclient.CephStatus) {
|
||||
// loop through the health codes and log what we find
|
||||
for healthCode, check := range status.Health.Checks {
|
||||
logger.Debugf("Health: %q, code: %q, message: %q", check.Severity, healthCode, check.Summary.Message)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "Health: %q, code: %q, message: %q", check.Severity, healthCode, check.Summary.Message)
|
||||
}
|
||||
|
||||
// disable the insecure global id if there are no old clients
|
||||
if _, ok := status.Health.Checks["AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED"]; ok {
|
||||
if _, ok := status.Health.Checks["AUTH_INSECURE_GLOBAL_ID_RECLAIM"]; !ok {
|
||||
logger.Info("Disabling the insecure global ID as no legacy clients are currently connected. If you still require the insecure connections, see the CVE to suppress the health warning and re-enable the insecure connections. https://docs.ceph.com/en/latest/security/CVE-2021-20288/")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Disabling the insecure global ID as no legacy clients are currently connected. If you still require the insecure connections, see the CVE to suppress the health warning and re-enable the insecure connections. https://docs.ceph.com/en/latest/security/CVE-2021-20288/")
|
||||
config.DisableInsecureGlobalID(c.context, c.clusterInfo)
|
||||
} else {
|
||||
logger.Warning("insecure clients are connected to the cluster, to resolve the AUTH_INSECURE_GLOBAL_ID_RECLAIM health warning please refer to the upgrade guide to ensure all Ceph daemons are updated.")
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "insecure clients are connected to the cluster, to resolve the AUTH_INSECURE_GLOBAL_ID_RECLAIM health warning please refer to the upgrade guide to ensure all Ceph daemons are updated.")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -168,10 +169,10 @@ func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus) {
|
||||
cephCluster, err := c.context.RookClientset.CephV1().CephClusters(clusterName.Namespace).Get(c.clusterInfo.Context, clusterName.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Debug("CephCluster resource not found. Ignoring since object must be deleted.")
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "CephCluster resource not found. Ignoring since object must be deleted.")
|
||||
return
|
||||
}
|
||||
logger.Errorf("failed to retrieve ceph cluster %q in namespace %q to update status to %+v", clusterName.Name, clusterName.Namespace, status)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to retrieve ceph cluster %q in namespace %q to update status to %+v", clusterName.Name, clusterName.Namespace, status)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -181,16 +182,16 @@ func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus) {
|
||||
// versions store the ceph version of all the ceph daemons and overall cluster version
|
||||
versions, err := cephclient.GetAllCephDaemonVersions(c.context, c.clusterInfo)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get ceph daemons versions. %v", err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get ceph daemons versions. %v", err)
|
||||
} else {
|
||||
// Update status with Ceph versions
|
||||
cephCluster.Status.CephStatus.Versions = versions
|
||||
}
|
||||
|
||||
// Update condition
|
||||
logger.Debugf("updating ceph cluster %q status to %+v", clusterName.Namespace, status)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "updating ceph cluster %q status to %+v", clusterName.Namespace, status)
|
||||
if err := reporting.UpdateStatus(c.context.Client, cephCluster); err != nil {
|
||||
logger.Errorf("failed to update cluster status to %+v. %v", status, err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to update cluster status to %+v. %v", status, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -237,15 +238,15 @@ func formatTime(t time.Time) string {
|
||||
}
|
||||
|
||||
func (c *ClusterController) updateClusterCephVersion(cluster *cluster, cephVersion cephver.CephVersion) {
|
||||
logger.Infof("cluster %q: version %q detected for image %q", cluster.namespacedName.Namespace, cephVersion.String(), cluster.Spec.CephVersion.Image)
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "cluster %q: version %q detected for image %q", cluster.namespacedName.Namespace, cephVersion.String(), cluster.Spec.CephVersion.Image)
|
||||
|
||||
cephCluster, err := c.context.RookClientset.CephV1().CephClusters(cluster.namespacedName.Namespace).Get(c.OpManagerCtx, cluster.namespacedName.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Debug("CephCluster resource not found. Ignoring since object must be deleted.")
|
||||
log.NamespacedDebug(cluster.Namespace, logger, "CephCluster resource not found. Ignoring since object must be deleted.")
|
||||
return
|
||||
}
|
||||
logger.Errorf("failed to retrieve ceph cluster %q to update ceph version to %+v. %v", cluster.namespacedName.Name, cephVersion, err)
|
||||
log.NamespacedError(cluster.Namespace, logger, "failed to retrieve ceph cluster %q to update ceph version to %+v. %v", cluster.namespacedName.Name, cephVersion, err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -257,7 +258,7 @@ func (c *ClusterController) updateClusterCephVersion(cluster *cluster, cephVersi
|
||||
// do not overwrite the ceph status that is updated in a separate goroutine
|
||||
cephCluster.Status.CephVersion = cephClusterVersion
|
||||
if err := reporting.UpdateStatus(c.client, cephCluster); err != nil {
|
||||
logger.Errorf("failed to update cluster %q version. %v", cluster.namespacedName.Name, err)
|
||||
log.NamespacedError(cluster.Namespace, logger, "failed to update cluster %q version. %v", cluster.namespacedName.Name, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -289,11 +290,11 @@ func (c *cephStatusChecker) forceDeleteStuckRookPodsOnNotReadyNodes(ctx context.
|
||||
for _, node := range nodes {
|
||||
pods, err := c.getRookPodsOnNode(node.Name)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get pods on NotReady node %q. %v", node.Name, err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get pods on NotReady node %q. %v", node.Name, err)
|
||||
}
|
||||
for _, pod := range pods {
|
||||
if err := k8sutil.ForceDeletePodIfStuck(ctx, c.context, pod); err != nil {
|
||||
logger.Warningf("skipping forced delete of stuck pod %q. %v", pod.Name, err)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "skipping forced delete of stuck pod %q. %v", pod.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/ceph/reporting"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/client-go/util/retry"
|
||||
)
|
||||
@@ -73,9 +74,9 @@ func claimAdminRotationLock(namespace string) error {
|
||||
adminRotationMutex.Lock()
|
||||
defer adminRotationMutex.Unlock()
|
||||
if _, ok := adminRotationInProgress[namespace]; ok {
|
||||
return fmt.Errorf("failed to claim admin rotation lock for cluster in namespace %q", namespace)
|
||||
return fmt.Errorf("failed to claim admin rotation lock")
|
||||
}
|
||||
logger.Debugf("claimed admin rotation lock for cluster in namespace %q", namespace)
|
||||
log.NamespacedDebug(namespace, logger, "claimed admin rotation lock")
|
||||
adminRotationInProgress[namespace] = struct{}{}
|
||||
return nil
|
||||
}
|
||||
@@ -118,7 +119,7 @@ func rotateAdminCephxKey(
|
||||
) error {
|
||||
if clusterInfo.CephCred.Username != cephclient.AdminUsername {
|
||||
// this shouldn't happen during normal runtime - this could indicate external mode cluster
|
||||
logger.Infof("cannot rotate admin cephx key with non-rotatable username %q for cluster in namespace %q", clusterInfo.CephCred.Username, clusterInfo.Namespace)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "cannot rotate admin cephx key with non-rotatable username %q", clusterInfo.CephCred.Username)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -129,7 +130,7 @@ func rotateAdminCephxKey(
|
||||
return errors.Wrap(err, "failed to determine if admin cephx key should be rotated")
|
||||
}
|
||||
if !shouldRotate {
|
||||
logger.Debugf("not rotating admin cephx key for cluster in namespace %q", clusterInfo.Namespace)
|
||||
log.NamespacedDebug(clusterInfo.Namespace, logger, "not rotating admin cephx key")
|
||||
// no rotation, but still update cephx status - critical for greenfield Uninitialized clusters
|
||||
err := updateCephClusterAdminCephxStatus(clusterdCtx, clusterInfo, false)
|
||||
if err != nil {
|
||||
@@ -143,7 +144,7 @@ func rotateAdminCephxKey(
|
||||
}
|
||||
defer releaseAdminRotationLock(clusterInfo.Namespace)
|
||||
|
||||
logger.Infof("beginning admin cephx key rotation for cluster in namespace %q", clusterInfo.Namespace)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "beginning admin cephx key rotation")
|
||||
|
||||
// As an optimization, use a (shallow) copy of clusterInfo with a context that won't be canceled
|
||||
// during normal runtime. This disallows the rotation process from being interrupted partway
|
||||
@@ -194,7 +195,7 @@ func recoverPriorAdminCephxKeyRotation(
|
||||
) error {
|
||||
if clusterInfo == nil {
|
||||
// new cluster hasn't been deployed yet. there should be no way for recovery to be needed
|
||||
logger.Debugf("no admin cephx key recovery possible for cluster with empty clusterInfo in namespace %q", clusterNamespace)
|
||||
log.NamespacedDebug(clusterNamespace, logger, "no admin cephx key recovery possible for cluster with empty clusterInfo")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -203,7 +204,7 @@ func recoverPriorAdminCephxKeyRotation(
|
||||
rotatorKeyring, err := s.GetKeyringFromSecret(adminRotatorSecretPrefix)
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Debugf("no admin cephx key recovery needed for cluster in namespace %q", clusterInfo.Namespace)
|
||||
log.NamespacedDebug(clusterNamespace, logger, "no admin cephx key recovery needed")
|
||||
return nil
|
||||
}
|
||||
return errors.Wrap(err, "failed to get admin rotator cephx key secret to check if admin cephx key rotation recovery is needed")
|
||||
@@ -214,7 +215,7 @@ func recoverPriorAdminCephxKeyRotation(
|
||||
}
|
||||
defer releaseAdminRotationLock(clusterNamespace)
|
||||
|
||||
logger.Infof("recovering from interrupted admin cephx key rotation for cluster in namespace %q", clusterInfo.Namespace)
|
||||
log.NamespacedInfo(clusterNamespace, logger, "recovering from interrupted admin cephx key rotation")
|
||||
|
||||
// As an optimization, use a (shallow) copy of clusterInfo with a context that won't be canceled
|
||||
// during normal runtime. This disallows the recovery process from being interrupted partway
|
||||
@@ -239,7 +240,7 @@ func recoverPriorAdminCephxKeyRotation(
|
||||
|
||||
authList, err := cephclient.AuthList(clusterdCtx, clusterInfoCopy)
|
||||
if err != nil {
|
||||
logger.Debugf("primary admin user failed to list ceph auth - this is an expected condition when recovering from admin cephx key rotation")
|
||||
log.NamespacedDebug(clusterNamespace, logger, "primary admin user failed to list ceph auth - this is an expected condition when recovering from admin cephx key rotation")
|
||||
} else if !entityExistsInAuthList(adminRotatorUsername, authList) { // err == nil
|
||||
// corner case: if the primary client.admin works AND client.admin-rotator is not present in
|
||||
// the `auth ls` output, it means rotation succeeded and was interrupted during cleanup.
|
||||
@@ -271,7 +272,7 @@ func rotateAdminCephxKeyUsingRotator(
|
||||
) error {
|
||||
// write admin rotator keyring file to tempdir so it can be used for rotation
|
||||
rotatorKeyringTmpfile := filepath.Join(tmpDir, adminRotatorUsername+".keyring")
|
||||
logger.Infof("temporarily storing admin rotator keyring at %q for cluster in namespace %q", rotatorKeyringTmpfile, clusterInfo.Namespace)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "temporarily storing admin rotator keyring at %q", rotatorKeyringTmpfile)
|
||||
err := cephclient.WriteKeyring(rotatorKeyringTmpfile, rotatorKeyring)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to write admin rotator keyring to %q", rotatorKeyringTmpfile)
|
||||
@@ -287,7 +288,7 @@ func rotateAdminCephxKeyUsingRotator(
|
||||
}
|
||||
|
||||
// as `client.admin-rotator`: run `ceph auth rotate client.admin`
|
||||
logger.Infof("admin cephx key will be rotated for cluster in namespace %q. rook will restart afterwards. some reconciles and health checks may fail in between - this is normal", clusterInfo.Namespace)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "admin cephx key will be rotated. rook will restart afterwards. some reconciles and health checks may fail in between - this is normal")
|
||||
newAdminKey, err := cephclient.AuthRotate(clusterdCtx, rotatorInfo, cephclient.AdminUsername)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to rotate admin key using admin rotator client")
|
||||
@@ -296,7 +297,7 @@ func rotateAdminCephxKeyUsingRotator(
|
||||
|
||||
// write new admin keyring to tempdir so it can be verified before it's stored permanently
|
||||
newAdminKeyringTmpfile := filepath.Join(tmpDir, cephclient.AdminUsername+".keyring")
|
||||
logger.Infof("temporarily storing rotated admin cephx keyring at %q for cluster in namespace %q", newAdminKeyringTmpfile, clusterInfo.Namespace)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "temporarily storing rotated admin cephx keyring at %q", newAdminKeyringTmpfile)
|
||||
err = cephclient.WriteKeyring(newAdminKeyringTmpfile, newAdminKeyring)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to write admin rotator keyring to %q", newAdminKeyringTmpfile)
|
||||
@@ -345,7 +346,7 @@ func finalizeAdminKeyRotation(
|
||||
// clean up tmpDir now that it's no longer useful for recovery
|
||||
if err := os.RemoveAll(tmpDir); err != nil {
|
||||
// most likely, operator pod got rescheduled to a different node where dir isn't present
|
||||
logger.Infof("non-critical failure removing admin cephx key rotation temp dir %q. %v", tmpDir, err)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "non-critical failure removing admin cephx key rotation temp dir %q. %v", tmpDir, err)
|
||||
}
|
||||
|
||||
// delete the `rook-ceph-admin-rotator-keyring` secret (store.Delete)
|
||||
@@ -361,7 +362,7 @@ func finalizeAdminKeyRotation(
|
||||
if err != nil {
|
||||
// this should be rare, and failure here doesn't need to block finalization of admin key
|
||||
// rotation. status will be missing ceph version, but cluster can continue to reconcile
|
||||
logger.Errorf("non-critical failure to determine missing ceph version of cluster in namespace %q for admin cephx key status", clusterInfo.Namespace)
|
||||
log.NamespacedError(clusterInfo.Namespace, logger, "non-critical failure to determine missing ceph version of cluster for admin cephx key status")
|
||||
} else {
|
||||
clusterInfo.CephVersion = ver
|
||||
}
|
||||
@@ -375,7 +376,7 @@ func finalizeAdminKeyRotation(
|
||||
|
||||
// restart Rook to force health checkers to restart with latest clusterInfo
|
||||
// do this even during rotation recovery since the health checkers could still be running
|
||||
logger.Infof("restarting rook operator after successful admin cephx key rotation for cluster in namespace %q", clusterInfo.Namespace)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "restarting rook operator after successful admin cephx key rotation")
|
||||
reloadManagerFunc()
|
||||
|
||||
return errSuccessfulAdminKeyRotation
|
||||
@@ -390,7 +391,7 @@ func updateCephClusterAdminCephxStatus(clusterdCtx *clusterd.Context, clusterInf
|
||||
}
|
||||
cephx := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, clusterInfo.CephVersion, cluster.Status.Cephx.Admin)
|
||||
cluster.Status.Cephx.Admin = cephx
|
||||
logger.Debugf("updating admin key cephx status to %+v", cephx)
|
||||
log.NamespacedDebug(clusterInfo.Namespace, logger, "updating admin key cephx status to %+v", cephx)
|
||||
if err := reporting.UpdateStatus(clusterdCtx.Client, cluster); err != nil {
|
||||
return errors.Wrap(err, "failed to update admin key cephx status")
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/file/mirror"
|
||||
"github.com/rook/rook/pkg/operator/ceph/object"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
batch "k8s.io/api/batch/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -59,10 +60,10 @@ var (
|
||||
)
|
||||
|
||||
func (c *ClusterController) startClusterCleanUp(context context.Context, cluster *cephv1.CephCluster, cephHosts []string, monSecret, clusterFSID string) {
|
||||
logger.Infof("starting clean up for cluster %q", cluster.Name)
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "starting clean up for cluster %q", cluster.Name)
|
||||
err := c.waitForCephDaemonCleanUp(context, cluster, time.Duration(clusterCleanUpPolicyRetryInterval)*time.Second)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to wait till ceph daemons are destroyed. %v", err)
|
||||
log.NamespacedError(cluster.Namespace, logger, "failed to wait till ceph daemons are destroyed. %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -71,7 +72,7 @@ func (c *ClusterController) startClusterCleanUp(context context.Context, cluster
|
||||
|
||||
func (c *ClusterController) startCleanUpJobs(cluster *cephv1.CephCluster, cephHosts []string, monSecret, clusterFSID string) {
|
||||
for _, hostName := range cephHosts {
|
||||
logger.Infof("starting clean up job on node %q", hostName)
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "starting clean up job on node %q", hostName)
|
||||
jobName := k8sutil.TruncateNodeNameForJob("cluster-cleanup-job-%s", hostName)
|
||||
podSpec := c.cleanUpJobTemplateSpec(cluster, monSecret, clusterFSID)
|
||||
podSpec.Spec.NodeSelector = map[string]string{k8sutil.LabelHostname(): hostName}
|
||||
@@ -93,7 +94,7 @@ func (c *ClusterController) startCleanUpJobs(cluster *cephv1.CephCluster, cephHo
|
||||
cephv1.GetCleanupLabels(cluster.Spec.Labels).ApplyToObjectMeta(&job.ObjectMeta)
|
||||
|
||||
if err := k8sutil.RunReplaceableJob(c.OpManagerCtx, c.context.Clientset, job, true); err != nil {
|
||||
logger.Errorf("failed to run cluster clean up job on node %q. %v", hostName, err)
|
||||
log.NamespacedError(cluster.Namespace, logger, "failed to run cluster clean up job on node %q. %v", hostName, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -205,7 +206,7 @@ func getCleanupPlacement(c cephv1.ClusterSpec) cephv1.Placement {
|
||||
}
|
||||
|
||||
func (c *ClusterController) waitForCephDaemonCleanUp(context context.Context, cluster *cephv1.CephCluster, retryInterval time.Duration) error {
|
||||
logger.Infof("waiting for all the ceph daemons to be cleaned up in the cluster %q", cluster.Namespace)
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "waiting for all the ceph daemons to be cleaned up in the cluster %q", cluster.Namespace)
|
||||
for {
|
||||
select {
|
||||
case <-time.After(retryInterval):
|
||||
@@ -215,11 +216,11 @@ func (c *ClusterController) waitForCephDaemonCleanUp(context context.Context, cl
|
||||
}
|
||||
|
||||
if len(cephHosts) == 0 {
|
||||
logger.Info("all ceph daemons are cleaned up")
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "all ceph daemons are cleaned up")
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Debugf("waiting for ceph daemons in cluster %q to be cleaned up. Retrying in %q",
|
||||
log.NamespacedDebug(cluster.Namespace, logger, "waiting for ceph daemons in cluster %q to be cleaned up. Retrying in %q",
|
||||
cluster.Namespace, retryInterval.String())
|
||||
case <-context.Done():
|
||||
return errors.Errorf("cancelling the host cleanup job. %s", context.Err())
|
||||
@@ -250,7 +251,7 @@ func (c *ClusterController) getCephHosts(namespace string) ([]string, error) {
|
||||
fmt.Fprintf(&b, "%s: %d. ", app, len(podList.Items))
|
||||
}
|
||||
|
||||
logger.Infof("existing ceph daemons in the namespace %q. %s", namespace, b.String())
|
||||
log.NamespacedInfo(namespace, logger, "existing ceph daemons in the namespace %q. %s", namespace, b.String())
|
||||
|
||||
for nodeName := range nodeNameList {
|
||||
podHostName, err := k8sutil.GetNodeHostName(c.OpManagerCtx, c.context.Clientset, nodeName)
|
||||
|
||||
@@ -46,6 +46,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/reporting"
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
rookversion "github.com/rook/rook/pkg/version"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -104,7 +105,7 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep
|
||||
|
||||
// Execute actions before the monitors are up and running, if needed during upgrades.
|
||||
// These actions would be skipped in a new cluster.
|
||||
logger.Debug("monitors are about to reconcile, executing pre actions")
|
||||
log.NamespacedDebug(c.Namespace, logger, "monitors are about to reconcile, executing pre actions")
|
||||
err = c.preMonStartupActions(cephVersion)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to execute actions before reconciling the ceph monitors")
|
||||
@@ -132,7 +133,7 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep
|
||||
}
|
||||
|
||||
// Execute actions after the monitors are up and running
|
||||
logger.Debug("monitors are up and running, executing post actions")
|
||||
log.NamespacedDebug(c.Namespace, logger, "monitors are up and running, executing post actions")
|
||||
err = c.postMonStartupActions()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to execute post actions after all the ceph monitors started")
|
||||
@@ -147,7 +148,7 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep
|
||||
}
|
||||
|
||||
// Execute actions after the managers are up and running
|
||||
logger.Debug("managers are up and running, executing post actions")
|
||||
log.NamespacedDebug(c.Namespace, logger, "managers are up and running, executing post actions")
|
||||
err = c.postMgrStartupActions()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to execute post actions after all the ceph managers started")
|
||||
@@ -168,7 +169,7 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("done reconciling ceph cluster in namespace %q", c.Namespace)
|
||||
log.NamespacedInfo(c.Namespace, logger, "done reconciling ceph cluster")
|
||||
|
||||
// We should be done updating by now
|
||||
if c.isUpgrade {
|
||||
@@ -186,7 +187,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error {
|
||||
cleanDataDirHostPath := path.Clean(cluster.Spec.DataDirHostPath)
|
||||
for _, b := range disallowedHostDirectories {
|
||||
if cleanDataDirHostPath == b {
|
||||
logger.Errorf("dataDirHostPath (given: %q) must not be used, conflicts with %q internal path", cluster.Spec.DataDirHostPath, b)
|
||||
log.NamespacedError(cluster.Namespace, logger, "dataDirHostPath (given: %q) must not be used, conflicts with %q internal path", cluster.Spec.DataDirHostPath, b)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -202,7 +203,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error {
|
||||
clusterInfo, _, _, err := controller.LoadClusterInfo(c.context, c.OpManagerCtx, cluster.Namespace, cluster.Spec)
|
||||
if err != nil {
|
||||
if errors.Is(err, controller.ClusterInfoNoClusterNoSecret) {
|
||||
logger.Info("clusterInfo not yet found, must be a new cluster.")
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "clusterInfo not yet found, must be a new cluster.")
|
||||
// ClusterInfoNoClusterNoSecret should return nil clusterInfo, but this is a
|
||||
// mandatory condition for recoverPriorAdminCephxKeyRotation() below, so ensure it
|
||||
clusterInfo = nil
|
||||
@@ -216,7 +217,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error {
|
||||
if cluster.mons.ClusterInfo == nil {
|
||||
// ClusterInfo stored in cluster.mons can be lost from the object stored in the
|
||||
// clusterMap during admin key rotation corner cases. rehydrate it if needed
|
||||
logger.Debugf("applying missing ClusterInfo to tracked mons info for cluster in namespace %q", cluster.Namespace)
|
||||
log.NamespacedDebug(cluster.Namespace, logger, "applying missing ClusterInfo to tracked mons info")
|
||||
cluster.mons.ClusterInfo = clusterInfo
|
||||
}
|
||||
}
|
||||
@@ -301,7 +302,7 @@ func (c *ClusterController) configureLocalCephCluster(cluster *cluster) error {
|
||||
// Validate the cluster Specs
|
||||
func preClusterStartValidation(cluster *cluster) error {
|
||||
if cluster.Spec.Mon.Count == 0 {
|
||||
logger.Warningf("mon count should be at least 1, will use default value of %d", mon.DefaultMonCount)
|
||||
log.NamespacedWarning(cluster.Namespace, logger, "mon count should be at least 1, will use default value of %d", mon.DefaultMonCount)
|
||||
cluster.Spec.Mon.Count = mon.DefaultMonCount
|
||||
}
|
||||
if !cluster.Spec.Mon.AllowMultiplePerNode {
|
||||
@@ -328,7 +329,7 @@ func preClusterStartValidation(cluster *cluster) error {
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debug("cluster spec successfully validated")
|
||||
log.NamespacedDebug(cluster.Namespace, logger, "cluster spec successfully validated")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -391,7 +392,7 @@ func (c *cluster) replaceDefaultReplicationRule(newRoot string) error {
|
||||
// CRUSH root
|
||||
// in both cases, we cannot do anything about it either way and
|
||||
// we’ll assume that the user knows what they’re doing.
|
||||
logger.Warning("replicated_rule is in use, not replaced")
|
||||
log.NamespacedWarning(c.Namespace, logger, "replicated_rule is in use, not replaced")
|
||||
return nil
|
||||
}
|
||||
// the error does not refer to EBUSY -> return as error
|
||||
@@ -432,7 +433,7 @@ func (c *cluster) removeDefaultCrushRoot() error {
|
||||
//
|
||||
// in all cases, we cannot do anything about it either way and
|
||||
// we’ll assume that the user knows what they’re doing.
|
||||
logger.Debug("default is not empty or is still in use, not removed")
|
||||
log.NamespacedDebug(c.Namespace, logger, "default is not empty or is still in use, not removed")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -446,19 +447,19 @@ func (c *cluster) removeDefaultCrushRoot() error {
|
||||
// Those objects may interfere with the normal operation of the cluster.
|
||||
// Note that errors which indicate that the objects are in use are ignored and the objects will continue to exist in that case.
|
||||
func (c *cluster) replaceDefaultCrushMap(newRoot string) (err error) {
|
||||
logger.Info("creating new CRUSH root if it does not exist")
|
||||
log.NamespacedInfo(c.Namespace, logger, "creating new CRUSH root if it does not exist")
|
||||
err = c.createCrushRoot(newRoot)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create CRUSH root")
|
||||
}
|
||||
|
||||
logger.Info("replacing default replicated_rule CRUSH rule for use of non-default CRUSH root")
|
||||
log.NamespacedInfo(c.Namespace, logger, "replacing default replicated_rule CRUSH rule for use of non-default CRUSH root")
|
||||
err = c.replaceDefaultReplicationRule(newRoot)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to replace default rule")
|
||||
}
|
||||
|
||||
logger.Info("replacing default CRUSH node if applicable")
|
||||
log.NamespacedInfo(c.Namespace, logger, "replacing default CRUSH node if applicable")
|
||||
err = c.removeDefaultCrushRoot()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to remove default CRUSH root")
|
||||
@@ -589,12 +590,12 @@ func (c *cluster) configureStorageSettings() error {
|
||||
func (c *cluster) setClusterFullRatio(ratioCommand string, desiredRatio *float64, actualRatio float64) error {
|
||||
if !shouldUpdateFloatSetting(desiredRatio, actualRatio) {
|
||||
if desiredRatio != nil {
|
||||
logger.Infof("desired value %s=%.2f is already set", ratioCommand, *desiredRatio)
|
||||
log.NamespacedInfo(c.Namespace, logger, "desired value %s=%.2f is already set", ratioCommand, *desiredRatio)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
desiredStringVal := fmt.Sprintf("%.2f", *desiredRatio)
|
||||
logger.Infof("updating %s from %.2f to %s", ratioCommand, actualRatio, desiredStringVal)
|
||||
log.NamespacedInfo(c.Namespace, logger, "updating %s from %.2f to %s", ratioCommand, actualRatio, desiredStringVal)
|
||||
args := []string{"osd", ratioCommand, desiredStringVal}
|
||||
cephCmd := client.NewCephCommand(c.context, c.ClusterInfo, args)
|
||||
output, err := cephCmd.Run()
|
||||
@@ -648,7 +649,7 @@ func (c *cluster) reportTelemetry() {
|
||||
}
|
||||
|
||||
func reportClusterTelemetry(c *cluster) {
|
||||
logger.Info("reporting cluster telemetry")
|
||||
log.NamespacedInfo(c.Namespace, logger, "reporting cluster telemetry")
|
||||
|
||||
// Identify this as a rook cluster for Ceph telemetry by setting the Rook version.
|
||||
telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.RookVersionKey, rookversion.Version)
|
||||
@@ -656,7 +657,7 @@ func reportClusterTelemetry(c *cluster) {
|
||||
// Report the K8s version
|
||||
serverVersion, err := c.context.Clientset.Discovery().ServerVersion()
|
||||
if err != nil {
|
||||
logger.Warningf("failed to report the K8s server version. %v", err)
|
||||
log.NamespacedWarning(c.Namespace, logger, "failed to report the K8s server version. %v", err)
|
||||
} else {
|
||||
telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.K8sVersionKey, serverVersion.String())
|
||||
}
|
||||
@@ -699,13 +700,13 @@ func reportClusterTelemetry(c *cluster) {
|
||||
}
|
||||
|
||||
func reportNodeTelemetry(c *cluster) {
|
||||
logger.Info("reporting node telemetry")
|
||||
log.NamespacedInfo(c.Namespace, logger, "reporting node telemetry")
|
||||
operatorNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
|
||||
|
||||
// Report the K8sNodeCount
|
||||
nodelist, err := c.context.Clientset.CoreV1().Nodes().List(c.ClusterInfo.Context, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
logger.Warningf("failed to report the K8s node count. %v", err)
|
||||
log.NamespacedWarning(c.Namespace, logger, "failed to report the K8s node count. %v", err)
|
||||
} else {
|
||||
telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.K8sNodeCount, strconv.Itoa(len(nodelist.Items)))
|
||||
}
|
||||
@@ -717,7 +718,7 @@ func reportNodeTelemetry(c *cluster) {
|
||||
listoption := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, nodedaemon.CrashCollectorAppName)}
|
||||
cephNodeList, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(c.ClusterInfo.Context, listoption)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to report the ceph node count. %v", err)
|
||||
log.NamespacedWarning(c.Namespace, logger, "failed to report the ceph node count. %v", err)
|
||||
} else {
|
||||
telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.CephNodeCount, strconv.Itoa(len(cephNodeList.Items)))
|
||||
}
|
||||
@@ -726,7 +727,7 @@ func reportNodeTelemetry(c *cluster) {
|
||||
listoption := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, csi.CsiRBDPlugin)}
|
||||
cephRbdNodelist, err := c.context.Clientset.CoreV1().Pods(operatorNamespace).List(c.ClusterInfo.Context, listoption)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to report the ceph rbd node count. %v", err)
|
||||
log.NamespacedWarning(c.Namespace, logger, "failed to report the ceph rbd node count. %v", err)
|
||||
} else {
|
||||
telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.RBDNodeCount, strconv.Itoa(len(cephRbdNodelist.Items)))
|
||||
}
|
||||
@@ -735,7 +736,7 @@ func reportNodeTelemetry(c *cluster) {
|
||||
listoption = metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, csi.CsiCephFSPlugin)}
|
||||
cephFSNodelist, err := c.context.Clientset.CoreV1().Pods(operatorNamespace).List(c.ClusterInfo.Context, listoption)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to report the ceph cephfs node count. %v", err)
|
||||
log.NamespacedWarning(c.Namespace, logger, "failed to report the ceph cephfs node count. %v", err)
|
||||
} else {
|
||||
telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.CephFSNodeCount, strconv.Itoa(len(cephFSNodelist.Items)))
|
||||
}
|
||||
@@ -744,7 +745,7 @@ func reportNodeTelemetry(c *cluster) {
|
||||
listoption = metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, csi.CsiNFSPlugin)}
|
||||
cephNFSNodelist, err := c.context.Clientset.CoreV1().Pods(operatorNamespace).List(c.ClusterInfo.Context, listoption)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to report the ceph nfs node count. %v", err)
|
||||
log.NamespacedWarning(c.Namespace, logger, "failed to report the ceph nfs node count. %v", err)
|
||||
} else {
|
||||
telemetry.ReportKeyValue(c.context, c.ClusterInfo, telemetry.NFSNodeCount, strconv.Itoa(len(cephNFSNodelist.Items)))
|
||||
}
|
||||
@@ -764,7 +765,7 @@ func (c *cluster) configureMsgr2() error {
|
||||
encryptionEnabled := c.Spec.NetworkEncryptionEnabled()
|
||||
|
||||
if encryptionEnabled {
|
||||
logger.Infof("setting msgr2 encryption mode to %q", encryptionSetting)
|
||||
log.NamespacedInfo(c.Namespace, logger, "setting msgr2 encryption mode to %q", encryptionSetting)
|
||||
if err := monStore.SetAll("global", encryptionGlobalConfigSettings); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -797,7 +798,7 @@ func (c *cluster) configureMsgr2() error {
|
||||
globalConfigSettings := map[string]string{
|
||||
"ms_osd_compress_mode": "force",
|
||||
}
|
||||
logger.Infof("setting msgr2 compression mode to %q", "force")
|
||||
log.NamespacedInfo(c.Namespace, logger, "setting msgr2 compression mode to %q", "force")
|
||||
if err := monStore.SetAll("global", globalConfigSettings); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -819,9 +820,9 @@ func (c *cluster) fetchCephConfigFromSecrets() (map[string]map[string]string, er
|
||||
key, module, selector.LocalObjectReference.Name, err)
|
||||
}
|
||||
|
||||
logger.Debugf("setting Ceph config key %q in module %q from secret %q",
|
||||
log.NamespacedDebug(c.Namespace, logger, "setting Ceph config key %q in module %q from secret %q",
|
||||
key, module, selector.LocalObjectReference.Name)
|
||||
logger.Tracef("setting Ceph config key %q in module %q to value %q from secret %q",
|
||||
log.NamespacedTrace(c.Namespace, logger, "setting Ceph config key %q in module %q to value %q from secret %q",
|
||||
key, module, val, selector.LocalObjectReference.Name)
|
||||
result[module][key] = val
|
||||
}
|
||||
@@ -851,7 +852,7 @@ func (c *cluster) fetchSecretValue(selector v1.SecretKeySelector) (string, error
|
||||
func initClusterCephxStatus(c *cluster) error {
|
||||
initErr := c.ClusterInfo.IsInitialized()
|
||||
if initErr == nil {
|
||||
logger.Debugf("not setting uninitialized cephx status on already initialized CephCluster in namespace %q", c.Namespace)
|
||||
log.NamespacedDebug(c.Namespace, logger, "not setting uninitialized cephx status on already initialized CephCluster")
|
||||
return nil
|
||||
}
|
||||
if c.ClusterInfo.Context.Err() != nil {
|
||||
@@ -874,7 +875,7 @@ func initClusterCephxStatus(c *cluster) error {
|
||||
}
|
||||
|
||||
uninitializedStatus := keyring.UninitializedCephxStatus()
|
||||
logger.Infof("initializing cephx status for CephCluster in namespace %q", c.Namespace)
|
||||
log.NamespacedInfo(c.Namespace, logger, "initializing cephx status")
|
||||
clusterObj.Status.Cephx = cephv1.ClusterCephxStatus{
|
||||
Mon: uninitializedStatus,
|
||||
Mgr: uninitializedStatus,
|
||||
@@ -901,9 +902,9 @@ func (c *cluster) deleteBootstrapKeys() {
|
||||
for _, bootstrapkey := range bootstrapKeys {
|
||||
err := client.AuthDelete(c.context, c.ClusterInfo, bootstrapkey)
|
||||
if err != nil {
|
||||
logger.Debugf("failed to delete bootstrap key %q in the namespace %q. Error: %v", bootstrapkey, c.Namespace, err)
|
||||
log.NamespacedDebug(c.Namespace, logger, "failed to delete bootstrap key %q. Error: %v", bootstrapkey, err)
|
||||
} else {
|
||||
logger.Infof("successfully deleted bootstrap key %q in the namespace %q", bootstrapkey, c.Namespace)
|
||||
log.NamespacedInfo(c.Namespace, logger, "successfully deleted bootstrap key %q", bootstrapkey)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,7 @@ import (
|
||||
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/ceph/csi"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -65,7 +66,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
|
||||
if cluster.Spec.CephVersion.Image == "" {
|
||||
err = mon.WriteConnectionConfig(c.context, cluster.ClusterInfo)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to write config. attempting to continue. %v", err)
|
||||
log.NamespacedError(cluster.Namespace, logger, "failed to write config. attempting to continue. %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,13 +82,13 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
|
||||
// If we don't do this, daemons will never start, waiting forever for this configmap to be present
|
||||
//
|
||||
// Only do this when doing a bit of management...
|
||||
logger.Infof("creating %q configmap", k8sutil.ConfigOverrideName)
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "creating %q configmap", k8sutil.ConfigOverrideName)
|
||||
err = populateConfigOverrideConfigMap(c.context, cluster.namespacedName.Namespace, cluster.ClusterInfo.OwnerInfo, cluster.clusterMetadata)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to populate config override config map")
|
||||
}
|
||||
|
||||
logger.Infof("creating %q secret", config.StoreName)
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "creating %q secret", config.StoreName)
|
||||
err = config.GetStore(c.context, cluster.namespacedName.Namespace, cluster.ClusterInfo.OwnerInfo).CreateOrUpdate(cluster.ClusterInfo)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to update the global config")
|
||||
@@ -98,7 +99,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
|
||||
if err := cluster.ClusterInfo.IsInitialized(); err != nil {
|
||||
return errors.Wrap(err, "the cluster identity was not established")
|
||||
}
|
||||
logger.Info("external cluster identity established")
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "external cluster identity established")
|
||||
|
||||
// Create CSI Secrets only if the user has provided the admin key
|
||||
if cluster.ClusterInfo.CephCred.Username == client.AdminUsername {
|
||||
@@ -117,7 +118,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
|
||||
cluster.Spec.Network.Connections = &cephv1.ConnectionsSpec{}
|
||||
}
|
||||
cluster.Spec.Network.Connections.RequireMsgr2 = true
|
||||
logger.Debugf("a v2 port was found for a mon endpoint, so msgr2 is required")
|
||||
log.NamespacedDebug(cluster.Namespace, logger, "a v2 port was found for a mon endpoint, so msgr2 is required")
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -136,7 +137,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to update csi cluster config")
|
||||
}
|
||||
logger.Info("successfully updated csi config map")
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "successfully updated csi config map")
|
||||
|
||||
// Create Crash Collector Secret
|
||||
if !cluster.Spec.CrashCollector.Disable {
|
||||
@@ -177,7 +178,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
|
||||
}
|
||||
|
||||
if csi.EnableCSIOperator() {
|
||||
logger.Info("create cephConnection and defaultClientProfile for external mode")
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "create cephConnection and defaultClientProfile for external mode")
|
||||
err = csi.CreateUpdateCephConnection(c.context.Client, cluster.ClusterInfo, *cluster.Spec)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create/update cephConnection")
|
||||
@@ -202,7 +203,7 @@ func purgeExternalCluster(clientset kubernetes.Interface, namespace string) {
|
||||
for _, cm := range cmsToDelete {
|
||||
err := clientset.CoreV1().ConfigMaps(namespace).Delete(ctx, cm, metav1.DeleteOptions{})
|
||||
if err != nil && !kerrors.IsNotFound(err) {
|
||||
logger.Errorf("failed to delete config map %q. %v", cm, err)
|
||||
log.NamespacedError(namespace, logger, "failed to delete config map %q. %v", cm, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -219,7 +220,7 @@ func purgeExternalCluster(clientset kubernetes.Interface, namespace string) {
|
||||
for _, secret := range secretsToDelete {
|
||||
err := clientset.CoreV1().Secrets(namespace).Delete(ctx, secret, metav1.DeleteOptions{})
|
||||
if err != nil && !kerrors.IsNotFound(err) {
|
||||
logger.Errorf("failed to delete secret %q. %v", secret, err)
|
||||
log.NamespacedError(namespace, logger, "failed to delete secret %q. %v", secret, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -255,12 +256,12 @@ func (c *ClusterController) configureExternalClusterMonitoring(context *clusterd
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Info("creating mgr external monitoring service")
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "creating mgr external monitoring service")
|
||||
_, err = k8sutil.CreateOrUpdateService(cluster.ClusterInfo.Context, context.Clientset, cluster.Namespace, service)
|
||||
if err != nil && !kerrors.IsAlreadyExists(err) {
|
||||
return errors.Wrap(err, "failed to create or update mgr service")
|
||||
}
|
||||
logger.Info("mgr external metrics service created")
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "mgr external metrics service created")
|
||||
|
||||
// Configure external metrics endpoint
|
||||
err = opcontroller.ConfigureExternalMetricsEndpoint(context, cluster.Spec.Monitoring, cluster.ClusterInfo, cluster.ownerInfo)
|
||||
@@ -269,13 +270,13 @@ func (c *ClusterController) configureExternalClusterMonitoring(context *clusterd
|
||||
}
|
||||
|
||||
// Deploy external ServiceMonitor
|
||||
logger.Info("creating external service monitor")
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "creating external service monitor")
|
||||
// servicemonitor takes some metadata from the service for easy mapping
|
||||
err = manager.EnableServiceMonitor()
|
||||
if err != nil {
|
||||
logger.Errorf("failed to enable external service monitor. %v", err)
|
||||
log.NamespacedError(cluster.Namespace, logger, "failed to enable external service monitor. %v", err)
|
||||
} else {
|
||||
logger.Info("external service monitor created")
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "external service monitor created")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -39,6 +39,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/csi"
|
||||
"github.com/rook/rook/pkg/operator/ceph/reporting"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
@@ -284,7 +285,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
|
||||
err := r.client.Get(r.opManagerContext, request.NamespacedName, cephCluster)
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Debug("cephCluster resource not found. Ignoring since object must be deleted.")
|
||||
log.NamespacedDebug(request.Namespace, logger, "cephCluster resource not found. Ignoring since object must be deleted.")
|
||||
return reconcile.Result{}, *cephCluster, nil
|
||||
}
|
||||
// Error reading the object - requeue the request.
|
||||
@@ -297,7 +298,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
|
||||
return reconcile.Result{}, *cephCluster, errors.Wrap(err, "failed to add finalizer")
|
||||
}
|
||||
if generationUpdated {
|
||||
logger.Infof("reconciling the ceph cluster %q after adding finalizer", cephCluster.Name)
|
||||
log.NamespacedInfo(request.Namespace, logger, "reconciling the ceph cluster %q after adding finalizer", cephCluster.Name)
|
||||
return reconcile.Result{}, *cephCluster, nil
|
||||
}
|
||||
|
||||
@@ -317,7 +318,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
|
||||
// started). So we would get messages from the other go routine printing a message
|
||||
// repeatedly that the context is cancelled.
|
||||
if errors.Is(r.opManagerContext.Err(), context.Canceled) {
|
||||
logger.Info("context cancelled, exiting reconcile")
|
||||
log.NamespacedInfo(request.Namespace, logger, "context cancelled, exiting reconcile")
|
||||
return reconcile.Result{}, *cephCluster, nil
|
||||
}
|
||||
|
||||
@@ -357,7 +358,7 @@ func (r *ReconcileCephCluster) reconcileDelete(cephCluster *cephv1.CephCluster)
|
||||
if cephCluster.Spec.CleanupPolicy.HasDataDirCleanPolicy() && !cephCluster.Spec.External.Enable {
|
||||
monSecret, clusterFSID, err := r.clusterController.getCleanUpDetails(&cephCluster.Spec, cephCluster.Namespace)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to get mon secret. skip cluster cleanup. remove finalizer. %v", err)
|
||||
log.NamespacedWarning(cephCluster.Namespace, logger, "failed to get mon secret. skip cluster cleanup. remove finalizer. %v", err)
|
||||
doCleanup = false
|
||||
}
|
||||
|
||||
@@ -400,7 +401,7 @@ func NewClusterController(context *clusterd.Context, rookImage string) *ClusterC
|
||||
|
||||
func (c *ClusterController) reconcileCephCluster(clusterObj *cephv1.CephCluster, ownerInfo *k8sutil.OwnerInfo) error {
|
||||
if clusterObj.Spec.CleanupPolicy.HasDataDirCleanPolicy() {
|
||||
logger.Infof("skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set", clusterObj.Name, clusterObj.Namespace)
|
||||
log.NamespacedInfo(clusterObj.Namespace, logger, "skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set", clusterObj.Name, clusterObj.Namespace)
|
||||
return nil
|
||||
}
|
||||
var clustr *cluster
|
||||
@@ -425,7 +426,7 @@ func (c *ClusterController) reconcileCephCluster(clusterObj *cephv1.CephCluster,
|
||||
clustr.Spec = &clusterObj.Spec
|
||||
|
||||
c.clusterMap.Store(clustr.Namespace, clustr)
|
||||
logger.Infof("reconciling ceph cluster in namespace %q", clustr.namespacedName.Namespace)
|
||||
log.NamedInfo(clustr.namespacedName, logger, "reconciling ceph cluster")
|
||||
|
||||
// Start the main ceph cluster orchestration
|
||||
return c.initializeCluster(clustr)
|
||||
@@ -438,14 +439,14 @@ func (c *ClusterController) requestClusterDelete(clusterObj *cephv1.CephCluster)
|
||||
if rawCluster, ok := c.clusterMap.Load(clusterObj.Namespace); ok {
|
||||
existingCluster = rawCluster.(*cluster)
|
||||
if existingCluster.namespacedName.Name != clusterObj.Name {
|
||||
logger.Errorf("skipping deletion of CephCluster %q. CephCluster CR %q already exists in this namespace. only one cluster cr per namespace is supported.",
|
||||
log.NamespacedError(nsName, logger, "skipping deletion of CephCluster %q. CephCluster CR %q already exists in this namespace. only one cluster cr per namespace is supported.",
|
||||
nsName, existingCluster.namespacedName.Name)
|
||||
return reconcile.Result{}, nil // do not requeue the delete
|
||||
}
|
||||
}
|
||||
_, err := c.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(c.OpManagerCtx, "networkfences.csiaddons.openshift.io", metav1.GetOptions{})
|
||||
if err == nil {
|
||||
logger.Info("removing networkFence if matching cephCluster UID found")
|
||||
log.NamespacedInfo(clusterObj.Namespace, logger, "removing networkFence if matching cephCluster UID found")
|
||||
networkFenceList := &addonsv1alpha1.NetworkFenceList{}
|
||||
labelSelector := labels.SelectorFromSet(map[string]string{
|
||||
networkFenceLabel: string(clusterObj.GetUID()),
|
||||
@@ -476,7 +477,7 @@ func (c *ClusterController) requestClusterDelete(clusterObj *cephv1.CephCluster)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("cleaning up CephCluster %q", nsName)
|
||||
log.NamespacedInfo(clusterObj.Namespace, logger, "cleaning up CephCluster %q", nsName)
|
||||
if existingCluster != nil {
|
||||
// We used to stop the bucket controller here but when we get a DELETE event for the CephCluster
|
||||
// we will reload the CRD manager anyway so the bucket controller go routine will be stopped
|
||||
@@ -495,7 +496,7 @@ func (c *ClusterController) requestClusterDelete(clusterObj *cephv1.CephCluster)
|
||||
}
|
||||
|
||||
if clusterObj.Spec.CleanupPolicy.AllowUninstallWithVolumes {
|
||||
logger.Info("skipping check for existing PVs as allowUninstallWithVolumes is set to true")
|
||||
log.NamespacedInfo(clusterObj.Namespace, logger, "skipping check for existing PVs as allowUninstallWithVolumes is set to true")
|
||||
} else {
|
||||
err := c.checkIfVolumesExist(clusterObj)
|
||||
if err != nil {
|
||||
@@ -510,7 +511,7 @@ func (c *ClusterController) requestClusterDelete(clusterObj *cephv1.CephCluster)
|
||||
// Delete keys from KMS
|
||||
err := c.deleteOSDEncryptionKeyFromKMS(clusterObj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to delete osd encryption keys for CephCluster %q from kms; deletion will continue. %v", nsName, err)
|
||||
log.NamespacedError(clusterObj.Namespace, logger, "failed to delete osd encryption keys for CephCluster %q from kms; deletion will continue. %v", nsName, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -534,7 +535,7 @@ func (c *ClusterController) checkIfVolumesExist(cluster *cephv1.CephCluster) err
|
||||
func (c *ClusterController) csiVolumesAllowForDeletion(cluster *cephv1.CephCluster) error {
|
||||
drivers := []string{csi.CephFSDriverName, csi.RBDDriverName}
|
||||
|
||||
logger.Infof("checking any PVC created by drivers %q and %q with clusterID %q", csi.CephFSDriverName, csi.RBDDriverName, cluster.Namespace)
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "checking any PVC created by drivers %q and %q with clusterID %q", csi.CephFSDriverName, csi.RBDDriverName, cluster.Namespace)
|
||||
// check any PV is created in this cluster
|
||||
attachmentsExist, err := c.checkPVPresentInCluster(drivers, cluster.Namespace)
|
||||
if err != nil {
|
||||
@@ -542,7 +543,7 @@ func (c *ClusterController) csiVolumesAllowForDeletion(cluster *cephv1.CephClust
|
||||
}
|
||||
// no PVC created in this cluster
|
||||
if !attachmentsExist {
|
||||
logger.Infof("no volume attachments for cluster %q", cluster.Namespace)
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "no volume attachments for cluster %q", cluster.Namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -557,7 +558,7 @@ func (c *ClusterController) checkPVPresentInCluster(drivers []string, clusterID
|
||||
|
||||
for _, p := range pv.Items {
|
||||
if p.Spec.CSI == nil {
|
||||
logger.Errorf("Spec.CSI is nil for PV %q", p.Name)
|
||||
log.NamespacedError(clusterID, logger, "Spec.CSI is nil for PV %q", p.Name)
|
||||
continue
|
||||
}
|
||||
if p.Spec.CSI.VolumeAttributes["clusterID"] == clusterID {
|
||||
@@ -600,7 +601,7 @@ func (r *ReconcileCephCluster) removeFinalizer(client client.Client, name types.
|
||||
err := client.Get(r.opManagerContext, name, obj)
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Debugf("%s resource not found. Ignoring since object must be deleted.", name.Name)
|
||||
log.NamedDebug(name, logger, "%s resource not found. Ignoring since object must be deleted.", name.Name)
|
||||
return nil
|
||||
}
|
||||
return errors.Wrapf(err, "failed to retrieve %q to remove finalizer", name.Name)
|
||||
@@ -666,7 +667,7 @@ func (c *ClusterController) deleteOSDEncryptionKeyFromKMS(currentCluster *cephv1
|
||||
// Generate and store the encrypted key in whatever KMS is configured
|
||||
err = kmsConfig.DeleteSecret(osdPVC.Name)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to delete secret. %v", err)
|
||||
log.NamespacedError(currentCluster.Namespace, logger, "failed to delete secret. %v", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
@@ -89,7 +90,7 @@ func (c *Cluster) generateKeyring(m *mgrConfig) (string, error) {
|
||||
}
|
||||
|
||||
if c.shouldRotateCephxKeys {
|
||||
logger.Infof("rotating cephx key for mgr daemon %q in the namespace %q", m.ResourceName, c.clusterInfo.Namespace)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "rotating cephx key for mgr daemon %q in the namespace %q", m.ResourceName, c.clusterInfo.Namespace)
|
||||
newKey, err := s.RotateKey(user)
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "failed to rotate cephx key for mgr daemon %q", m.ResourceName)
|
||||
@@ -101,9 +102,9 @@ func (c *Cluster) generateKeyring(m *mgrConfig) (string, error) {
|
||||
err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Delete(c.clusterInfo.Context, m.ResourceName, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Debugf("legacy mgr key %q is already removed", m.ResourceName)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "legacy mgr key %q is already removed", m.ResourceName)
|
||||
} else {
|
||||
logger.Warningf("legacy mgr key %q could not be removed. %v", m.ResourceName, err)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "legacy mgr key %q could not be removed. %v", m.ResourceName, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util"
|
||||
"github.com/rook/rook/pkg/util/exec"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -91,7 +92,7 @@ func (c *Cluster) configureDashboardModules() error {
|
||||
}
|
||||
} else {
|
||||
if err := client.MgrDisableModule(c.context, c.clusterInfo, dashboardModuleName); err != nil {
|
||||
logger.Errorf("failed to disable mgr dashboard module. %v", err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to disable mgr dashboard module. %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -106,7 +107,7 @@ func (c *Cluster) configureDashboardModules() error {
|
||||
return err
|
||||
}
|
||||
if secureRequiresRestart || configChanged {
|
||||
logger.Info("dashboard config has changed. restarting the dashboard module")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "dashboard config has changed. restarting the dashboard module")
|
||||
return c.restartMgrModule(dashboardModuleName)
|
||||
}
|
||||
return nil
|
||||
@@ -131,16 +132,16 @@ func (c *Cluster) deleteManagerDaemonConfiguration() bool {
|
||||
for _, key := range mgrKeysToDelete {
|
||||
err := monStore.Delete(mgrDaemonID, key)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to delete configuration entry %q %q, err: %v", mgrDaemonID, key, err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to delete configuration entry %q %q, err: %v", mgrDaemonID, key, err)
|
||||
success = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if success {
|
||||
logger.Info("All per-daemon mgr configuration has been deleted successfully.")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "All per-daemon mgr configuration has been deleted successfully.")
|
||||
} else {
|
||||
logger.Error("At least one delete operation failed while trying to delete per-daemon mgr configuration.")
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "At least one delete operation failed while trying to delete per-daemon mgr configuration.")
|
||||
}
|
||||
|
||||
return success
|
||||
@@ -236,10 +237,10 @@ func (c *Cluster) createSelfSignedCert() (bool, error) {
|
||||
args := []string{"config-key", "get", "mgr/dashboard/crt"}
|
||||
output, err := client.NewCephCommand(c.context, c.clusterInfo, args).RunWithTimeout(exec.CephCommandsTimeout)
|
||||
if err == nil && len(output) > 0 {
|
||||
logger.Info("dashboard is already initialized with a cert")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "dashboard is already initialized with a cert")
|
||||
return true, nil
|
||||
}
|
||||
logger.Debugf("dashboard cert does not appear to exist. err=%v", err)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "dashboard cert does not appear to exist. err=%v", err)
|
||||
|
||||
// create a self-signed cert for the https connections
|
||||
args = []string{"dashboard", "create-self-signed-cert"}
|
||||
@@ -248,14 +249,14 @@ func (c *Cluster) createSelfSignedCert() (bool, error) {
|
||||
for i := 0; i < 5; i++ {
|
||||
_, err := client.NewCephCommand(c.context, c.clusterInfo, args).RunWithTimeout(exec.CephCommandsTimeout)
|
||||
if err == context.DeadlineExceeded {
|
||||
logger.Warning("cert creation timed out. trying again")
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "cert creation timed out. trying again")
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
exitCode, parsed := c.exitCode(err)
|
||||
if parsed {
|
||||
if exitCode == invalidArgErrorCode {
|
||||
logger.Info("dashboard module is not ready yet. trying again")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "dashboard module is not ready yet. trying again")
|
||||
time.Sleep(dashboardInitWaitTime)
|
||||
continue
|
||||
}
|
||||
@@ -263,16 +264,16 @@ func (c *Cluster) createSelfSignedCert() (bool, error) {
|
||||
return false, errors.Wrap(err, "failed to create self signed cert on mgr")
|
||||
}
|
||||
}
|
||||
logger.Info("dashboard cert created")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "dashboard cert created")
|
||||
return false, nil
|
||||
}
|
||||
logger.Info("dashboard cert creation exceeded retries")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "dashboard cert creation exceeded retries")
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) setLoginCredentials(password string) error {
|
||||
// Set the login credentials. Write the command/args to the debug log so we don't write the password by default to the log.
|
||||
logger.Infof("setting ceph dashboard %q login creds", dashboardUsername)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "setting ceph dashboard %q login creds", dashboardUsername)
|
||||
|
||||
var args []string
|
||||
// for latest Ceph versions
|
||||
@@ -284,7 +285,7 @@ func (c *Cluster) setLoginCredentials(password string) error {
|
||||
|
||||
defer func() {
|
||||
if err := os.Remove(file.Name()); err != nil {
|
||||
logger.Errorf("failed to clean up dashboard password file %q. %v", file.Name(), err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to clean up dashboard password file %q. %v", file.Name(), err)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -314,14 +315,14 @@ func (c *Cluster) setLoginCredentials(password string) error {
|
||||
return errors.Wrapf(err, "failed to set password for user %q", dashboardUsername)
|
||||
}
|
||||
|
||||
logger.Info("successfully set ceph dashboard creds")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "successfully set ceph dashboard creds")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) getOrGenerateDashboardPassword() (string, error) {
|
||||
secret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(c.clusterInfo.Context, dashboardPasswordName, metav1.GetOptions{})
|
||||
if err == nil {
|
||||
logger.Info("the dashboard secret was already generated")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "the dashboard secret was already generated")
|
||||
return decodeSecret(secret)
|
||||
}
|
||||
if !kerrors.IsNotFound(err) {
|
||||
|
||||
@@ -19,6 +19,7 @@ package mgr
|
||||
import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
policyv1 "k8s.io/api/policy/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -63,13 +64,13 @@ func (c *Cluster) deleteMgrPDB() {
|
||||
err := c.context.Client.Get(c.clusterInfo.Context, pdbRequest, mgrPDB)
|
||||
if err != nil {
|
||||
if !kerrors.IsNotFound(err) {
|
||||
logger.Errorf("failed to get mgr pdb %q. %v", mgrPDBName, err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get mgr pdb %q. %v", mgrPDBName, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
logger.Debugf("ensuring the mgr pdb %q is deleted", mgrPDBName)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "ensuring the mgr pdb %q is deleted", mgrPDBName)
|
||||
err = c.context.Client.Delete(c.clusterInfo.Context, mgrPDB)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to delete mgr pdb %q. %v", mgrPDBName, err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to delete mgr pdb %q. %v", mgrPDBName, err)
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,7 @@ import (
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/exec"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
v1 "k8s.io/api/apps/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -115,7 +116,7 @@ func (c *Cluster) Start() error {
|
||||
return errors.Wrap(err, "error checking pod memory")
|
||||
}
|
||||
|
||||
logger.Infof("start running mgr")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "start running mgr")
|
||||
daemonIDs := c.getDaemonIDs()
|
||||
var deploymentsToWaitFor []*v1.Deployment
|
||||
|
||||
@@ -130,7 +131,7 @@ func (c *Cluster) Start() error {
|
||||
}
|
||||
|
||||
if c.shouldRotateCephxKeys {
|
||||
logger.Infof("cephx keys for mgr daemons in the namespace %q will be rotated", c.clusterInfo.Namespace)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "cephx keys for mgr daemons in the namespace %q will be rotated", c.clusterInfo.Namespace)
|
||||
}
|
||||
|
||||
for _, daemonID := range daemonIDs {
|
||||
@@ -167,7 +168,7 @@ func (c *Cluster) Start() error {
|
||||
}
|
||||
|
||||
if mgrsToSkipReconcile.Has(daemonID) {
|
||||
logger.Warningf("Skipping reconcile of mgr %q labeled with %q", daemonID, cephv1.SkipReconcileLabelKey)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "Skipping reconcile of mgr %q labeled with %q", daemonID, cephv1.SkipReconcileLabelKey)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -176,13 +177,13 @@ func (c *Cluster) Start() error {
|
||||
if !kerrors.IsAlreadyExists(err) {
|
||||
return errors.Wrapf(err, "failed to create mgr deployment %s", resourceName)
|
||||
}
|
||||
logger.Infof("deployment for mgr %s already exists. updating if needed", resourceName)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "deployment for mgr %s already exists. updating if needed", resourceName)
|
||||
if err := updateDeploymentAndWait(c.context, c.clusterInfo, d, config.MgrType, mgrConfig.DaemonID, c.spec.SkipUpgradeChecks, c.spec.ContinueUpgradeAfterChecksEvenIfNotHealthy); err != nil {
|
||||
logger.Errorf("failed to update mgr deployment %q. %v", resourceName, err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to update mgr deployment %q. %v", resourceName, err)
|
||||
if c.spec.ContinueUpgradeAfterChecksEvenIfNotHealthy {
|
||||
logger.Infof("continuing reconcile of ceph cluster in namespace %s after error waiting for mgr because continueUpgradeAfterChecksEvenIfNotHealthy is true", c.clusterInfo.Namespace)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "continuing reconcile of ceph cluster in namespace %s after error waiting for mgr because continueUpgradeAfterChecksEvenIfNotHealthy is true", c.clusterInfo.Namespace)
|
||||
} else {
|
||||
logger.Infof("stopping reconcile of ceph cluster in namespace %s after error waiting for mgr because continueUpgradeAfterChecksEvenIfNotHealthy is false", c.clusterInfo.Namespace)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "stopping reconcile of ceph cluster in namespace %s after error waiting for mgr because continueUpgradeAfterChecksEvenIfNotHealthy is false", c.clusterInfo.Namespace)
|
||||
return errors.Wrapf(err, "failed to update mgr deployment %q", resourceName)
|
||||
}
|
||||
}
|
||||
@@ -237,11 +238,11 @@ func (c *Cluster) removeExtraMgrs(daemonIDs []string) {
|
||||
options := metav1.ListOptions{LabelSelector: "app=" + AppName}
|
||||
mgrDeployments, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(c.clusterInfo.Context, options)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to check for extra mgrs. %v", err)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to check for extra mgrs. %v", err)
|
||||
return
|
||||
}
|
||||
if len(mgrDeployments.Items) == len(daemonIDs) {
|
||||
logger.Debugf("expected number %d of mgrs found", len(daemonIDs))
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "expected number %d of mgrs found", len(daemonIDs))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -263,9 +264,9 @@ func (c *Cluster) removeExtraMgrs(daemonIDs []string) {
|
||||
if !found {
|
||||
err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(c.clusterInfo.Context, mgrDeployment.Name, metav1.DeleteOptions{})
|
||||
if err == nil {
|
||||
logger.Infof("removed extra mgr %q", mgrDeployment.Name)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "removed extra mgr %q", mgrDeployment.Name)
|
||||
} else {
|
||||
logger.Warningf("failed to remove extra mgr %q. %v", mgrDeployment.Name, err)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to remove extra mgr %q. %v", mgrDeployment.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -278,7 +279,7 @@ func (c *Cluster) SetMgrRoleLabel(daemonNameToUpdate string, isActive bool) erro
|
||||
LabelSelector: fmt.Sprintf("%s=%s,%s=%s", k8sutil.AppAttr, AppName, controller.DaemonIDLabel, daemonNameToUpdate),
|
||||
})
|
||||
if err != nil {
|
||||
logger.Infof("cannot get pod for mgr daemon %s", daemonNameToUpdate)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "cannot get pod for mgr daemon %s", daemonNameToUpdate)
|
||||
return err // force mrg_role update in the next call
|
||||
}
|
||||
|
||||
@@ -295,7 +296,7 @@ func (c *Cluster) SetMgrRoleLabel(daemonNameToUpdate string, isActive bool) erro
|
||||
labels := pod.GetLabels()
|
||||
currMgrRole, mgrHasLabel := labels[mgrRoleLabelName]
|
||||
if !mgrHasLabel || currMgrRole != newMgrRole {
|
||||
logger.Infof("updating mgr_role label value of daemon %s to '%s'. New active mgr is %s.", daemonNameToUpdate, newMgrRole, daemonNameToUpdate)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "updating mgr_role label value of daemon %s to '%s'. New active mgr is %s.", daemonNameToUpdate, newMgrRole, daemonNameToUpdate)
|
||||
labels[mgrRoleLabelName] = newMgrRole
|
||||
pod.SetLabels(labels)
|
||||
_, err = c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).Update(c.clusterInfo.Context, &pods.Items[i], metav1.UpdateOptions{})
|
||||
@@ -346,7 +347,7 @@ func (c *Cluster) reconcileServices() error {
|
||||
if err := c.EnableServiceMonitor(); err != nil {
|
||||
// We don't want to return an error to block the cluster reconcile
|
||||
// since monitoring is an optional service.
|
||||
logger.Errorf("failed to enable service monitor, prometheus may need to be installed. %v", err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to enable service monitor, prometheus may need to be installed. %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -361,7 +362,7 @@ func (c *Cluster) updateServiceSelectors() {
|
||||
selector := metav1.ListOptions{LabelSelector: "app=rook-ceph-mgr"}
|
||||
services, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).List(c.clusterInfo.Context, selector)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to query mgr services to update labels: %v", err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to query mgr services to update labels: %v", err)
|
||||
return
|
||||
}
|
||||
for i, service := range services.Items {
|
||||
@@ -374,7 +375,7 @@ func (c *Cluster) updateServiceSelectors() {
|
||||
// Check if the service has a DaemonIDLabel (legacy mgr HA implementation) and remove it
|
||||
_, hasDaemonLabel := service.Spec.Selector[controller.DaemonIDLabel]
|
||||
if hasDaemonLabel {
|
||||
logger.Infof("removing %s selector label on mgr service %q", controller.DaemonIDLabel, service.Name)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "removing %s selector label on mgr service %q", controller.DaemonIDLabel, service.Name)
|
||||
delete(service.Spec.Selector, controller.DaemonIDLabel)
|
||||
updateService = true
|
||||
}
|
||||
@@ -382,16 +383,16 @@ func (c *Cluster) updateServiceSelectors() {
|
||||
// In case the service doesn't have the new mgr_role DaemonIDLabel we add it
|
||||
_, hasMgrRoleLabel := service.Spec.Selector[mgrRoleLabelName]
|
||||
if !hasMgrRoleLabel {
|
||||
logger.Infof("adding %s selector label on mgr service %q", mgrRoleLabelName, service.Name)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "adding %s selector label on mgr service %q", mgrRoleLabelName, service.Name)
|
||||
service.Spec.Selector[mgrRoleLabelName] = activeMgrStatus
|
||||
updateService = true
|
||||
}
|
||||
|
||||
if updateService {
|
||||
if _, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Update(c.clusterInfo.Context, &services.Items[i], metav1.UpdateOptions{}); err != nil {
|
||||
logger.Errorf("failed to update service %q. %v", service.Name, err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to update service %q. %v", service.Name, err)
|
||||
} else {
|
||||
logger.Infof("service %q successfully updated", service.Name)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "service %q successfully updated", service.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -399,22 +400,22 @@ func (c *Cluster) updateServiceSelectors() {
|
||||
|
||||
func (c *Cluster) configureModules(daemonIDs []string) {
|
||||
// Configure the modules asynchronously so we can complete all the configuration much sooner.
|
||||
startModuleConfiguration("prometheus", c.configurePrometheusModule)
|
||||
startModuleConfiguration("dashboard", c.configureDashboardModules)
|
||||
c.startModuleConfiguration("prometheus", c.configurePrometheusModule)
|
||||
c.startModuleConfiguration("dashboard", c.configureDashboardModules)
|
||||
|
||||
// It is a bit confusing but modules that are in the "always_on_modules" list
|
||||
// are "just" enabled, but still they must be configured to work properly
|
||||
startModuleConfiguration("balancer", c.enableBalancerModule)
|
||||
startModuleConfiguration("mgr module(s) from the spec", c.configureMgrModules)
|
||||
c.startModuleConfiguration("balancer", c.enableBalancerModule)
|
||||
c.startModuleConfiguration("mgr module(s) from the spec", c.configureMgrModules)
|
||||
}
|
||||
|
||||
func startModuleConfiguration(description string, configureModules func() error) {
|
||||
func (c *Cluster) startModuleConfiguration(description string, configureModules func() error) {
|
||||
go func() {
|
||||
err := configureModules()
|
||||
if err != nil {
|
||||
logger.Errorf("failed modules: %q. %v", description, err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed modules: %q. %v", description, err)
|
||||
} else {
|
||||
logger.Infof("successful modules: %s", description)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "successful modules: %s", description)
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -427,7 +428,7 @@ func (c *Cluster) configurePrometheusModule() error {
|
||||
}
|
||||
} else {
|
||||
if err := cephclient.MgrDisableModule(c.context, c.clusterInfo, PrometheusModuleName); err != nil {
|
||||
logger.Errorf("failed to disable mgr prometheus module. %v", err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to disable mgr prometheus module. %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -446,7 +447,7 @@ func (c *Cluster) configurePrometheusModule() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Infof("prometheus config will change, port: %s", port)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "prometheus config will change, port: %s", port)
|
||||
}
|
||||
// scrape interval
|
||||
if c.spec.Monitoring.Interval != nil {
|
||||
@@ -455,18 +456,18 @@ func (c *Cluster) configurePrometheusModule() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Infof("prometheus config will change, interval: %v", interval)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "prometheus config will change, interval: %v", interval)
|
||||
}
|
||||
|
||||
if portHasChanged || intervalHasChanged {
|
||||
logger.Info("prometheus config has changed. restarting the prometheus module")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "prometheus config has changed. restarting the prometheus module")
|
||||
return c.restartMgrModule(PrometheusModuleName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) restartMgrModule(name string) error {
|
||||
logger.Infof("restarting the mgr module: %s", name)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "restarting the mgr module: %s", name)
|
||||
if err := cephclient.MgrDisableModule(c.context, c.clusterInfo, name); err != nil {
|
||||
return errors.Wrapf(err, "failed to disable mgr module %q.", name)
|
||||
}
|
||||
@@ -520,7 +521,7 @@ func (c *Cluster) configureMgrModules() error {
|
||||
// Configure special settings for individual modules that are enabled
|
||||
switch module.Name {
|
||||
case rookModuleName:
|
||||
startModuleConfiguration("orchestrator modules", c.configureOrchestratorModules)
|
||||
c.startModuleConfiguration("orchestrator modules", c.configureOrchestratorModules)
|
||||
}
|
||||
|
||||
} else {
|
||||
@@ -605,10 +606,10 @@ func applyMonitoringLabels(c *Cluster, serviceMonitor *monitoringv1.ServiceMonit
|
||||
serviceMonitor.Spec.Endpoints[0].RelabelConfigs = append(
|
||||
serviceMonitor.Spec.Endpoints[0].RelabelConfigs, relabelConfig)
|
||||
} else {
|
||||
logger.Info("rook.io/managedBy not specified in monitoring labels")
|
||||
log.NamespacedInfo(serviceMonitor.Namespace, logger, "rook.io/managedBy not specified in monitoring labels")
|
||||
}
|
||||
} else {
|
||||
logger.Debug("monitoring labels not specified")
|
||||
log.NamespacedDebug(serviceMonitor.Namespace, logger, "monitoring labels not specified")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -638,11 +639,11 @@ func updateMgrCephxStatus(c *clusterd.Context, clusterInfo *cephclient.ClusterIn
|
||||
}
|
||||
updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, clusterInfo.CephVersion, cluster.Status.Cephx.Mgr)
|
||||
cluster.Status.Cephx.Mgr = updatedStatus
|
||||
logger.Debugf("updating mgr daemon cephx status to %+v", cluster.Status.Cephx.Mgr)
|
||||
log.NamespacedDebug(clusterInfo.Namespace, logger, "updating mgr daemon cephx status to %+v", cluster.Status.Cephx.Mgr)
|
||||
if err := reporting.UpdateStatus(c.Client, cluster); err != nil {
|
||||
return errors.Wrap(err, "failed to update cluster cephx status for mgr daemon")
|
||||
}
|
||||
logger.Info("successfully updated the cephx status for mgr daemon")
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "successfully updated the cephx status for mgr daemon")
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -41,7 +42,7 @@ const (
|
||||
)
|
||||
|
||||
func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) {
|
||||
logger.Debugf("mgrConfig: %+v", mgrConfig)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "mgrConfig: %+v", mgrConfig)
|
||||
|
||||
volumes := controller.DaemonVolumes(mgrConfig.DataPathMap, mgrConfig.ResourceName, c.spec.DataDirHostPath)
|
||||
if c.spec.Network.IsMultus() {
|
||||
|
||||
@@ -19,6 +19,7 @@ package mon
|
||||
import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
policyv1 "k8s.io/api/policy/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
@@ -38,7 +39,7 @@ func (c *Cluster) reconcileMonPDB() error {
|
||||
|
||||
monCount := c.spec.Mon.Count
|
||||
if monCount <= 2 {
|
||||
logger.Debug("managePodBudgets is set, but mon-count <= 2. Not creating a disruptionbudget for Mons")
|
||||
log.NamespacedDebug(c.Namespace, logger, "managePodBudgets is set, but mon-count <= 2. Not creating a disruptionbudget for Mons")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -76,7 +77,7 @@ func (c *Cluster) blockMonDrain(request types.NamespacedName) error {
|
||||
if !c.spec.DisruptionManagement.ManagePodBudgets {
|
||||
return nil
|
||||
}
|
||||
logger.Info("prevent voluntary mon drain while failing over")
|
||||
log.NamespacedInfo(c.Namespace, logger, "prevent voluntary mon drain while failing over")
|
||||
// change MaxUnavailable mon PDB to 0
|
||||
_, err := c.createOrUpdateMonPDB(0)
|
||||
if err != nil {
|
||||
@@ -90,7 +91,7 @@ func (c *Cluster) allowMonDrain(request types.NamespacedName) error {
|
||||
if !c.spec.DisruptionManagement.ManagePodBudgets {
|
||||
return nil
|
||||
}
|
||||
logger.Info("allow voluntary mon drain after failover")
|
||||
log.NamespacedInfo(c.Namespace, logger, "allow voluntary mon drain after failover")
|
||||
_, err := c.createOrUpdateMonPDB(c.getMaxUnavailableMonPodCount())
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to update MaxUnavailable for mon PDB %q", request.Name)
|
||||
@@ -100,7 +101,7 @@ func (c *Cluster) allowMonDrain(request types.NamespacedName) error {
|
||||
|
||||
func (c *Cluster) getMaxUnavailableMonPodCount() int32 {
|
||||
if c.spec.Mon.Count >= 5 {
|
||||
logger.Debug("setting the mon pdb max unavailable count to 2 in case there are 5 or more mons")
|
||||
log.NamespacedDebug(c.Namespace, logger, "setting the mon pdb max unavailable count to 2 in case there are 5 or more mons")
|
||||
return 2
|
||||
}
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
@@ -71,7 +72,7 @@ func updateMonTimeout(monCluster *Cluster) {
|
||||
if monCRDTimeoutSetting != "" {
|
||||
if monTimeout, err := time.ParseDuration(monCRDTimeoutSetting); err == nil {
|
||||
if monTimeout == timeZero {
|
||||
logger.Warning("monitor failover is disabled")
|
||||
log.NamespacedWarning(monCluster.Namespace, logger, "monitor failover is disabled")
|
||||
}
|
||||
MonOutTimeout = monTimeout
|
||||
}
|
||||
@@ -95,7 +96,7 @@ func updateMonInterval(monCluster *Cluster, h *HealthChecker) {
|
||||
checkInterval := monCluster.spec.HealthCheck.DaemonHealth.Monitor.Interval
|
||||
// allow overriding the check interval
|
||||
if checkInterval != nil {
|
||||
logger.Debugf("ceph mon status in namespace %q check interval %q", monCluster.Namespace, checkInterval.Duration.String())
|
||||
log.NamespacedDebug(monCluster.Namespace, logger, "ceph mon status check interval %q", checkInterval.Duration.String())
|
||||
h.interval = checkInterval.Duration
|
||||
}
|
||||
}
|
||||
@@ -113,6 +114,7 @@ func NewHealthChecker(monCluster *Cluster) *HealthChecker {
|
||||
|
||||
// Check periodically checks the health of the monitors
|
||||
func (hc *HealthChecker) Check(monitoringRoutines *sync.Map, daemon string) {
|
||||
namespace := hc.monCluster.Namespace
|
||||
for {
|
||||
// Update Mon Timeout with CR details
|
||||
updateMonTimeout(hc.monCluster)
|
||||
@@ -124,28 +126,28 @@ func (hc *HealthChecker) Check(monitoringRoutines *sync.Map, daemon string) {
|
||||
// we will get an invalid pointer error and the go routine will panic
|
||||
v, ok := monitoringRoutines.Load(daemon)
|
||||
if !ok {
|
||||
logger.Infof("ceph cluster %q has been deleted. stopping monitoring of mons", hc.monCluster.Namespace)
|
||||
log.NamespacedInfo(namespace, logger, "ceph cluster has been deleted. stopping monitoring of mons")
|
||||
return
|
||||
}
|
||||
|
||||
health := v.(*controller.ClusterHealth)
|
||||
select {
|
||||
case <-health.InternalCtx.Done():
|
||||
logger.Infof("stopping monitoring of mons in namespace %q (internal context canceled)", hc.monCluster.Namespace)
|
||||
log.NamespacedInfo(namespace, logger, "stopping monitoring of mons (internal context canceled)")
|
||||
monitoringRoutines.Delete(daemon)
|
||||
return
|
||||
|
||||
// Since c.ClusterInfo.IsInitialized() below uses a different context, we need to check if the context is done
|
||||
case <-hc.monCluster.ClusterInfo.Context.Done():
|
||||
logger.Infof("stopping monitoring of mons in namespace %q (mon cluster info context canceled)", hc.monCluster.Namespace)
|
||||
log.NamespacedInfo(namespace, logger, "stopping monitoring of mons (mon cluster info context canceled)")
|
||||
monitoringRoutines.Delete(daemon)
|
||||
return
|
||||
|
||||
case <-time.After(hc.interval):
|
||||
logger.Debugf("checking health of mons")
|
||||
log.NamespacedDebug(namespace, logger, "checking health of mons")
|
||||
err := hc.monCluster.checkHealth(health.InternalCtx)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to check mon health. %v", err)
|
||||
log.NamespacedWarning(namespace, logger, "failed to check mon health. %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -170,11 +172,11 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
return errors.Wrap(err, "failed to check for mons to skip reconcile")
|
||||
}
|
||||
if monsToSkipReconcile.Len() > 0 {
|
||||
logger.Warningf("skipping mon health check since mons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(monsToSkipReconcile))
|
||||
log.NamespacedWarning(c.Namespace, logger, "skipping mon health check since mons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(monsToSkipReconcile))
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Debugf("Checking health for mons in cluster %q", c.ClusterInfo.Namespace)
|
||||
log.NamespacedDebug(c.Namespace, logger, "Checking health for mons in cluster %q", c.ClusterInfo.Namespace)
|
||||
|
||||
// For an external connection we use a special function to get the status
|
||||
if c.spec.External.Enable {
|
||||
@@ -203,7 +205,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to get mon quorum status")
|
||||
}
|
||||
logger.Debugf("Mon quorum status: %+v", quorumStatus)
|
||||
log.NamespacedDebug(c.Namespace, logger, "Mon quorum status: %+v", quorumStatus)
|
||||
|
||||
// handle external Mons
|
||||
quorumStatus, err = c.reconcileExternalMons(ctx, quorumStatus)
|
||||
@@ -214,7 +216,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
// Use a local mon count in case the user updates the crd in another goroutine.
|
||||
// We need to complete a health check with a consistent value.
|
||||
desiredMonCount := c.spec.Mon.Count
|
||||
logger.Debugf("targeting the mon count %d", desiredMonCount)
|
||||
log.NamespacedDebug(c.Namespace, logger, "targeting the mon count %d", desiredMonCount)
|
||||
|
||||
// Source of truth of which mons should exist is our *clusterInfo*
|
||||
monsNotFound := map[string]interface{}{}
|
||||
@@ -235,14 +237,14 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
// when the mon isn't in the clusterInfo, but is in quorum and there are
|
||||
// enough mons, remove it else remove it on the next run
|
||||
if inQuorum && len(quorumStatus.Quorum) > desiredMonCount {
|
||||
logger.Warningf("mon %q not in source of truth but in quorum, removing", mon.Name)
|
||||
log.NamespacedWarning(c.Namespace, logger, "mon %q not in source of truth but in quorum, removing", mon.Name)
|
||||
if err := c.removeMon(mon.Name); err != nil {
|
||||
logger.Warningf("failed to remove mon %q. %v", mon.Name, err)
|
||||
log.NamespacedWarning(c.Namespace, logger, "failed to remove mon %q. %v", mon.Name, err)
|
||||
}
|
||||
// only remove one extra mon per health check
|
||||
return nil
|
||||
}
|
||||
logger.Warningf(
|
||||
log.NamespacedWarning(c.Namespace, logger,
|
||||
"mon %q not in source of truth and not in quorum, not enough mons to remove now (wanted: %d, current: %d)",
|
||||
mon.Name,
|
||||
desiredMonCount,
|
||||
@@ -254,16 +256,16 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
if _, err := c.trackMonInOrOutOfQuorum(mon.Name, true); err != nil {
|
||||
return errors.Wrapf(err, "failed to track out of quorum mon %q", mon.Name)
|
||||
}
|
||||
logger.Debugf("mon %q found in quorum", mon.Name)
|
||||
log.NamespacedDebug(c.Namespace, logger, "mon %q found in quorum", mon.Name)
|
||||
// delete the "timeout" for a mon if the pod is in quorum again
|
||||
if _, ok := c.monTimeoutList[mon.Name]; ok {
|
||||
delete(c.monTimeoutList, mon.Name)
|
||||
logger.Infof("mon %q is back in quorum, removed from mon out timeout list", mon.Name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "mon %q is back in quorum, removed from mon out timeout list", mon.Name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
logger.Debugf("mon %q NOT found in quorum. Mon quorum status: %+v", mon.Name, quorumStatus)
|
||||
log.NamespacedDebug(c.Namespace, logger, "mon %q NOT found in quorum. Mon quorum status: %+v", mon.Name, quorumStatus)
|
||||
allMonsInQuorum = false
|
||||
if _, err := c.trackMonInOrOutOfQuorum(mon.Name, false); err != nil {
|
||||
return errors.Wrapf(err, "failed to track out of quorum mon %q", mon.Name)
|
||||
@@ -271,7 +273,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
|
||||
// if the time out is set to 0 this indicate that we don't want to trigger mon failover
|
||||
if MonOutTimeout == timeZero {
|
||||
logger.Warningf("mon %q NOT found in quorum and health timeout is 0, mon will never fail over", mon.Name)
|
||||
log.NamespacedWarning(c.Namespace, logger, "mon %q NOT found in quorum and health timeout is 0, mon will never fail over", mon.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -290,7 +292,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
if !rescheduleImmediately {
|
||||
if time.Since(c.monTimeoutList[mon.Name]) <= MonOutTimeout {
|
||||
timeToFailover := int(MonOutTimeout.Seconds() - time.Since(c.monTimeoutList[mon.Name]).Seconds())
|
||||
logger.Warningf("mon %q not found in quorum, waiting for timeout (%d seconds left) before failover", mon.Name, timeToFailover)
|
||||
log.NamespacedWarning(c.Namespace, logger, "mon %q not found in quorum, waiting for timeout (%d seconds left) before failover", mon.Name, timeToFailover)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -298,15 +300,15 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
monLabelSelector := fmt.Sprintf("%s=%s,%s=%s", k8sutil.AppAttr, AppName, controller.DaemonIDLabel, mon.Name)
|
||||
isScheduled, err := k8sutil.IsPodScheduled(ctx, c.context.Clientset, c.Namespace, monLabelSelector)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to check if mon %q is assigned to a node, continuing with mon failover. %v", mon.Name, err)
|
||||
log.NamespacedWarning(c.Namespace, logger, "failed to check if mon %q is assigned to a node, continuing with mon failover. %v", mon.Name, err)
|
||||
} else if !isScheduled && retriesBeforeNodeDrainFailover > 0 {
|
||||
logger.Warningf("mon %q NOT found in quorum after timeout. Mon pod is not scheduled. Retrying with a timeout of %.2f seconds before failover", mon.Name, MonOutTimeout.Seconds())
|
||||
log.NamespacedWarning(c.Namespace, logger, "mon %q NOT found in quorum after timeout. Mon pod is not scheduled. Retrying with a timeout of %.2f seconds before failover", mon.Name, MonOutTimeout.Seconds())
|
||||
delete(c.monTimeoutList, mon.Name)
|
||||
retriesBeforeNodeDrainFailover = retriesBeforeNodeDrainFailover - 1
|
||||
return nil
|
||||
}
|
||||
retriesBeforeNodeDrainFailover = 1
|
||||
logger.Warningf("mon %q NOT found in quorum and timeout exceeded, mon will be failed over", mon.Name)
|
||||
log.NamespacedWarning(c.Namespace, logger, "mon %q NOT found in quorum and timeout exceeded, mon will be failed over", mon.Name)
|
||||
}
|
||||
|
||||
if !c.failMon(len(quorumStatus.MonMap.Mons), desiredMonCount, mon.Name) {
|
||||
@@ -328,7 +330,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
// after all unhealthy mons have been removed or failed over
|
||||
// handle all mons that haven't been in the Ceph mon map
|
||||
for mon := range monsNotFound {
|
||||
logger.Warningf("mon %s NOT found in ceph mon map, failover", mon)
|
||||
log.NamespacedWarning(c.Namespace, logger, "mon %s NOT found in ceph mon map, failover", mon)
|
||||
c.failMon(len(c.ClusterInfo.InternalMonitors), desiredMonCount, mon)
|
||||
// only deal with one "not found in ceph mon map" mon per health check
|
||||
return nil
|
||||
@@ -336,23 +338,23 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
|
||||
// create/start new mons when there are fewer mons than the desired count in the CRD
|
||||
if len(quorumStatus.MonMap.Mons) < desiredMonCount {
|
||||
logger.Infof("adding mons. currently %d mons are in quorum and the desired count is %d.", len(quorumStatus.MonMap.Mons), desiredMonCount)
|
||||
log.NamespacedInfo(c.Namespace, logger, "adding mons. currently %d mons are in quorum and the desired count is %d.", len(quorumStatus.MonMap.Mons), desiredMonCount)
|
||||
return c.startMons(desiredMonCount)
|
||||
}
|
||||
|
||||
// remove extra mons if the desired count has decreased in the CRD and all the mons are currently healthy
|
||||
if allMonsInQuorum && len(quorumStatus.MonMap.Mons) > desiredMonCount {
|
||||
if desiredMonCount < 2 && len(quorumStatus.MonMap.Mons) == 2 {
|
||||
logger.Warningf("cannot reduce mon quorum size from 2 to 1")
|
||||
log.NamespacedWarning(c.Namespace, logger, "cannot reduce mon quorum size from 2 to 1")
|
||||
} else {
|
||||
logger.Infof("removing an extra mon. currently %d are in quorum and only %d are desired", len(quorumStatus.MonMap.Mons), desiredMonCount)
|
||||
log.NamespacedInfo(c.Namespace, logger, "removing an extra mon. currently %d are in quorum and only %d are desired", len(quorumStatus.MonMap.Mons), desiredMonCount)
|
||||
return c.removeMon(c.determineExtraMonToRemove())
|
||||
}
|
||||
}
|
||||
|
||||
if allMonsInQuorum && len(quorumStatus.MonMap.Mons) == desiredMonCount {
|
||||
// remove any pending/not needed mon canary deployment if everything is ok
|
||||
logger.Debug("mon cluster is healthy, removing any existing canary deployment")
|
||||
log.NamespacedDebug(c.Namespace, logger, "mon cluster is healthy, removing any existing canary deployment")
|
||||
c.removeCanaryDeployments(monCanaryLabelSelector)
|
||||
|
||||
// Check whether two healthy mons are on the same node when they should not be.
|
||||
@@ -382,7 +384,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
// failover any mons present in the mon fail over list
|
||||
for _, mon := range c.ClusterInfo.InternalMonitors {
|
||||
if _, ok := c.monsToFailover[mon.Name]; ok {
|
||||
logger.Infof("fail over mon %q from the mon fail over list", mon.Name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "fail over mon %q from the mon fail over list", mon.Name)
|
||||
c.failMon(len(c.ClusterInfo.InternalMonitors), desiredMonCount, mon.Name)
|
||||
delete(c.monsToFailover, mon.Name)
|
||||
return nil
|
||||
@@ -394,25 +396,25 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
|
||||
func (c *Cluster) shouldFailoverMonImmediately(ctx context.Context, monName string) bool {
|
||||
// If the assigned node does not exist, reschedule immediately
|
||||
logger.Debugf("checking if mon %q is scheduled on a node", monName)
|
||||
log.NamespacedDebug(c.Namespace, logger, "checking if mon %q is scheduled on a node", monName)
|
||||
assignedNode := ""
|
||||
if mapping, ok := c.mapping.Schedule[monName]; ok {
|
||||
if mapping == nil {
|
||||
// mon schedule information is not available in the rook-ceph-mon-endpoints configmap in case of mons on PVC
|
||||
logger.Debugf("mon schedule information is not available for mon %q", monName)
|
||||
log.NamespacedDebug(c.Namespace, logger, "mon schedule information is not available for mon %q", monName)
|
||||
return false
|
||||
}
|
||||
assignedNode = mapping.Hostname
|
||||
}
|
||||
if assignedNode == "" {
|
||||
logger.Debugf("mon %q is not assigned to a node", monName)
|
||||
log.NamespacedDebug(c.Namespace, logger, "mon %q is not assigned to a node", monName)
|
||||
return false
|
||||
}
|
||||
|
||||
logger.Debugf("mon %q is scheduled on node %q", monName, assignedNode)
|
||||
log.NamespacedDebug(c.Namespace, logger, "mon %q is scheduled on node %q", monName, assignedNode)
|
||||
nodeExists, err := k8sutil.NodeWithHostnameExists(ctx, c.context.Clientset, assignedNode)
|
||||
if err == nil && !nodeExists {
|
||||
logger.Warningf("mon %q NOT found in quorum and its assigned node %q is not found, failing over immediately", monName, assignedNode)
|
||||
log.NamespacedWarning(c.Namespace, logger, "mon %q NOT found in quorum and its assigned node %q is not found, failing over immediately", monName, assignedNode)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -423,11 +425,11 @@ func (c *Cluster) shouldFailoverMonImmediately(ctx context.Context, monName stri
|
||||
func (c *Cluster) reconcileExternalMons(ctx context.Context, quorumStatus cephclient.MonStatusResponse) (cephclient.MonStatusResponse, error) {
|
||||
if len(c.spec.Mon.ExternalMonIDs) != 0 {
|
||||
if c.spec.External.Enable {
|
||||
logger.Warning("ignore external mon IDs in cluster spec: cluster is running in external mode")
|
||||
log.NamespacedWarning(c.Namespace, logger, "ignore external mon IDs in cluster spec: cluster is running in external mode")
|
||||
return quorumStatus, nil
|
||||
}
|
||||
if c.spec.IsStretchCluster() {
|
||||
logger.Warning("ignore external mon IDs in cluster spec: cluster is running in stretch mode")
|
||||
log.NamespacedWarning(c.Namespace, logger, "ignore external mon IDs in cluster spec: cluster is running in stretch mode")
|
||||
return quorumStatus, nil
|
||||
}
|
||||
}
|
||||
@@ -444,19 +446,19 @@ func (c *Cluster) reconcileExternalMons(ctx context.Context, quorumStatus cephcl
|
||||
}
|
||||
// existing external mon was removed from Cluster CRD spec:
|
||||
// remove it from CLusterInfo
|
||||
logger.Debugf("existing external mon %q was removed from spec: removing it", extID)
|
||||
log.NamespacedDebug(c.Namespace, logger, "existing external mon %q was removed from spec: removing it", extID)
|
||||
delete(c.ClusterInfo.ExternalMons, extID)
|
||||
extMonsChanged = true
|
||||
}
|
||||
|
||||
// handle external monitors if configured in cluster CRD:
|
||||
logger.Debugf("external mon IDs: %v", extMonIDs)
|
||||
log.NamespacedDebug(c.Namespace, logger, "external mon IDs: %v", extMonIDs)
|
||||
for _, extID := range extMonIDs {
|
||||
monStatus, inQuorum := getMonByID(extID, quorumStatus)
|
||||
if inQuorum {
|
||||
logger.Debugf("external mon %q in quorum", extID)
|
||||
log.NamespacedDebug(c.Namespace, logger, "external mon %q in quorum", extID)
|
||||
} else {
|
||||
logger.Debugf("external mon %q not in quorum %+v, %+v", extID, quorumStatus.Quorum, quorumStatus.MonMap.Mons)
|
||||
log.NamespacedDebug(c.Namespace, logger, "external mon %q not in quorum %+v, %+v", extID, quorumStatus.Quorum, quorumStatus.MonMap.Mons)
|
||||
}
|
||||
_, inInfo := c.ClusterInfo.ExternalMons[extID]
|
||||
if inQuorum && !inInfo {
|
||||
@@ -464,12 +466,12 @@ func (c *Cluster) reconcileExternalMons(ctx context.Context, quorumStatus cephcl
|
||||
monInfo := c.monStatusToInfo(monStatus)
|
||||
c.ClusterInfo.ExternalMons[extID] = monInfo
|
||||
extMonsChanged = true
|
||||
logger.Infof("new external mon %q found: %s, adding it", extID, monInfo.Endpoint)
|
||||
log.NamespacedInfo(c.Namespace, logger, "new external mon %q found: %s, adding it", extID, monInfo.Endpoint)
|
||||
} else if !inQuorum && inInfo {
|
||||
// remove external mon from cluster info if it is out of quorum:
|
||||
delete(c.ClusterInfo.ExternalMons, extID)
|
||||
extMonsChanged = true
|
||||
logger.Infof("new external mon %q not in quorum: removing it", extID)
|
||||
log.NamespacedInfo(c.Namespace, logger, "new external mon %q not in quorum: removing it", extID)
|
||||
}
|
||||
}
|
||||
if extMonsChanged {
|
||||
@@ -516,7 +518,7 @@ func (c *Cluster) trackMonInOrOutOfQuorum(monName string, inQuorum bool) (bool,
|
||||
// All mons are in quorum, so make sure no mons are marked out of quorum
|
||||
for monName, mon := range c.ClusterInfo.InternalMonitors {
|
||||
if mon.OutOfQuorum {
|
||||
logger.Infof("resetting mon %q to be back in quorum", monName)
|
||||
log.NamespacedInfo(c.Namespace, logger, "resetting mon %q to be back in quorum", monName)
|
||||
mon.OutOfQuorum = false
|
||||
updateNeeded = true
|
||||
}
|
||||
@@ -524,18 +526,18 @@ func (c *Cluster) trackMonInOrOutOfQuorum(monName string, inQuorum bool) (bool,
|
||||
} else {
|
||||
mon, ok := c.ClusterInfo.InternalMonitors[monName]
|
||||
if !ok {
|
||||
logger.Infof("mon %q not found to keep track of being out of quorum", monName)
|
||||
log.NamespacedInfo(c.Namespace, logger, "mon %q not found to keep track of being out of quorum", monName)
|
||||
return false, nil
|
||||
}
|
||||
// Mark the mon in quorum
|
||||
if inQuorum && mon.OutOfQuorum {
|
||||
logger.Infof("marking mon %q back in quorum", monName)
|
||||
log.NamespacedInfo(c.Namespace, logger, "marking mon %q back in quorum", monName)
|
||||
mon.OutOfQuorum = false
|
||||
updateNeeded = true
|
||||
}
|
||||
// Mark the mon out of quorum
|
||||
if !inQuorum && !mon.OutOfQuorum {
|
||||
logger.Infof("marking mon %q out of quorum", monName)
|
||||
log.NamespacedInfo(c.Namespace, logger, "marking mon %q out of quorum", monName)
|
||||
mon.OutOfQuorum = true
|
||||
updateNeeded = true
|
||||
}
|
||||
@@ -570,7 +572,7 @@ func (c *Cluster) determineExtraMonToRemove() string {
|
||||
if stretchMonToRemove != "" {
|
||||
return stretchMonToRemove
|
||||
}
|
||||
logger.Infof("did not find an extra mon to remove from the stretch cluster")
|
||||
log.NamespacedInfo(c.Namespace, logger, "did not find an extra mon to remove from the stretch cluster")
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -578,13 +580,13 @@ func (c *Cluster) determineExtraMonToRemove() string {
|
||||
arbitraryMon := ""
|
||||
for _, mon := range mons {
|
||||
if mon.NodeName == "" {
|
||||
logger.Debugf("mon %q is not scheduled to a specific host", mon.DaemonName)
|
||||
log.NamespacedDebug(c.Namespace, logger, "mon %q is not scheduled to a specific host", mon.DaemonName)
|
||||
arbitraryMon = mon.DaemonName
|
||||
continue
|
||||
}
|
||||
// Check if there are multiple mons on the node
|
||||
if existingMon, ok := nodesWithMons[mon.NodeName]; ok {
|
||||
logger.Infof("found mons %q and %q on node %s, removing mon %q", existingMon, mon.DaemonName, mon.NodeName, mon.DaemonName)
|
||||
log.NamespacedInfo(c.Namespace, logger, "found mons %q and %q on node %s, removing mon %q", existingMon, mon.DaemonName, mon.NodeName, mon.DaemonName)
|
||||
return mon.DaemonName
|
||||
}
|
||||
nodesWithMons[mon.NodeName] = mon.DaemonName
|
||||
@@ -593,7 +595,7 @@ func (c *Cluster) determineExtraMonToRemove() string {
|
||||
arbitraryMon = mon.DaemonName
|
||||
}
|
||||
|
||||
logger.Infof("removing arbitrary extra mon %q", arbitraryMon)
|
||||
log.NamespacedInfo(c.Namespace, logger, "removing arbitrary extra mon %q", arbitraryMon)
|
||||
return arbitraryMon
|
||||
}
|
||||
|
||||
@@ -603,7 +605,7 @@ func (c *Cluster) findExtraMonToRemoveFromStretchCluster(mons []*monConfig) stri
|
||||
monInZones := map[string]string{}
|
||||
for _, m := range mons {
|
||||
if m.Zone == "" {
|
||||
logger.Warningf("zone not found on mon %q", m.DaemonName)
|
||||
log.NamespacedWarning(c.Namespace, logger, "zone not found on mon %q", m.DaemonName)
|
||||
continue
|
||||
}
|
||||
zoneCount[m.Zone]++
|
||||
@@ -620,12 +622,12 @@ func (c *Cluster) findExtraMonToRemoveFromStretchCluster(mons []*monConfig) stri
|
||||
}
|
||||
if zone.Arbiter {
|
||||
if count > 1 {
|
||||
logger.Infof("removing extra mon %q in arbiter zone %q", monInZones[zone.Name], zone.Name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "removing extra mon %q in arbiter zone %q", monInZones[zone.Name], zone.Name)
|
||||
return monInZones[zone.Name]
|
||||
}
|
||||
} else {
|
||||
if count > 2 {
|
||||
logger.Infof("removing extra mon %q in zone %q", monInZones[zone.Name], zone.Name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "removing extra mon %q in zone %q", monInZones[zone.Name], zone.Name)
|
||||
return monInZones[zone.Name]
|
||||
}
|
||||
}
|
||||
@@ -639,69 +641,69 @@ func (c *Cluster) findExtraMonToRemoveFromStretchCluster(mons []*monConfig) stri
|
||||
func (c *Cluster) failMon(monCount, desiredMonCount int, name string) bool {
|
||||
// make sure the failed mon is marked out of quorum
|
||||
if _, err := c.trackMonInOrOutOfQuorum(name, false); err != nil {
|
||||
logger.Errorf("failed to track failed mon %q. %v", name, err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to track failed mon %q. %v", name, err)
|
||||
}
|
||||
|
||||
if monCount > desiredMonCount {
|
||||
// no need to create a new mon since we have an extra
|
||||
if err := c.removeMon(name); err != nil {
|
||||
logger.Errorf("failed to remove mon %q. %v", name, err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to remove mon %q. %v", name, err)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// prevent any voluntary mon drain while failing over
|
||||
if err := c.blockMonDrain(types.NamespacedName{Name: monPDBName, Namespace: c.Namespace}); err != nil {
|
||||
logger.Errorf("failed to block mon drain. %v", err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to block mon drain. %v", err)
|
||||
}
|
||||
|
||||
// bring up a new mon to replace the unhealthy mon
|
||||
if err := c.failoverMon(name); err != nil {
|
||||
logger.Errorf("failed to failover mon %q. %v", name, err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to failover mon %q. %v", name, err)
|
||||
}
|
||||
|
||||
// allow any voluntary mon drain after failover
|
||||
if err := c.allowMonDrain(types.NamespacedName{Name: monPDBName, Namespace: c.Namespace}); err != nil {
|
||||
logger.Errorf("failed to allow mon drain. %v", err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to allow mon drain. %v", err)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *Cluster) removeOrphanMonResources() {
|
||||
if c.spec.Mon.VolumeClaimTemplate == nil {
|
||||
logger.Debug("skipping check for orphaned mon pvcs since using the host path")
|
||||
log.NamespacedDebug(c.Namespace, logger, "skipping check for orphaned mon pvcs since using the host path")
|
||||
return
|
||||
}
|
||||
|
||||
logger.Info("checking for orphaned mon resources")
|
||||
log.NamespacedInfo(c.Namespace, logger, "checking for orphaned mon resources")
|
||||
|
||||
opts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, AppName)}
|
||||
pvcs, err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).List(c.ClusterInfo.Context, opts)
|
||||
if err != nil {
|
||||
logger.Infof("failed to check for orphaned mon pvcs. %v", err)
|
||||
log.NamespacedInfo(c.Namespace, logger, "failed to check for orphaned mon pvcs. %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, pvc := range pvcs.Items {
|
||||
logger.Debugf("checking if pvc %q is orphaned", pvc.Name)
|
||||
log.NamespacedDebug(c.Namespace, logger, "checking if pvc %q is orphaned", pvc.Name)
|
||||
|
||||
_, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, pvc.Name, metav1.GetOptions{})
|
||||
if err == nil {
|
||||
logger.Debugf("skipping pvc removal since the mon daemon %q still requires it", pvc.Name)
|
||||
log.NamespacedDebug(c.Namespace, logger, "skipping pvc removal since the mon daemon %q still requires it", pvc.Name)
|
||||
continue
|
||||
}
|
||||
if !kerrors.IsNotFound(err) {
|
||||
logger.Infof("skipping pvc removal since the mon daemon %q might still require it. %v", pvc.Name, err)
|
||||
log.NamespacedInfo(c.Namespace, logger, "skipping pvc removal since the mon daemon %q might still require it. %v", pvc.Name, err)
|
||||
continue
|
||||
}
|
||||
|
||||
logger.Infof("removing pvc %q since it is no longer needed for the mon daemon", pvc.Name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "removing pvc %q since it is no longer needed for the mon daemon", pvc.Name)
|
||||
var gracePeriod int64 // delete immediately
|
||||
propagation := metav1.DeletePropagationForeground
|
||||
options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
|
||||
err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(c.ClusterInfo.Context, pvc.Name, *options)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to delete orphaned monitor pvc %q. %v", pvc.Name, err)
|
||||
log.NamespacedWarning(c.Namespace, logger, "failed to delete orphaned monitor pvc %q. %v", pvc.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -722,7 +724,7 @@ func (c *Cluster) updateMonDeploymentReplica(name string, enabled bool) error {
|
||||
d.Spec.Replicas = &desiredReplicas
|
||||
|
||||
// update the deployment
|
||||
logger.Infof("scaling the mon %q deployment to replica %d", name, desiredReplicas)
|
||||
log.NamespacedInfo(c.Namespace, logger, "scaling the mon %q deployment to replica %d", name, desiredReplicas)
|
||||
_, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Update(c.ClusterInfo.Context, d, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to update mon %q replicas from %d to %d", name, originalReplicas, desiredReplicas)
|
||||
@@ -731,7 +733,7 @@ func (c *Cluster) updateMonDeploymentReplica(name string, enabled bool) error {
|
||||
}
|
||||
|
||||
func (c *Cluster) failoverMon(name string) error {
|
||||
logger.Infof("Failing over monitor %q", name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "Failing over monitor %q", name)
|
||||
|
||||
// remove the failed mon from a local list of the existing mons for finding a stretch zone
|
||||
existingMons := c.clusterInfoToMonConfigWithExclude(name)
|
||||
@@ -743,13 +745,13 @@ func (c *Cluster) failoverMon(name string) error {
|
||||
|
||||
// Start a new monitor
|
||||
m := c.newMonConfig(c.maxMonID+1, zone)
|
||||
logger.Infof("starting new mon: %+v", m)
|
||||
log.NamespacedInfo(c.Namespace, logger, "starting new mon: %+v", m)
|
||||
|
||||
// Scale down the failed mon to allow a new one to start
|
||||
if c.stopMonDuringFailover(name) {
|
||||
if err := c.updateMonDeploymentReplica(name, false); err != nil {
|
||||
// attempt to continue with the failover even if the bad mon could not be stopped
|
||||
logger.Warningf("failed to stop mon %q for failover. %v", name, err)
|
||||
log.NamespacedWarning(c.Namespace, logger, "failed to stop mon %q for failover. %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -761,21 +763,21 @@ func (c *Cluster) failoverMon(name string) error {
|
||||
// do nothing if the new mon was started successfully, the deployment will anyway be deleted
|
||||
return
|
||||
}
|
||||
logger.Warningf("failover of mon %q unsuccessful, cleaning up replacement mon %q", name, m.DaemonName)
|
||||
log.NamespacedWarning(c.Namespace, logger, "failover of mon %q unsuccessful, cleaning up replacement mon %q", name, m.DaemonName)
|
||||
if err := c.updateMonDeploymentReplica(name, true); err != nil {
|
||||
// attempt to continue even if the bad mon could not be restarted
|
||||
logger.Warningf("failed to restart failed mon %q after new mon wouldn't start. %v", name, err)
|
||||
log.NamespacedWarning(c.Namespace, logger, "failed to restart failed mon %q after new mon wouldn't start. %v", name, err)
|
||||
}
|
||||
if err := c.removeMonWithOptionalQuorum(m.DaemonName, newMonMightBeInQuorum); err != nil {
|
||||
logger.Infof("failed to remove mon %q from quorum. %v", m.DaemonName, err)
|
||||
log.NamespacedInfo(c.Namespace, logger, "failed to remove mon %q from quorum. %v", m.DaemonName, err)
|
||||
}
|
||||
|
||||
// Make sure the maxMonID is reverted to its previous value
|
||||
// The maxMonId is committed to a configmap immediately after the mon deployment
|
||||
// is started, even though c.maxMonID is not incremented until the mon failover is successful
|
||||
logger.Infof("reverting maxMonId to %d", c.maxMonID)
|
||||
log.NamespacedInfo(c.Namespace, logger, "reverting maxMonId to %d", c.maxMonID)
|
||||
if err := c.commitMaxMonIDRequireIncrementing(c.maxMonID, false); err != nil {
|
||||
logger.Errorf("failed to revert maxMonId after starting mon %q", m.DaemonName)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to revert maxMonId after starting mon %q", m.DaemonName)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -803,7 +805,7 @@ func (c *Cluster) failoverMon(name string) error {
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to export service %q", monService.Name)
|
||||
}
|
||||
logger.Infof("mon %q exported IP is %s", m.DaemonName, exportedIP)
|
||||
log.NamespacedInfo(c.Namespace, logger, "mon %q exported IP is %s", m.DaemonName, exportedIP)
|
||||
m.PublicIP = exportedIP
|
||||
} else {
|
||||
m.PublicIP = monService.Spec.ClusterIP
|
||||
@@ -846,7 +848,7 @@ func (c *Cluster) stopMonDuringFailover(name string) bool {
|
||||
if mon, ok := c.monsToFailover[name]; ok && !mon.UseHostNetwork {
|
||||
return true
|
||||
}
|
||||
logger.Infof("skipping stopping mon %q during failover, since with host networking a new mon cannot be started on the same node", name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "skipping stopping mon %q during failover, since with host networking a new mon cannot be started on the same node", name)
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -858,15 +860,15 @@ func (c *Cluster) removeMon(daemonName string) error {
|
||||
// make a best effort to remove the mon and all its resources
|
||||
func (c *Cluster) removeMonWithOptionalQuorum(daemonName string, shouldRemoveFromQuorum bool) error {
|
||||
if daemonName == "" {
|
||||
logger.Info("did not identify a mon to remove")
|
||||
log.NamespacedInfo(c.Namespace, logger, "did not identify a mon to remove")
|
||||
return nil
|
||||
}
|
||||
logger.Infof("ensuring removal of unhealthy monitor %s", daemonName)
|
||||
log.NamespacedInfo(c.Namespace, logger, "ensuring removal of unhealthy monitor %s", daemonName)
|
||||
|
||||
// Remove the bad monitor from quorum
|
||||
if shouldRemoveFromQuorum {
|
||||
if err := c.removeMonitorFromQuorum(daemonName); err != nil {
|
||||
logger.Errorf("failed to remove mon %q from quorum. %v", daemonName, err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to remove mon %q from quorum. %v", daemonName, err)
|
||||
}
|
||||
}
|
||||
delete(c.ClusterInfo.InternalMonitors, daemonName)
|
||||
@@ -899,39 +901,39 @@ func (c *Cluster) removeMonResources(daemonName string) {
|
||||
options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
|
||||
if err := c.context.Clientset.CoreV1().Services(c.Namespace).Delete(c.ClusterInfo.Context, resourceName, *options); err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Infof("dead mon service %s was already gone", resourceName)
|
||||
log.NamespacedInfo(c.Namespace, logger, "dead mon service %s was already gone", resourceName)
|
||||
} else {
|
||||
logger.Errorf("failed to remove dead mon service %q. %v", resourceName, err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to remove dead mon service %q. %v", resourceName, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Remove the PVC backing the mon if it existed
|
||||
if err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(c.ClusterInfo.Context, resourceName, metav1.DeleteOptions{}); err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Infof("mon pvc did not exist %q", resourceName)
|
||||
log.NamespacedInfo(c.Namespace, logger, "mon pvc did not exist %q", resourceName)
|
||||
} else {
|
||||
logger.Errorf("failed to remove dead mon pvc %q. %v", resourceName, err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to remove dead mon pvc %q. %v", resourceName, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Remove the mon pod if it is still there
|
||||
if err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Delete(c.ClusterInfo.Context, resourceName, *options); err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Infof("dead mon %s was already gone", resourceName)
|
||||
log.NamespacedInfo(c.Namespace, logger, "dead mon %s was already gone", resourceName)
|
||||
} else {
|
||||
logger.Errorf("failed to remove dead mon deployment %q. %v", resourceName, err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to remove dead mon deployment %q. %v", resourceName, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) removeMonitorFromQuorum(name string) error {
|
||||
logger.Debugf("removing monitor %s", name)
|
||||
log.NamespacedDebug(c.Namespace, logger, "removing monitor %s", name)
|
||||
args := []string{"mon", "remove", name}
|
||||
if _, err := cephclient.NewCephCommand(c.context, c.ClusterInfo, args).Run(); err != nil {
|
||||
return errors.Wrapf(err, "mon %s remove failed", name)
|
||||
}
|
||||
|
||||
logger.Infof("removed monitor %s", name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "removed monitor %s", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -969,11 +971,11 @@ func (c *Cluster) addOrRemoveExternalMonitor(status cephclient.MonStatusResponse
|
||||
oldClusterInfoMonitors[mon.Name] = mon
|
||||
delete(c.ClusterInfo.InternalMonitors, monName)
|
||||
}
|
||||
logger.Debugf("ClusterInfo is now Empty, refilling it from status.MonMap.Mons")
|
||||
log.NamespacedDebug(c.Namespace, logger, "ClusterInfo is now Empty, refilling it from status.MonMap.Mons")
|
||||
|
||||
monCount := len(status.MonMap.Mons)
|
||||
if monCount == 1 {
|
||||
logger.Warning("external cluster mon count is 1, consider adding new monitors.")
|
||||
log.NamespacedWarning(c.Namespace, logger, "external cluster mon count is 1, consider adding new monitors.")
|
||||
}
|
||||
|
||||
// Iterate over the mons first and compare it with ClusterInfo
|
||||
@@ -984,25 +986,25 @@ func (c *Cluster) addOrRemoveExternalMonitor(status cephclient.MonStatusResponse
|
||||
// If the mon is part of the quorum
|
||||
if inQuorum {
|
||||
info := c.monStatusToInfo(mon)
|
||||
logger.Infof("new external mon %q found: %s, adding it", mon.Name, info.Endpoint)
|
||||
log.NamespacedInfo(c.Namespace, logger, "new external mon %q found: %s, adding it", mon.Name, info.Endpoint)
|
||||
c.ClusterInfo.InternalMonitors[mon.Name] = info
|
||||
} else {
|
||||
logger.Debugf("mon %q is not in quorum and not in ClusterInfo", mon.Name)
|
||||
log.NamespacedDebug(c.Namespace, logger, "mon %q is not in quorum and not in ClusterInfo", mon.Name)
|
||||
}
|
||||
changed = true
|
||||
} else {
|
||||
// mon is in ClusterInfo
|
||||
logger.Debugf("mon %q is in ClusterInfo, let's test if it's in quorum", mon.Name)
|
||||
log.NamespacedDebug(c.Namespace, logger, "mon %q is in ClusterInfo, let's test if it's in quorum", mon.Name)
|
||||
if !inQuorum {
|
||||
// this mon was in clusterInfo but is not part of the quorum anymore
|
||||
// thus don't add it again to ClusterInfo
|
||||
logger.Infof("monitor %q is not part of the external cluster monitor quorum, removing it", mon.Name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "monitor %q is not part of the external cluster monitor quorum, removing it", mon.Name)
|
||||
changed = true
|
||||
} else {
|
||||
// this mon was in clusterInfo and is still in the quorum
|
||||
// add it again
|
||||
c.ClusterInfo.InternalMonitors[mon.Name] = oldClusterInfoMonitors[mon.Name]
|
||||
logger.Debugf("everything is fine mon %q in the clusterInfo and its quorum status is %v", mon.Name, inQuorum)
|
||||
log.NamespacedDebug(c.Namespace, logger, "everything is fine mon %q in the clusterInfo and its quorum status is %v", mon.Name, inQuorum)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1019,7 +1021,7 @@ func (c *Cluster) addOrRemoveExternalMonitor(status cephclient.MonStatusResponse
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debugf("ClusterInfo.Monitors is %+v", c.ClusterInfo.InternalMonitors)
|
||||
log.NamespacedDebug(c.Namespace, logger, "ClusterInfo.Monitors is %+v", c.ClusterInfo.InternalMonitors)
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
@@ -1041,11 +1043,11 @@ func (c *Cluster) monStatusToInfo(mon cephclient.MonMapEntry) *cephclient.MonInf
|
||||
|
||||
func (c *Cluster) evictMonIfMultipleOnSameNode() error {
|
||||
if c.spec.Mon.AllowMultiplePerNode {
|
||||
logger.Debug("skipping check for multiple mons on same node since multiple mons are allowed")
|
||||
log.NamespacedDebug(c.Namespace, logger, "skipping check for multiple mons on same node since multiple mons are allowed")
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Info("checking if multiple mons are on the same node")
|
||||
log.NamespacedInfo(c.Namespace, logger, "checking if multiple mons are on the same node")
|
||||
|
||||
// Get all the mon pods
|
||||
label := fmt.Sprintf("app=%s", AppName)
|
||||
@@ -1056,13 +1058,13 @@ func (c *Cluster) evictMonIfMultipleOnSameNode() error {
|
||||
|
||||
nodesToMons := map[string]string{}
|
||||
for _, pod := range pods.Items {
|
||||
logger.Debugf("analyzing mon pod %q on node %q", pod.Name, pod.Spec.NodeName)
|
||||
log.NamespacedDebug(c.Namespace, logger, "analyzing mon pod %q on node %q", pod.Name, pod.Spec.NodeName)
|
||||
if _, ok := pod.Labels["mon_canary"]; ok {
|
||||
logger.Debugf("skipping mon canary pod %q", pod.Name)
|
||||
log.NamespacedDebug(c.Namespace, logger, "skipping mon canary pod %q", pod.Name)
|
||||
continue
|
||||
}
|
||||
if pod.Spec.NodeName == "" {
|
||||
logger.Warningf("mon %q is not assigned to a node", pod.Name)
|
||||
log.NamespacedWarning(c.Namespace, logger, "mon %q is not assigned to a node", pod.Name)
|
||||
continue
|
||||
}
|
||||
monName := pod.Labels["mon"]
|
||||
@@ -1073,7 +1075,7 @@ func (c *Cluster) evictMonIfMultipleOnSameNode() error {
|
||||
continue
|
||||
}
|
||||
|
||||
logger.Warningf("Both mons %q and %q are on node %q. Evicting mon %q", monName, previousMonName, pod.Spec.NodeName, monName)
|
||||
log.NamespacedWarning(c.Namespace, logger, "Both mons %q and %q are on node %q. Evicting mon %q", monName, previousMonName, pod.Spec.NodeName, monName)
|
||||
return c.failoverMon(monName)
|
||||
}
|
||||
|
||||
|
||||
@@ -44,6 +44,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/reporting"
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
discoveryv1 "k8s.io/api/discovery/v1"
|
||||
@@ -217,21 +218,21 @@ func (c *Cluster) Start(clusterInfo *cephclient.ClusterInfo, rookImage string, c
|
||||
return nil, errors.Wrap(err, "failed to check pod memory")
|
||||
}
|
||||
|
||||
logger.Infof("start running mons")
|
||||
log.NamespacedInfo(c.Namespace, logger, "start running mons")
|
||||
|
||||
logger.Debugf("establishing ceph cluster info")
|
||||
log.NamespacedDebug(c.Namespace, logger, "establishing ceph cluster info")
|
||||
if err := c.initClusterInfo(cephVersion, c.ClusterInfo.NamespacedName().Name); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to initialize ceph cluster info")
|
||||
}
|
||||
|
||||
logger.Infof("targeting the mon count %d", c.spec.Mon.Count)
|
||||
log.NamespacedInfo(c.Namespace, logger, "targeting the mon count %d", c.spec.Mon.Count)
|
||||
|
||||
monsToSkipReconcile, err := controller.GetDaemonsToSkipReconcile(c.ClusterInfo.Context, c.context, c.Namespace, config.MonType, AppName)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to check for mons to skip reconcile")
|
||||
}
|
||||
if monsToSkipReconcile.Len() > 0 {
|
||||
logger.Warningf("skipping mon reconcile since mons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(monsToSkipReconcile))
|
||||
log.NamespacedWarning(c.Namespace, logger, "skipping mon reconcile since mons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(monsToSkipReconcile))
|
||||
return c.ClusterInfo, nil
|
||||
}
|
||||
|
||||
@@ -263,7 +264,7 @@ func (c *Cluster) startMons(targetCount int) error {
|
||||
// fixed by updating the mon deployments. Instead of returning error here, log a
|
||||
// warning, and retry setting this later.
|
||||
setConfigsNeedsRetry = true
|
||||
logger.Warningf("failed to set Rook and/or user-defined Ceph config options before starting mons; will retry after starting mons. %v", err)
|
||||
log.NamespacedWarning(c.Namespace, logger, "failed to set Rook and/or user-defined Ceph config options before starting mons; will retry after starting mons. %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -321,7 +322,7 @@ func (c *Cluster) startMons(targetCount int) error {
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debugf("mon endpoints used are: %s", flattenMonEndpoints(c.ClusterInfo.AllMonitors()))
|
||||
log.NamespacedDebug(c.Namespace, logger, "mon endpoints used are: %s", flattenMonEndpoints(c.ClusterInfo.AllMonitors()))
|
||||
|
||||
// reconcile mon PDB
|
||||
if err := c.reconcileMonPDB(); err != nil {
|
||||
@@ -376,14 +377,14 @@ func (c *Cluster) ConfigureArbiter() error {
|
||||
|
||||
monDump, err := cephclient.GetMonDump(c.context, c.ClusterInfo)
|
||||
if err != nil {
|
||||
logger.Warningf("attempting to enable arbiter after failed to detect if already enabled. %v", err)
|
||||
log.NamespacedWarning(c.Namespace, logger, "attempting to enable arbiter after failed to detect if already enabled. %v", err)
|
||||
} else if monDump.StretchMode {
|
||||
if monDump.TiebreakerMon == c.arbiterMon {
|
||||
logger.Infof("stretch mode is already enabled with tiebreaker %q", c.arbiterMon)
|
||||
log.NamespacedInfo(c.Namespace, logger, "stretch mode is already enabled with tiebreaker %q", c.arbiterMon)
|
||||
return nil
|
||||
}
|
||||
// Set the new mon tiebreaker
|
||||
logger.Infof("updating tiebreaker mon from %q to %q", monDump.TiebreakerMon, c.arbiterMon)
|
||||
log.NamespacedInfo(c.Namespace, logger, "updating tiebreaker mon from %q to %q", monDump.TiebreakerMon, c.arbiterMon)
|
||||
if err := cephclient.SetNewTiebreaker(c.context, c.ClusterInfo, c.arbiterMon); err != nil {
|
||||
return errors.Wrap(err, "failed to set new mon tiebreaker")
|
||||
}
|
||||
@@ -394,7 +395,7 @@ func (c *Cluster) ConfigureArbiter() error {
|
||||
// The timeout is relatively short since the operator will requeue the reconcile
|
||||
// and try again at a higher level if not yet found
|
||||
failureDomain := c.getFailureDomainName()
|
||||
logger.Infof("enabling stretch mode... waiting for two failure domains of type %q to be found in the CRUSH map after OSD initialization", failureDomain)
|
||||
log.NamespacedInfo(c.Namespace, logger, "enabling stretch mode... waiting for two failure domains of type %q to be found in the CRUSH map after OSD initialization", failureDomain)
|
||||
pollInterval := 5 * time.Second
|
||||
totalWaitTime := 2 * time.Minute
|
||||
err = wait.PollUntilContextTimeout(c.ClusterInfo.Context, pollInterval, totalWaitTime, true, func(ctx context.Context) (bool, error) {
|
||||
@@ -408,7 +409,7 @@ func (c *Cluster) ConfigureArbiter() error {
|
||||
// Wait for the .mgr pool to be created, which we expect is defined as a CephBlockPool
|
||||
// We may be able to remove this code waiting for the pool once this is in the ceph release:
|
||||
// https://github.com/ceph/ceph/pull/61371
|
||||
logger.Info("enabling stretch mode... waiting for the builtin .mgr pool to be created")
|
||||
log.NamespacedInfo(c.Namespace, logger, "enabling stretch mode... waiting for the builtin .mgr pool to be created")
|
||||
err = wait.PollUntilContextTimeout(c.ClusterInfo.Context, pollInterval, totalWaitTime, true, func(ctx context.Context) (bool, error) {
|
||||
return c.builtinMgrPoolExists(), nil
|
||||
})
|
||||
@@ -443,7 +444,7 @@ func (c *Cluster) readyToConfigureArbiter(checkOSDPods bool) (bool, error) {
|
||||
return false, errors.Wrap(err, "failed to check whether all osds are running before enabling the arbiter")
|
||||
}
|
||||
if !allRunning {
|
||||
logger.Infof("waiting for all OSD pods to be in running state")
|
||||
log.NamespacedInfo(c.Namespace, logger, "waiting for all OSD pods to be in running state")
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
@@ -460,10 +461,10 @@ func (c *Cluster) readyToConfigureArbiter(checkOSDPods bool) (bool, error) {
|
||||
if bucket.TypeName == failureDomain {
|
||||
// skip zones specific to device classes
|
||||
if strings.Index(bucket.Name, "~") > 0 {
|
||||
logger.Debugf("skipping device class bucket %q", bucket.Name)
|
||||
log.NamespacedDebug(c.Namespace, logger, "skipping device class bucket %q", bucket.Name)
|
||||
continue
|
||||
}
|
||||
logger.Infof("found %s %q in CRUSH map with weight %d", failureDomain, bucket.Name, bucket.Weight)
|
||||
log.NamespacedInfo(c.Namespace, logger, "found %s %q in CRUSH map with weight %d", failureDomain, bucket.Name, bucket.Weight)
|
||||
zoneCount++
|
||||
|
||||
// check that the weights of the failure domains are all the same
|
||||
@@ -471,7 +472,7 @@ func (c *Cluster) readyToConfigureArbiter(checkOSDPods bool) (bool, error) {
|
||||
// found the first matching bucket
|
||||
zoneWeight = bucket.Weight
|
||||
} else if zoneWeight != bucket.Weight {
|
||||
logger.Infof("found failure domains that have different weights")
|
||||
log.NamespacedInfo(c.Namespace, logger, "found failure domains that have different weights")
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
@@ -483,7 +484,7 @@ func (c *Cluster) readyToConfigureArbiter(checkOSDPods bool) (bool, error) {
|
||||
if zoneCount > 2 {
|
||||
return false, fmt.Errorf("cannot configure stretch cluster with more than 2 failure domains, and found %d of type %q", zoneCount, failureDomain)
|
||||
}
|
||||
logger.Infof("found two expected failure domains %q for the stretch cluster", failureDomain)
|
||||
log.NamespacedInfo(c.Namespace, logger, "found two expected failure domains %q for the stretch cluster", failureDomain)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -494,9 +495,9 @@ func (c *Cluster) readyToConfigureArbiter(checkOSDPods bool) (bool, error) {
|
||||
// This is needed when the operator is restarted and all mons may not be up or in quorum.
|
||||
func (c *Cluster) ensureMonsRunning(mons []*monConfig, i, targetCount int, requireAllInQuorum bool) error {
|
||||
if requireAllInQuorum {
|
||||
logger.Infof("creating mon %s", mons[i].DaemonName)
|
||||
log.NamespacedInfo(c.Namespace, logger, "creating mon %s", mons[i].DaemonName)
|
||||
} else {
|
||||
logger.Info("checking for basic quorum with existing mons")
|
||||
log.NamespacedInfo(c.Namespace, logger, "checking for basic quorum with existing mons")
|
||||
}
|
||||
|
||||
// Calculate how many mons we expected to exist after this method is completed.
|
||||
@@ -609,7 +610,7 @@ func (c *Cluster) clusterInfoToMonConfigWithExclude(excludedMon string) []*monCo
|
||||
isHostNetwork = true
|
||||
}
|
||||
}
|
||||
logger.Debugf("Host network for mon %q is %t", monitor.Name, isHostNetwork)
|
||||
log.NamespacedDebug(c.Namespace, logger, "Host network for mon %q is %t", monitor.Name, isHostNetwork)
|
||||
|
||||
mons = append(mons, &monConfig{
|
||||
ResourceName: resourceName(monitor.Name),
|
||||
@@ -739,10 +740,10 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) {
|
||||
|
||||
_, err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Create(c.ClusterInfo.Context, pvc, metav1.CreateOptions{})
|
||||
if err == nil {
|
||||
logger.Infof("created canary monitor %s pvc %s", d.Name, pvc.Name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "created canary monitor %s pvc %s", d.Name, pvc.Name)
|
||||
} else {
|
||||
if kerrors.IsAlreadyExists(err) {
|
||||
logger.Debugf("creating mon %s pvc %s: already exists.", d.Name, pvc.Name)
|
||||
log.NamespacedDebug(c.Namespace, logger, "creating mon %s pvc %s: already exists.", d.Name, pvc.Name)
|
||||
} else {
|
||||
return nil, errors.Wrapf(err, "failed to create mon %s pvc %s", d.Name, pvc.Name)
|
||||
}
|
||||
@@ -763,13 +764,13 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) {
|
||||
_, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(c.ClusterInfo.Context, d, metav1.CreateOptions{})
|
||||
if err == nil {
|
||||
createdDeployment = true
|
||||
logger.Infof("created canary deployment %s", d.Name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "created canary deployment %s", d.Name)
|
||||
break
|
||||
} else if kerrors.IsAlreadyExists(err) {
|
||||
if err := k8sutil.DeleteDeployment(c.ClusterInfo.Context, c.context.Clientset, c.Namespace, d.Name); err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to delete canary deployment %s", d.Name)
|
||||
}
|
||||
logger.Infof("deleted existing canary deployment %s", d.Name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "deleted existing canary deployment %s", d.Name)
|
||||
time.Sleep(time.Second * canaryRetryDelaySeconds)
|
||||
} else {
|
||||
return nil, errors.Wrapf(err, "failed to create canary monitor deployment %s", d.Name)
|
||||
@@ -824,13 +825,13 @@ func realWaitForMonitorScheduling(c *Cluster, d *apps.Deployment) (SchedulingRes
|
||||
}
|
||||
|
||||
if len(pods.Items) == 0 {
|
||||
logger.Infof("waiting for canary pod creation %s", d.Name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "waiting for canary pod creation %s", d.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
pod := pods.Items[0]
|
||||
if pod.Spec.NodeName == "" {
|
||||
logger.Debugf("monitor %s canary pod %s not yet scheduled", d.Name, pod.Name)
|
||||
log.NamespacedDebug(c.Namespace, logger, "monitor %s canary pod %s not yet scheduled", d.Name, pod.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -840,7 +841,7 @@ func realWaitForMonitorScheduling(c *Cluster, d *apps.Deployment) (SchedulingRes
|
||||
}
|
||||
|
||||
result.Node = node
|
||||
logger.Infof("canary monitor deployment %s scheduled to %s", d.Name, node.Name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "canary monitor deployment %s scheduled to %s", d.Name, node.Name)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -853,7 +854,7 @@ func (c *Cluster) initMonIPs(mons []*monConfig) error {
|
||||
return c.ClusterInfo.Context.Err()
|
||||
}
|
||||
if m.UseHostNetwork {
|
||||
logger.Infof("setting mon %q endpoints for hostnetwork mode", m.DaemonName)
|
||||
log.NamespacedInfo(c.Namespace, logger, "setting mon %q endpoints for hostnetwork mode", m.DaemonName)
|
||||
node, ok := c.mapping.Schedule[m.DaemonName]
|
||||
if !ok || node == nil {
|
||||
return errors.Errorf("failed to find node for mon %q in assignment map", m.DaemonName)
|
||||
@@ -871,7 +872,7 @@ func (c *Cluster) initMonIPs(mons []*monConfig) error {
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to export service %q", monService.Name)
|
||||
}
|
||||
logger.Infof("mon %q exported IP is %s", m.DaemonName, exportedIP)
|
||||
log.NamespacedInfo(c.Namespace, logger, "mon %q exported IP is %s", m.DaemonName, exportedIP)
|
||||
m.PublicIP = exportedIP
|
||||
} else {
|
||||
m.PublicIP = monService.Spec.ClusterIP
|
||||
@@ -889,15 +890,15 @@ func (c *Cluster) initMonIPs(mons []*monConfig) error {
|
||||
func (c *Cluster) removeCanaryDeployments(labelSelector string) {
|
||||
canaryDeployments, err := k8sutil.GetDeployments(c.ClusterInfo.Context, c.context.Clientset, c.Namespace, labelSelector)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to get the list of monitor canary deployments. %v", err)
|
||||
log.NamespacedWarning(c.Namespace, logger, "failed to get the list of monitor canary deployments. %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Delete the canary mons
|
||||
for _, canary := range canaryDeployments.Items {
|
||||
logger.Infof("cleaning up canary monitor deployment %q", canary.Name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "cleaning up canary monitor deployment %q", canary.Name)
|
||||
if err := k8sutil.DeleteDeployment(c.ClusterInfo.Context, c.context.Clientset, c.Namespace, canary.Name); err != nil {
|
||||
logger.Warningf("failed to delete canary monitor deployment %q. %v", canary.Name, err)
|
||||
log.NamespacedWarning(c.Namespace, logger, "failed to delete canary monitor deployment %q. %v", canary.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -929,7 +930,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error {
|
||||
}
|
||||
// scheduling for this monitor has already been completed
|
||||
if _, ok := c.mapping.Schedule[mon.DaemonName]; ok {
|
||||
logger.Debugf("mon %s already scheduled", mon.DaemonName)
|
||||
log.NamespacedDebug(c.Namespace, logger, "mon %s already scheduled", mon.DaemonName)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -951,14 +952,14 @@ func (c *Cluster) assignMons(mons []*monConfig) error {
|
||||
|
||||
result, err := waitForMonitorScheduling(c, deployment)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to schedule mon %q. %v", mon.DaemonName, err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to schedule mon %q. %v", mon.DaemonName, err)
|
||||
failedMonSchedule = true
|
||||
return
|
||||
}
|
||||
|
||||
nodeChoice := result.Node
|
||||
if nodeChoice == nil {
|
||||
logger.Errorf("failed to schedule monitor %q", mon.DaemonName)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to schedule monitor %q", mon.DaemonName)
|
||||
failedMonSchedule = true
|
||||
return
|
||||
}
|
||||
@@ -968,21 +969,21 @@ func (c *Cluster) assignMons(mons []*monConfig) error {
|
||||
// directly to a node selector on the monitor pod.
|
||||
var schedule *controller.MonScheduleInfo
|
||||
if c.spec.Network.IsHost() || c.monVolumeClaimTemplate(mon) == nil {
|
||||
logger.Infof("mon %s assigned to node %s", mon.DaemonName, nodeChoice.Name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "mon %s assigned to node %s", mon.DaemonName, nodeChoice.Name)
|
||||
schedule, err = getNodeInfoFromNode(*nodeChoice)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get node info for node %q. %v", nodeChoice.Name, err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to get node info for node %q. %v", nodeChoice.Name, err)
|
||||
failedMonSchedule = true
|
||||
return
|
||||
}
|
||||
} else {
|
||||
logger.Infof("mon %q placement using native scheduler", mon.DaemonName)
|
||||
log.NamespacedInfo(c.Namespace, logger, "mon %q placement using native scheduler", mon.DaemonName)
|
||||
}
|
||||
if c.spec.ZonesRequired() {
|
||||
if schedule == nil {
|
||||
schedule = &controller.MonScheduleInfo{}
|
||||
}
|
||||
logger.Infof("mon %q is assigned to zone %q", mon.DaemonName, mon.Zone)
|
||||
log.NamespacedInfo(c.Namespace, logger, "mon %q is assigned to zone %q", mon.DaemonName, mon.Zone)
|
||||
schedule.Zone = mon.Zone
|
||||
}
|
||||
|
||||
@@ -998,7 +999,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error {
|
||||
return errors.New("failed to schedule mons")
|
||||
}
|
||||
|
||||
logger.Debug("mons have been scheduled")
|
||||
log.NamespacedDebug(c.Namespace, logger, "mons have been scheduled")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1040,10 +1041,10 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e
|
||||
deployments, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).List(c.ClusterInfo.Context, metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", AppName)})
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Infof("0 of %d expected mon deployments exist. creating new deployment(s).", len(mons))
|
||||
log.NamespacedInfo(c.Namespace, logger, "0 of %d expected mon deployments exist. creating new deployment(s).", len(mons))
|
||||
onlyCheckQuorumOnce = true
|
||||
} else {
|
||||
logger.Warningf("failed to list mon deployments. attempting to continue. %v", err)
|
||||
log.NamespacedWarning(c.Namespace, logger, "failed to list mon deployments. attempting to continue. %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1055,10 +1056,10 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e
|
||||
}
|
||||
}
|
||||
if len(deployments.Items) < len(mons) {
|
||||
logger.Infof("%d of %d expected mon deployments exist. creating new deployment(s).", len(deployments.Items), len(mons))
|
||||
log.NamespacedInfo(c.Namespace, logger, "%d of %d expected mon deployments exist. creating new deployment(s).", len(deployments.Items), len(mons))
|
||||
onlyCheckQuorumOnce = true
|
||||
} else if readyReplicas == 0 {
|
||||
logger.Infof("%d of %d expected mons are ready. creating or updating deployments without checking quorum in attempt to achieve a healthy mon cluster", readyReplicas, len(mons))
|
||||
log.NamespacedInfo(c.Namespace, logger, "%d of %d expected mons are ready. creating or updating deployments without checking quorum in attempt to achieve a healthy mon cluster", readyReplicas, len(mons))
|
||||
onlyCheckQuorumOnce = true
|
||||
}
|
||||
|
||||
@@ -1076,7 +1077,7 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e
|
||||
// we need to do everything possible to verify the basic health of a cluster, complete the first orchestration,
|
||||
// and start watching for all the CRs. If mons still have quorum we can continue with the orchestration even
|
||||
// if they aren't all up.
|
||||
logger.Errorf("attempting to continue after failing to start mon %q. %v", mons[i].DaemonName, err)
|
||||
log.NamespacedError(c.Namespace, logger, "attempting to continue after failing to start mon %q. %v", mons[i].DaemonName, err)
|
||||
}
|
||||
|
||||
// For the initial deployment (first creation) it's expected to not have all the monitors in quorum
|
||||
@@ -1091,13 +1092,13 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("mons created: %d", len(mons))
|
||||
log.NamespacedInfo(c.Namespace, logger, "mons created: %d", len(mons))
|
||||
// Final verification that **all** mons are in quorum
|
||||
// Do not proceed if one monitor is still syncing
|
||||
// Only do this when monitors versions are different so we don't block the orchestration if a mon is down.
|
||||
versions, err := cephclient.GetAllCephDaemonVersions(c.context, c.ClusterInfo)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to get ceph daemons versions; this likely means there is no cluster yet. %v", err)
|
||||
log.NamespacedWarning(c.Namespace, logger, "failed to get ceph daemons versions; this likely means there is no cluster yet. %v", err)
|
||||
} else {
|
||||
if len(versions.Mon) != 1 {
|
||||
requireAllInQuorum = true
|
||||
@@ -1113,20 +1114,20 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e
|
||||
func (c *Cluster) checkForExtraMonResources(mons []*monConfig, deployments []apps.Deployment) string {
|
||||
// If there are fewer mon deployments than the desired count, no need to remove an extra.
|
||||
if len(deployments) <= c.spec.Mon.Count || len(deployments) <= len(mons) {
|
||||
logger.Debug("no extra mon deployments to remove")
|
||||
log.NamespacedDebug(c.Namespace, logger, "no extra mon deployments to remove")
|
||||
return ""
|
||||
}
|
||||
// If there are fewer mons in the list than expected, either new mons are being created for
|
||||
// a new cluster, or a mon failover is in progress and the list of mons only
|
||||
// includes the single mon that was just started
|
||||
if len(mons) < c.spec.Mon.Count {
|
||||
logger.Debug("new cluster or mon failover in progress, not checking for extra mon deployments")
|
||||
log.NamespacedDebug(c.Namespace, logger, "new cluster or mon failover in progress, not checking for extra mon deployments")
|
||||
return ""
|
||||
}
|
||||
|
||||
// If there are more deployments than expected mons from the ceph quorum,
|
||||
// find the extra mon deployment and clean it up.
|
||||
logger.Infof("there is an extra mon deployment that is not needed and not in quorum")
|
||||
log.NamespacedInfo(c.Namespace, logger, "there is an extra mon deployment that is not needed and not in quorum")
|
||||
for _, deploy := range deployments {
|
||||
monName := deploy.Labels[controller.DaemonIDLabel]
|
||||
found := false
|
||||
@@ -1138,7 +1139,7 @@ func (c *Cluster) checkForExtraMonResources(mons []*monConfig, deployments []app
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
logger.Infof("deleting extra mon deployment %q", deploy.Name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "deleting extra mon deployment %q", deploy.Name)
|
||||
c.removeMonResources(monName)
|
||||
return monName
|
||||
}
|
||||
@@ -1227,7 +1228,7 @@ func (c *Cluster) persistExpectedMonDaemonsAsEndpointSlice() error {
|
||||
// Theoretically, we could now go ahead with the normal code path to
|
||||
// delete (both IPv4 and IPv6) EndpointSlices, but 0 mons is certainly
|
||||
// an error state, so it's better to do nothing destructive right now.
|
||||
logger.Debug("no mon addresses found, skipping endpointslice resource reconciliation")
|
||||
log.NamespacedDebug(c.Namespace, logger, "no mon addresses found, skipping endpointslice resource reconciliation")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1241,7 +1242,7 @@ func (c *Cluster) persistExpectedMonDaemonsAsEndpointSlice() error {
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
if ip == nil {
|
||||
logger.Warningf("invalid IP parsed from mon endpoint: %s", mon.Endpoint)
|
||||
log.NamespacedWarning(c.Namespace, logger, "invalid IP parsed from mon endpoint: %s", mon.Endpoint)
|
||||
continue
|
||||
}
|
||||
if ip.To4() != nil {
|
||||
@@ -1271,12 +1272,12 @@ func (c *Cluster) createEndpointSliceForAddresses(addresses []string, addressTyp
|
||||
}
|
||||
|
||||
if len(addresses) == 0 {
|
||||
logger.Debugf("no %s addresses found, deleting existing %q endpointslice if exists", addressType, sliceName)
|
||||
log.NamespacedDebug(c.Namespace, logger, "no %s addresses found, deleting existing %q endpointslice if exists", addressType, sliceName)
|
||||
if err := client.Delete(c.ClusterInfo.Context, sliceName, metav1.DeleteOptions{}); err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Debugf("endpointslice %q not found, nothing to delete", sliceName)
|
||||
log.NamespacedDebug(c.Namespace, logger, "endpointslice %q not found, nothing to delete", sliceName)
|
||||
} else {
|
||||
logger.Errorf("failed to delete endpointslice %q: %v", sliceName, err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to delete endpointslice %q: %v", sliceName, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -1324,13 +1325,13 @@ func (c *Cluster) createEndpointSliceForAddresses(addresses []string, addressTyp
|
||||
return errors.Wrapf(err, "failed to create %s endpointslice", addressType)
|
||||
}
|
||||
|
||||
logger.Debugf("updating existing %s endpointslice %s", addressType, sliceName)
|
||||
log.NamespacedDebug(c.Namespace, logger, "updating existing %s endpointslice %s", addressType, sliceName)
|
||||
if _, err = client.Update(c.ClusterInfo.Context, endpointSlice, metav1.UpdateOptions{}); err != nil {
|
||||
return errors.Wrapf(err, "failed to update %s endpointslice", addressType)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("created/updated %s endpointslice with addresses: %+v", addressType, addresses)
|
||||
log.NamespacedInfo(c.Namespace, logger, "created/updated %s endpointslice with addresses: %+v", addressType, addresses)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1397,12 +1398,12 @@ func (c *Cluster) persistExpectedMonDaemonsInConfigMap() error {
|
||||
return errors.Wrap(err, "failed to create mon endpoint config map")
|
||||
}
|
||||
|
||||
logger.Debugf("updating config map %s that already exists", configMap.Name)
|
||||
log.NamespacedDebug(c.Namespace, logger, "updating config map %s that already exists", configMap.Name)
|
||||
if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(c.ClusterInfo.Context, configMap, metav1.UpdateOptions{}); err != nil {
|
||||
return errors.Wrap(err, "failed to update mon endpoint config map")
|
||||
}
|
||||
}
|
||||
logger.Infof("saved mon endpoints to config map %+v", configMap.Data)
|
||||
log.NamespacedInfo(c.Namespace, logger, "saved mon endpoints to config map %+v", configMap.Data)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1420,7 +1421,7 @@ func (c *Cluster) getStoredMaxMonID() (string, error) {
|
||||
// if the configmap cannot be loaded, assume a new cluster. If the mons have previously
|
||||
// been created, the maxMonID will anyway analyze them to ensure the index is correct
|
||||
// even if this error occurs.
|
||||
logger.Infof("existing maxMonID not found or failed to load. %v", err)
|
||||
log.NamespacedInfo(c.Namespace, logger, "existing maxMonID not found or failed to load. %v", err)
|
||||
return "-1", nil
|
||||
}
|
||||
|
||||
@@ -1446,11 +1447,11 @@ func (c *Cluster) commitMaxMonIDRequireIncrementing(desiredMaxMonID int, require
|
||||
}
|
||||
|
||||
if requireIncrementing && existingMax >= desiredMaxMonID {
|
||||
logger.Infof("no need to commit maxMonID %d since it is not greater than existing maxMonID %d", desiredMaxMonID, existingMax)
|
||||
log.NamespacedInfo(c.Namespace, logger, "no need to commit maxMonID %d since it is not greater than existing maxMonID %d", desiredMaxMonID, existingMax)
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Infof("updating maxMonID from %d to %d", existingMax, desiredMaxMonID)
|
||||
log.NamespacedInfo(c.Namespace, logger, "updating maxMonID from %d to %d", existingMax, desiredMaxMonID)
|
||||
configmap.Data[controller.MaxMonIDKey] = strconv.Itoa(desiredMaxMonID)
|
||||
|
||||
if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(c.ClusterInfo.Context, configmap, metav1.UpdateOptions{}); err != nil {
|
||||
@@ -1476,7 +1477,7 @@ func (c *Cluster) updateMon(m *monConfig, d *apps.Deployment) error {
|
||||
_ = k8sutil.ExpandPVCIfRequired(c.ClusterInfo.Context, c.context.Client, desiredPvc, existingPvc)
|
||||
}
|
||||
|
||||
logger.Infof("deployment for mon %s already exists. updating if needed",
|
||||
log.NamespacedInfo(c.Namespace, logger, "deployment for mon %s already exists. updating if needed",
|
||||
d.Name)
|
||||
|
||||
err := updateDeploymentAndWait(c.context, c.ClusterInfo, d, config.MonType, m.DaemonName, c.spec.SkipUpgradeChecks, false)
|
||||
@@ -1519,6 +1520,7 @@ func (c *Cluster) updateMon(m *monConfig, d *apps.Deployment) error {
|
||||
func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) error {
|
||||
// check if the monitor deployment already exists. if the deployment does
|
||||
// exist, also determine if it using pvc storage.
|
||||
log.NamespacedInfo(c.Namespace, logger, "starting mon %q", m.DaemonName)
|
||||
pvcExists := false
|
||||
deploymentExists := false
|
||||
|
||||
@@ -1554,10 +1556,10 @@ func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) e
|
||||
pvcName := m.ResourceName
|
||||
d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, controller.DaemonVolumesDataPVC(pvcName))
|
||||
controller.AddVolumeMountSubPath(&d.Spec.Template.Spec, "ceph-daemon-data")
|
||||
logger.Debugf("adding pvc volume source %s to mon deployment %s", pvcName, d.Name)
|
||||
log.NamespacedDebug(c.Namespace, logger, "adding pvc volume source %s to mon deployment %s", pvcName, d.Name)
|
||||
} else {
|
||||
d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, controller.DaemonVolumesDataHostPath(m.DataPathMap)...)
|
||||
logger.Debugf("adding host path volume source to mon deployment %s", d.Name)
|
||||
log.NamespacedDebug(c.Namespace, logger, "adding host path volume source to mon deployment %s", d.Name)
|
||||
}
|
||||
|
||||
// placement settings from the CRD
|
||||
@@ -1611,7 +1613,7 @@ func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) e
|
||||
_, err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Create(c.ClusterInfo.Context, pvc, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
if kerrors.IsAlreadyExists(err) {
|
||||
logger.Debugf("cannot create mon %s pvc %s: already exists.", d.Name, pvc.Name)
|
||||
log.NamespacedDebug(c.Namespace, logger, "cannot create mon %s pvc %s: already exists.", d.Name, pvc.Name)
|
||||
} else {
|
||||
return errors.Wrapf(err, "failed to create mon %s pvc %s", d.Name, pvc.Name)
|
||||
}
|
||||
@@ -1631,7 +1633,7 @@ func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) e
|
||||
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), k8sutil.LabelHostname(),
|
||||
map[string]string{k8sutil.AppAttr: AppName}, nodeSelector)
|
||||
|
||||
logger.Debugf("Starting mon: %+v", d.Name)
|
||||
log.NamespacedDebug(c.Namespace, logger, "Starting mon: %+v", d.Name)
|
||||
_, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(c.ClusterInfo.Context, d, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to create mon deployment %s", d.Name)
|
||||
@@ -1667,10 +1669,10 @@ func isMonIPUpdateRequiredForHostNetwork(mon string, isMonUsingHostNetwork bool,
|
||||
|
||||
func hasMonPathChanged(d *apps.Deployment, claim *corev1.PersistentVolumeClaim) bool {
|
||||
if d.Labels["pvc_name"] == "" && claim != nil {
|
||||
logger.Infof("skipping update for mon %q where path has changed from hostPath to pvc", d.Name)
|
||||
log.NamespacedInfo(d.Namespace, logger, "skipping update for mon %q where path has changed from hostPath to pvc", d.Name)
|
||||
return true
|
||||
} else if d.Labels["pvc_name"] != "" && claim == nil {
|
||||
logger.Infof("skipping update for mon %q where path has changed from pvc to hostPath", d.Name)
|
||||
log.NamespacedInfo(d.Namespace, logger, "skipping update for mon %q where path has changed from pvc to hostPath", d.Name)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -1678,7 +1680,7 @@ func hasMonPathChanged(d *apps.Deployment, claim *corev1.PersistentVolumeClaim)
|
||||
}
|
||||
|
||||
func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, mons []string, sleepTime int, requireAllInQuorum bool) error {
|
||||
logger.Infof("waiting for mon quorum with %v", mons)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "waiting for mon quorum with %v", mons)
|
||||
|
||||
// wait for monitors to establish quorum
|
||||
retryCount := 0
|
||||
@@ -1705,18 +1707,18 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.Cl
|
||||
for _, m := range mons {
|
||||
running, err := k8sutil.PodsRunningWithLabel(clusterInfo.Context, context.Clientset, clusterInfo.Namespace, fmt.Sprintf("app=%s,mon=%s", AppName, m))
|
||||
if err != nil {
|
||||
logger.Infof("failed to query mon pod status, trying again. %v", err)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "failed to query mon pod status, trying again. %v", err)
|
||||
continue
|
||||
}
|
||||
if running > 0 {
|
||||
runningMonNames = append(runningMonNames, m)
|
||||
} else {
|
||||
allPodsRunning = false
|
||||
logger.Infof("mon %s is not yet running", m)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "mon %s is not yet running", m)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("mons running: %v", runningMonNames)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "mons running: %v", runningMonNames)
|
||||
if !allPodsRunning && requireAllInQuorum {
|
||||
continue
|
||||
}
|
||||
@@ -1725,12 +1727,12 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.Cl
|
||||
// their quorum status
|
||||
monQuorumStatusResp, err := cephclient.GetMonQuorumStatus(context, clusterInfo)
|
||||
if err != nil {
|
||||
logger.Debugf("failed to get quorum_status. %v", err)
|
||||
log.NamespacedDebug(clusterInfo.Namespace, logger, "failed to get quorum_status. %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if !requireAllInQuorum {
|
||||
logQuorumMembers(monQuorumStatusResp)
|
||||
logQuorumMembers(clusterInfo.Namespace, monQuorumStatusResp)
|
||||
break
|
||||
}
|
||||
|
||||
@@ -1739,14 +1741,14 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.Cl
|
||||
for _, name := range mons {
|
||||
if !monFoundInQuorum(name, monQuorumStatusResp) {
|
||||
// found an initial monitor that is not in quorum, bail out of this retry
|
||||
logger.Warningf("monitor %s is not in quorum list", name)
|
||||
log.NamespacedWarning(clusterInfo.Namespace, logger, "monitor %s is not in quorum list", name)
|
||||
allInQuorum = false
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if allInQuorum {
|
||||
logQuorumMembers(monQuorumStatusResp)
|
||||
logQuorumMembers(clusterInfo.Namespace, monQuorumStatusResp)
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -1754,14 +1756,14 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.Cl
|
||||
return nil
|
||||
}
|
||||
|
||||
func logQuorumMembers(monQuorumStatusResp cephclient.MonStatusResponse) {
|
||||
func logQuorumMembers(namespace string, monQuorumStatusResp cephclient.MonStatusResponse) {
|
||||
var monsInQuorum []string
|
||||
for _, m := range monQuorumStatusResp.MonMap.Mons {
|
||||
if monFoundInQuorum(m.Name, monQuorumStatusResp) {
|
||||
monsInQuorum = append(monsInQuorum, m.Name)
|
||||
}
|
||||
}
|
||||
logger.Infof("Monitors in quorum: %v", monsInQuorum)
|
||||
log.NamespacedInfo(namespace, logger, "Monitors in quorum: %v", monsInQuorum)
|
||||
}
|
||||
|
||||
func monFoundInQuorum(name string, monQuorumStatusResp cephclient.MonStatusResponse) bool {
|
||||
@@ -1796,14 +1798,14 @@ func requiredDuringScheduling(spec *cephv1.ClusterSpec) bool {
|
||||
}
|
||||
|
||||
func (c *Cluster) acquireOrchestrationLock() {
|
||||
logger.Debugf("Acquiring lock for mon orchestration")
|
||||
log.NamespacedDebug(c.Namespace, logger, "Acquiring lock for mon orchestration")
|
||||
c.orchestrationMutex.Lock()
|
||||
logger.Debugf("Acquired lock for mon orchestration")
|
||||
log.NamespacedDebug(c.Namespace, logger, "Acquired lock for mon orchestration")
|
||||
}
|
||||
|
||||
func (c *Cluster) releaseOrchestrationLock() {
|
||||
c.orchestrationMutex.Unlock()
|
||||
logger.Debugf("Released lock for mon orchestration")
|
||||
log.NamespacedDebug(c.Namespace, logger, "Released lock for mon orchestration")
|
||||
}
|
||||
|
||||
func (c *Cluster) RotateMonCephxKeys(clusterObj *cephv1.CephCluster) (bool, error) {
|
||||
@@ -1817,16 +1819,16 @@ func (c *Cluster) RotateMonCephxKeys(clusterObj *cephv1.CephCluster) (bool, erro
|
||||
}
|
||||
|
||||
if !shouldRotateMonKeys {
|
||||
logger.Debugf("cephx key rotation for mon daemon in the namespace %q is not required", c.ClusterInfo.Namespace)
|
||||
log.NamespacedDebug(c.Namespace, logger, "cephx key rotation for mon daemon in the namespace %q is not required", c.ClusterInfo.Namespace)
|
||||
return shouldRotateMonKeys, nil
|
||||
}
|
||||
|
||||
if !c.ClusterInfo.CephVersion.IsAtLeast(keyring.CephAuthMonRotateSupportedVersion) {
|
||||
logger.Debugf("cephx key rotation for mons in namespace %q is indicated, but ceph version %#v does not support mon key rotation", c.Namespace, c.ClusterInfo.CephVersion)
|
||||
log.NamespacedDebug(c.Namespace, logger, "cephx key rotation for mons in namespace %q is indicated, but ceph version %#v does not support mon key rotation", c.Namespace, c.ClusterInfo.CephVersion)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
logger.Infof("cephx keys for mon daemons in the namespace %q will be rotated", c.ClusterInfo.Namespace)
|
||||
log.NamespacedInfo(c.Namespace, logger, "cephx keys for mon daemons in the namespace %q will be rotated", c.ClusterInfo.Namespace)
|
||||
|
||||
k := keyring.GetSecretStore(c.context, c.ClusterInfo, c.ClusterInfo.OwnerInfo)
|
||||
newKey, err := k.RotateKey(controller.MonCephxUser)
|
||||
@@ -1847,7 +1849,7 @@ func (c *Cluster) RotateMonCephxKeys(clusterObj *cephv1.CephCluster) (bool, erro
|
||||
return shouldRotateMonKeys, errors.Wrapf(err, "failed to save mon keyring secret after rotating the mon cephx keys in the namespace %q", c.ClusterInfo.Namespace)
|
||||
}
|
||||
|
||||
logger.Infof("successfully rotated cephx keys for mon daemons in the namespace %q", c.ClusterInfo.Namespace)
|
||||
log.NamespacedInfo(c.Namespace, logger, "successfully rotated cephx keys for mon daemons in the namespace %q", c.ClusterInfo.Namespace)
|
||||
|
||||
return shouldRotateMonKeys, nil
|
||||
}
|
||||
@@ -1860,11 +1862,11 @@ func (c *Cluster) UpdateMonCephxStatus(didRotate bool) error {
|
||||
}
|
||||
updatedStatus := keyring.UpdatedCephxStatus(didRotate, cluster.Spec.Security.CephX.Daemon, c.ClusterInfo.CephVersion, cluster.Status.Cephx.Mon)
|
||||
cluster.Status.Cephx.Mon = updatedStatus
|
||||
logger.Debugf("updating mon daemon cephx status to %+v", cluster.Status.Cephx.Mgr)
|
||||
log.NamespacedDebug(c.Namespace, logger, "updating mon daemon cephx status to %+v", cluster.Status.Cephx.Mgr)
|
||||
if err := reporting.UpdateStatus(c.context.Client, cluster); err != nil {
|
||||
return errors.Wrapf(err, "failed to update cluster cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace)
|
||||
}
|
||||
logger.Infof("successfully updated the cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace)
|
||||
log.NamespacedInfo(c.Namespace, logger, "successfully updated the cephx status for mon daemon in the namespace %q", c.ClusterInfo.Namespace)
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"sort"
|
||||
|
||||
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/event"
|
||||
"sigs.k8s.io/controller-runtime/pkg/predicate"
|
||||
@@ -42,8 +43,8 @@ func PredicateMonEndpointChanges[T *corev1.ConfigMap]() predicate.TypedFuncs[T]
|
||||
cmOld := (*corev1.ConfigMap)(e.ObjectOld)
|
||||
cmNew := (*corev1.ConfigMap)(e.ObjectNew)
|
||||
if cmNew.GetName() == EndpointConfigMapName {
|
||||
if wereMonEndpointsUpdated(cmOld.Data, cmNew.Data) {
|
||||
logger.Info("monitor endpoints changed, updating the bootstrap peer token")
|
||||
if wereMonEndpointsUpdated(cmNew.Namespace, cmOld.Data, cmNew.Data) {
|
||||
log.NamespacedInfo(cmNew.Namespace, logger, "monitor endpoints changed, updating the bootstrap peer token")
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -52,7 +53,7 @@ func PredicateMonEndpointChanges[T *corev1.ConfigMap]() predicate.TypedFuncs[T]
|
||||
}
|
||||
}
|
||||
|
||||
func wereMonEndpointsUpdated(oldCMData, newCMData map[string]string) bool {
|
||||
func wereMonEndpointsUpdated(namespace string, oldCMData, newCMData map[string]string) bool {
|
||||
// Check the mapping key first
|
||||
if oldMapping, ok := oldCMData["mapping"]; ok {
|
||||
if newMapping, ok := newCMData["mapping"]; ok {
|
||||
@@ -60,20 +61,20 @@ func wereMonEndpointsUpdated(oldCMData, newCMData map[string]string) bool {
|
||||
var oldMappingToGo opcontroller.Mapping
|
||||
err := json.Unmarshal([]byte(oldMapping), &oldMappingToGo)
|
||||
if err != nil {
|
||||
logger.Debugf("failed to unmarshal new. %v", err)
|
||||
log.NamespacedDebug(namespace, logger, "failed to unmarshal new. %v", err)
|
||||
return false
|
||||
}
|
||||
|
||||
var newMappingToGo opcontroller.Mapping
|
||||
err = json.Unmarshal([]byte(newMapping), &newMappingToGo)
|
||||
if err != nil {
|
||||
logger.Debugf("failed to unmarshal new. %v", err)
|
||||
log.NamespacedDebug(namespace, logger, "failed to unmarshal new. %v", err)
|
||||
return false
|
||||
}
|
||||
|
||||
// If length is different, monitors are different
|
||||
if len(oldMappingToGo.Schedule) != len(newMappingToGo.Schedule) {
|
||||
logger.Debugf("mons were added or removed from the endpoints cm")
|
||||
log.NamespacedDebug(namespace, logger, "mons were added or removed from the endpoints cm")
|
||||
return true
|
||||
}
|
||||
// Since Schedule is map, it's unordered, so let's order it
|
||||
@@ -92,7 +93,7 @@ func wereMonEndpointsUpdated(oldCMData, newCMData map[string]string) bool {
|
||||
// Iterate over the map and compare the values
|
||||
for _, v := range oldKeys {
|
||||
if !reflect.DeepEqual(oldMappingToGo.Schedule[v], newMappingToGo.Schedule[v]) {
|
||||
logger.Debugf("oldMappingToGo.Schedule[v] AND newMappingToGo.Schedule[v]: %v | %v", oldMappingToGo.Schedule[v], newMappingToGo.Schedule[v])
|
||||
log.NamespacedDebug(namespace, logger, "oldMappingToGo.Schedule[v] AND newMappingToGo.Schedule[v]: %v | %v", oldMappingToGo.Schedule[v], newMappingToGo.Schedule[v])
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ func TestWereMonEndpointsUpdated(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := wereMonEndpointsUpdated(tt.args.oldCMData, tt.args.newCMData); got != tt.want {
|
||||
if got := wereMonEndpointsUpdated("ns", tt.args.oldCMData, tt.args.newCMData); got != tt.want {
|
||||
t.Errorf("whereMonEndpointsUpdated() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -54,7 +55,7 @@ func (c *Cluster) createService(mon *monConfig) (*v1.Service, error) {
|
||||
if mon.PublicIP != "" {
|
||||
_, err := c.context.Clientset.CoreV1().Services(c.Namespace).Get(c.ClusterInfo.Context, svcDef.Name, metav1.GetOptions{})
|
||||
if err != nil && kerrors.IsNotFound(err) {
|
||||
logger.Infof("ensuring the clusterIP for mon %q is %q", mon.DaemonName, mon.PublicIP)
|
||||
log.NamespacedInfo(c.Namespace, logger, "ensuring the clusterIP for mon %q is %q", mon.DaemonName, mon.PublicIP)
|
||||
svcDef.Spec.ClusterIP = mon.PublicIP
|
||||
}
|
||||
}
|
||||
@@ -69,7 +70,7 @@ func (c *Cluster) createService(mon *monConfig) (*v1.Service, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
logger.Infof("mon %q cluster IP is %s", mon.DaemonName, s.Spec.ClusterIP)
|
||||
log.NamespacedInfo(c.Namespace, logger, "mon %q cluster IP is %s", mon.DaemonName, s.Spec.ClusterIP)
|
||||
return s, nil
|
||||
}
|
||||
|
||||
@@ -78,12 +79,12 @@ func (c *Cluster) exportService(service *v1.Service, monDaemon string) (string,
|
||||
// query on <service>.<ns>.svc.clusterset.local requires the mon canary pod to be running
|
||||
defer c.removeCanaryDeployments(monCanaryLabelSelector + fmt.Sprintf(",mon=%s", monDaemon))
|
||||
|
||||
logger.Infof("exporting service %q", service.Name)
|
||||
log.NamespacedInfo(c.Namespace, logger, "exporting service %q", service.Name)
|
||||
exportedIP, err := k8sutil.ExportService(c.ClusterInfo.Context, c.context, service, c.spec.Network.MultiClusterService.ClusterID)
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "failed to export service %q", service.Name)
|
||||
}
|
||||
logger.Infof("mon %q exported IP is %s", service.Name, exportedIP)
|
||||
log.NamespacedInfo(c.Namespace, logger, "mon %q exported IP is %s", service.Name, exportedIP)
|
||||
|
||||
return exportedIP, nil
|
||||
}
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
@@ -187,7 +188,7 @@ func makeMonSecurityContext() *corev1.PodSecurityContext {
|
||||
}
|
||||
|
||||
func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*corev1.Pod, error) {
|
||||
logger.Debugf("monConfig: %+v", monConfig)
|
||||
log.NamespacedDebug(c.Namespace, logger, "monConfig: %+v", monConfig)
|
||||
podSpec := corev1.PodSpec{
|
||||
InitContainers: []corev1.Container{
|
||||
c.makeChownInitContainer(monConfig),
|
||||
@@ -393,17 +394,16 @@ func UpdateCephDeploymentAndWait(context *clusterd.Context, clusterInfo *client.
|
||||
callback := func(action string) error {
|
||||
// At this point, we are in an upgrade
|
||||
if skipUpgradeChecks {
|
||||
logger.Warningf("this is a Ceph upgrade, not performing upgrade checks because skipUpgradeChecks is %t", skipUpgradeChecks)
|
||||
log.NamespacedWarning(clusterInfo.Namespace, logger, "this is a Ceph upgrade, not performing upgrade checks because skipUpgradeChecks is %t", skipUpgradeChecks)
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Infof("checking if we can %s the deployment %s", action, deployment.Name)
|
||||
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "checking if we can %s the deployment %s", action, deployment.Name)
|
||||
if action == "stop" {
|
||||
err := client.OkToStop(context, clusterInfo, deployment.Name, daemonType, daemonName)
|
||||
if err != nil {
|
||||
if continueUpgradeAfterChecksEvenIfNotHealthy {
|
||||
logger.Infof("The %s daemon %s is not ok-to-stop but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so proceeding to stop...", daemonType, daemonName)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "The %s daemon %s is not ok-to-stop but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so proceeding to stop...", daemonType, daemonName)
|
||||
return nil
|
||||
}
|
||||
return errors.Wrapf(err, "failed to check if we can %s the deployment %s", action, deployment.Name)
|
||||
@@ -414,7 +414,7 @@ func UpdateCephDeploymentAndWait(context *clusterd.Context, clusterInfo *client.
|
||||
err := client.OkToContinue(context, clusterInfo, deployment.Name, daemonType, daemonName)
|
||||
if err != nil {
|
||||
if continueUpgradeAfterChecksEvenIfNotHealthy {
|
||||
logger.Infof("The %s daemon %s is not ok-to-continue but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so continuing...", daemonType, daemonName)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "The %s daemon %s is not ok-to-continue but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so continuing...", daemonType, daemonName)
|
||||
return nil
|
||||
}
|
||||
return errors.Wrapf(err, "failed to check if we can %s the deployment %s", action, deployment.Name)
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
|
||||
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
)
|
||||
|
||||
var monitorDaemonList = []string{"mon", "osd", "status"}
|
||||
@@ -37,7 +38,7 @@ func (c *ClusterController) configureCephMonitoring(cluster *cluster, clusterInf
|
||||
if health, ok := cluster.monitoringRoutines.Load(daemon); ok {
|
||||
// If the context Err() is nil this means it hasn't been cancelled yet
|
||||
if health.(*opcontroller.ClusterHealth).InternalCtx.Err() == nil {
|
||||
logger.Debugf("monitoring routine for %q is already running", daemon)
|
||||
log.NamespacedDebug(cluster.Namespace, logger, "monitoring routine for %q is already running", daemon)
|
||||
if !isEnabled {
|
||||
health.(*opcontroller.ClusterHealth).InternalCancel()
|
||||
}
|
||||
@@ -79,19 +80,19 @@ func (c *ClusterController) startMonitoringCheck(cluster *cluster, clusterInfo *
|
||||
switch daemon {
|
||||
case "mon":
|
||||
healthChecker := mon.NewHealthChecker(cluster.mons)
|
||||
logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "enabling ceph %s monitoring goroutine", daemon)
|
||||
go healthChecker.Check(&cluster.monitoringRoutines, daemon)
|
||||
|
||||
case "osd":
|
||||
if !cluster.Spec.External.Enable {
|
||||
osdChecker := osd.NewOSDHealthMonitor(c.context, clusterInfo, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Spec.HealthCheck)
|
||||
logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "enabling ceph %s monitoring goroutine", daemon)
|
||||
go osdChecker.Start(&cluster.monitoringRoutines, daemon)
|
||||
}
|
||||
|
||||
case "status":
|
||||
cephChecker := newCephStatusChecker(c.context, clusterInfo, cluster.Spec)
|
||||
logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "enabling ceph %s monitoring goroutine", daemon)
|
||||
go cephChecker.checkCephStatus(&cluster.monitoringRoutines, daemon)
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
@@ -57,11 +58,11 @@ func (r *ReconcileNode) createOrUpdateCephExporter(node corev1.Node, tolerations
|
||||
// CephVersion change is done temporarily, as some regression was detected in Ceph version 17.2.6 which is summarised here https://github.com/ceph/ceph/pull/50718#issuecomment-1505608312.
|
||||
// Thus, disabling ceph-exporter for now until all the regression are fixed.
|
||||
if !cephVersion.IsAtLeast(MinVersionForCephExporter) {
|
||||
logger.Infof("Skipping exporter reconcile on ceph version %q", cephVersion.String())
|
||||
log.NamespacedInfo(cephCluster.Namespace, logger, "Skipping exporter reconcile on ceph version %q", cephVersion.String())
|
||||
return controllerutil.OperationResultNone, nil
|
||||
}
|
||||
if cephCluster.Spec.Monitoring.MetricsDisabled {
|
||||
logger.Info("Skipping exporter reconcile since monitoring.metricsDisabled is true")
|
||||
log.NamespacedInfo(cephCluster.Namespace, logger, "Skipping exporter reconcile since monitoring.metricsDisabled is true")
|
||||
return controllerutil.OperationResultNone, nil
|
||||
}
|
||||
|
||||
@@ -283,10 +284,10 @@ func applyCephExporterLabels(cephCluster cephv1.CephCluster, serviceMonitor *mon
|
||||
serviceMonitor.Spec.Endpoints[0].RelabelConfigs = append(
|
||||
serviceMonitor.Spec.Endpoints[0].RelabelConfigs, relabelConfig)
|
||||
} else {
|
||||
logger.Info("rook.io/managedBy not specified in ceph-exporter labels")
|
||||
log.NamespacedInfo(cephCluster.Namespace, logger, "rook.io/managedBy not specified in ceph-exporter labels")
|
||||
}
|
||||
} else {
|
||||
logger.Debug("ceph-exporter labels not specified")
|
||||
log.NamespacedDebug(cephCluster.Namespace, logger, "ceph-exporter labels not specified")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
"github.com/rook/rook/pkg/operator/ceph/reporting"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/util/retry"
|
||||
@@ -96,7 +97,7 @@ func createCrashCollectorKeyring(s *keyring.SecretStore, context *clusterd.Conte
|
||||
}
|
||||
|
||||
if shouldRotateCephxKeys {
|
||||
logger.Infof("rotating cephx key for crash collector %q", crashCollectorKeyringUsername)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "rotating cephx key for crash collector %q", crashCollectorKeyringUsername)
|
||||
newKey, err := s.RotateKey(crashCollectorKeyringUsername)
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "failed to rotate cephx key for crash collector %q", crashCollectorKeyringUsername)
|
||||
@@ -124,7 +125,7 @@ func updateCrashCollectorCephxStatus(context *clusterd.Context, clusterInfo *cli
|
||||
if err := reporting.UpdateStatus(context.Client, cluster); err != nil {
|
||||
return errors.Wrap(err, "failed to update cluster cephx status for crash collector daemon")
|
||||
}
|
||||
logger.Debugf("successfully updated the crash collector cephx status for cluster in namespace %q to %+v", cluster.Namespace, cluster.Status.Cephx.CrashCollector)
|
||||
log.NamespacedDebug(clusterInfo.Namespace, logger, "successfully updated the crash collector cephx status for cluster in namespace %q to %+v", cluster.Namespace, cluster.Status.Cephx.CrashCollector)
|
||||
|
||||
return nil
|
||||
})
|
||||
@@ -161,7 +162,7 @@ func createOrUpdateCrashCollectorSecret(clusterInfo *client.ClusterInfo, crashCo
|
||||
return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", s.Name, clusterInfo.Namespace)
|
||||
}
|
||||
|
||||
logger.Infof("created kubernetes crash collector secret for cluster %q", clusterInfo.Namespace)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "created kubernetes crash collector secret")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -212,7 +213,7 @@ func createExporterKeyring(s *keyring.SecretStore, context *clusterd.Context, cl
|
||||
}
|
||||
|
||||
if shouldRotateCephxKeys {
|
||||
logger.Infof("rotating cephx key for ceph exporter %q", exporterKeyringUsername)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "rotating cephx key for ceph exporter %q", exporterKeyringUsername)
|
||||
newKey, err := s.RotateKey(exporterKeyringUsername)
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "failed to rotate cephx key for ceph exporter %q", exporterKeyringUsername)
|
||||
@@ -239,7 +240,7 @@ func updateCephExporterCephxStatus(context *clusterd.Context, clusterInfo *clien
|
||||
if err := reporting.UpdateStatus(context.Client, cluster); err != nil {
|
||||
return errors.Wrap(err, "failed to update cluster cephx status for ceph exporter daemon")
|
||||
}
|
||||
logger.Debugf("successfully updated the ceph exporter cephx status for cluster in namespace %q to %+v", cluster.Namespace, cluster.Status.Cephx.CephExporter)
|
||||
log.NamespacedDebug(clusterInfo.Namespace, logger, "successfully updated the ceph exporter cephx status for cluster in namespace %q to %+v", cluster.Namespace, cluster.Status.Cephx.CephExporter)
|
||||
|
||||
return nil
|
||||
})
|
||||
@@ -276,6 +277,6 @@ func createOrUpdateExporterSecret(clusterInfo *client.ClusterInfo, exporterSecre
|
||||
return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", s.Name, clusterInfo.Namespace)
|
||||
}
|
||||
|
||||
logger.Infof("created kubernetes exporter secret for cluster %q", clusterInfo.Namespace)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "created kubernetes exporter secret for cluster %q", clusterInfo.Namespace)
|
||||
return nil
|
||||
}
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
v1 "k8s.io/api/batch/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
@@ -34,7 +35,7 @@ import (
|
||||
|
||||
func (r *ReconcileNode) reconcileCrashPruner(namespace string, cephCluster cephv1.CephCluster, tolerations []corev1.Toleration) error {
|
||||
if cephCluster.Spec.CrashCollector.Disable {
|
||||
logger.Debugf("crash collector is disabled in namespace %q so skipping crash retention reconcile", namespace)
|
||||
log.NamespacedDebug(cephCluster.Namespace, logger, "crash collector is disabled in namespace %q so skipping crash retention reconcile", namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -44,27 +45,27 @@ func (r *ReconcileNode) reconcileCrashPruner(namespace string, cephCluster cephv
|
||||
}
|
||||
|
||||
if cephCluster.Spec.CrashCollector.DaysToRetain == 0 {
|
||||
logger.Debug("deleting cronjob if it exists...")
|
||||
log.NamespacedDebug(cephCluster.Namespace, logger, "deleting cronjob if it exists...")
|
||||
|
||||
cronJob := &v1.CronJob{ObjectMeta: objectMeta}
|
||||
|
||||
err := r.client.Delete(r.opManagerContext, cronJob)
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Debug("cronJob resource not found. Ignoring since object must be deleted.")
|
||||
log.NamespacedDebug(cephCluster.Namespace, logger, "cronJob resource not found. Ignoring since object must be deleted.")
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
logger.Debug("successfully deleted crash pruner cronjob.")
|
||||
log.NamespacedDebug(cephCluster.Namespace, logger, "successfully deleted crash pruner cronjob.")
|
||||
}
|
||||
} else {
|
||||
logger.Debugf("daysToRetain set to: %d", cephCluster.Spec.CrashCollector.DaysToRetain)
|
||||
log.NamespacedDebug(cephCluster.Namespace, logger, "daysToRetain set to: %d", cephCluster.Spec.CrashCollector.DaysToRetain)
|
||||
op, err := r.createOrUpdateCephCron(cephCluster, tolerations)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "node reconcile failed on op %q", op)
|
||||
}
|
||||
logger.Debugf("cronjob successfully reconciled. operation: %q", op)
|
||||
log.NamespacedDebug(cephCluster.Namespace, logger, "cronjob successfully reconciled. operation: %q", op)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/rbd"
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
@@ -77,7 +78,7 @@ func (r *ReconcileNode) Reconcile(context context.Context, request reconcile.Req
|
||||
// workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface
|
||||
result, err := r.reconcile(request)
|
||||
if err != nil {
|
||||
logger.Error(err)
|
||||
log.NamespacedError(request.Namespace, logger, "%s", err.Error())
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
@@ -95,7 +96,7 @@ func (r *ReconcileNode) cleanupExporterResources(ns string, nodeName string) (re
|
||||
}
|
||||
|
||||
func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result, error) {
|
||||
logger.Debugf("reconciling node: %q", request.Name)
|
||||
log.NamespacedDebug(request.Namespace, logger, "reconciling node: %q", request.Name)
|
||||
|
||||
// get the node object
|
||||
node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: request.Name}}
|
||||
@@ -105,7 +106,7 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result,
|
||||
// if a node is not present, check if there are any node daemons to remove
|
||||
err := r.listNodeDaemonsAndDelete(request.Name, "")
|
||||
if err != nil {
|
||||
logger.Errorf("failed to list and delete deployment on node %q; user should delete them manually. %v", request.Name, err)
|
||||
log.NamespacedError(request.Namespace, logger, "failed to list and delete deployment on node %q; user should delete them manually. %v", request.Name, err)
|
||||
}
|
||||
} else {
|
||||
return reconcile.Result{}, errors.Wrapf(err, "could not get node %q", request.Name)
|
||||
@@ -144,13 +145,13 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result,
|
||||
return reconcile.Result{}, errors.Wrapf(err, "could not get cephcluster in namespaces %q", namespace)
|
||||
}
|
||||
if len(cephClusters.Items) < 1 {
|
||||
logger.Debugf("no CephCluster found in the namespace %q", namespace)
|
||||
log.NamespacedDebug(namespace, logger, "no CephCluster found in the namespace %q", namespace)
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
cephCluster := cephClusters.Items[0]
|
||||
if len(cephClusters.Items) > 1 {
|
||||
logger.Errorf("more than one CephCluster found in the namespace %q, choosing the first one %q", namespace, cephCluster.GetName())
|
||||
log.NamespacedError(request.Namespace, logger, "more than one CephCluster found in the namespace %q, choosing the first one %q", namespace, cephCluster.GetName())
|
||||
}
|
||||
|
||||
allDisabled := r.removeDisabledCrashCollectorDaemons(cephCluster.Spec, namespace) && r.removeDisabledCephExporterDaemons(cephCluster.Spec, namespace)
|
||||
@@ -164,7 +165,7 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result,
|
||||
err = r.client.Get(r.opManagerContext, types.NamespacedName{Name: crashCollectorKeyName, Namespace: namespace}, secret)
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Debugf("secret %q in namespace %q not found. retrying in %q. %v", crashCollectorKeyName, namespace, waitForRequeueIfSecretNotCreated.RequeueAfter.String(), err)
|
||||
log.NamespacedDebug(namespace, logger, "secret %q in namespace %q not found. retrying in %q. %v", crashCollectorKeyName, namespace, waitForRequeueIfSecretNotCreated.RequeueAfter.String(), err)
|
||||
return waitForRequeueIfSecretNotCreated, nil
|
||||
}
|
||||
|
||||
@@ -174,7 +175,7 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result,
|
||||
clusterImage := cephCluster.Spec.CephVersion.Image
|
||||
cephVersion, err := opcontroller.GetImageVersion(cephCluster)
|
||||
if err != nil {
|
||||
logger.Errorf("ceph version not found for image %q used by cluster %q in namespace %q. %v", clusterImage, cephCluster.Name, cephCluster.Namespace, err)
|
||||
log.NamespacedError(request.Namespace, logger, "ceph version not found for image %q used by cluster %q in namespace %q. %v", clusterImage, cephCluster.Name, cephCluster.Namespace, err)
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
@@ -233,19 +234,19 @@ func (r *ReconcileNode) createOrUpdateNodeDaemons(node corev1.Node, tolerations
|
||||
op, err := r.createOrUpdateCephCrash(node, tolerations, cephCluster, cephVersion)
|
||||
if err != nil {
|
||||
if op == "unchanged" {
|
||||
logger.Debugf("crash collector unchanged on node %q", node.Name)
|
||||
log.NamespacedDebug(cephCluster.Namespace, logger, "crash collector unchanged on node %q", node.Name)
|
||||
} else {
|
||||
return errors.Wrapf(err, "crash collector reconcile failed on op %q", op)
|
||||
}
|
||||
} else {
|
||||
logger.Debugf("crash collector successfully reconciled for node %q. operation: %q", node.Name, op)
|
||||
log.NamespacedDebug(cephCluster.Namespace, logger, "crash collector successfully reconciled for node %q. operation: %q", node.Name, op)
|
||||
}
|
||||
}
|
||||
if cephVersion.IsAtLeast(MinVersionForCephExporter) && !cephCluster.Spec.Monitoring.MetricsDisabled {
|
||||
op, err := r.createOrUpdateCephExporter(node, tolerations, cephCluster, cephVersion)
|
||||
if err != nil {
|
||||
if op == "unchanged" {
|
||||
logger.Debugf("ceph exporter unchanged on node %q", node.Name)
|
||||
log.NamespacedDebug(cephCluster.Namespace, logger, "ceph exporter unchanged on node %q", node.Name)
|
||||
} else {
|
||||
return errors.Wrapf(err, "ceph exporter reconcile failed on op %q", op)
|
||||
}
|
||||
@@ -253,7 +254,7 @@ func (r *ReconcileNode) createOrUpdateNodeDaemons(node corev1.Node, tolerations
|
||||
// CephVersion change is done temporarily, as some regression was detected in Ceph version 17.2.6 which is summarised here https://github.com/ceph/ceph/pull/50718#issuecomment-1505608312.
|
||||
// Thus, disabling ceph-exporter for now until all the regression are fixed.
|
||||
if cephVersion.IsAtLeast(MinVersionForCephExporter) {
|
||||
logger.Debugf("ceph exporter successfully reconciled for node %q. operation: %q", node.Name, op)
|
||||
log.NamespacedDebug(cephCluster.Namespace, logger, "ceph exporter successfully reconciled for node %q. operation: %q", node.Name, op)
|
||||
// create the metrics service
|
||||
service, err := MakeCephExporterMetricsService(cephCluster, exporterServiceMetricName, r.scheme)
|
||||
if err != nil {
|
||||
@@ -267,7 +268,7 @@ func (r *ReconcileNode) createOrUpdateNodeDaemons(node corev1.Node, tolerations
|
||||
if err := EnableCephExporterServiceMonitor(r.context, cephCluster, r.scheme, r.opManagerContext, exporterServiceMetricName); err != nil {
|
||||
return errors.Wrap(err, "failed to enable service monitor")
|
||||
}
|
||||
logger.Debug("service monitor for ceph exporter was enabled successfully")
|
||||
log.NamespacedDebug(cephCluster.Namespace, logger, "service monitor for ceph exporter was enabled successfully")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -302,12 +303,12 @@ func (r *ReconcileNode) listDeploymentAndDelete(appName, nodeName, ns string) er
|
||||
return errors.Wrapf(err, "failed to list deployments in namespace %q", ns)
|
||||
}
|
||||
for _, d := range deploymentList.Items {
|
||||
logger.Infof("deleting deployment %q for node %q", d.ObjectMeta.Name, nodeName)
|
||||
log.NamespacedInfo(d.Namespace, logger, "deleting deployment %q for node %q", d.ObjectMeta.Name, nodeName)
|
||||
err := r.deleteDeployment(d)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to delete deployment %q in namespace %q", d.Name, d.Namespace)
|
||||
}
|
||||
logger.Infof("successfully removed deployment %q in namespace %q from node %q", d.Name, d.Namespace, nodeName)
|
||||
log.NamespacedInfo(d.Namespace, logger, "successfully removed deployment %q in namespace %q from node %q", d.Name, d.Namespace, nodeName)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -320,7 +321,7 @@ func (r *ReconcileNode) deleteNodeDaemon(appName, namespace string) {
|
||||
// Try to fetch the list of existing deployment and remove them
|
||||
err := r.client.List(r.opManagerContext, deploymentList, client.MatchingLabels{k8sutil.AppAttr: appName}, namespaceListOpts)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to list deployments in namespace %q, delete it/them manually. %v", namespace, err)
|
||||
log.NamespacedError(namespace, logger, "failed to list deployments in namespace %q, delete it/them manually. %v", namespace, err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -328,10 +329,10 @@ func (r *ReconcileNode) deleteNodeDaemon(appName, namespace string) {
|
||||
for _, d := range deploymentList.Items {
|
||||
err := r.deleteDeployment(d)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to delete deployment %q in namespace %q, delete it manually. %v", d.Name, d.Namespace, err)
|
||||
log.NamespacedError(namespace, logger, "failed to delete deployment %q in namespace %q, delete it manually. %v", d.Name, d.Namespace, err)
|
||||
continue
|
||||
}
|
||||
logger.Infof("Deployments %q in namespace %q successfully removed", d.Name, d.Namespace)
|
||||
log.NamespacedInfo(d.Namespace, logger, "Deployments %q in namespace %q successfully removed", d.Name, d.Namespace)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
)
|
||||
@@ -79,20 +80,20 @@ func (c *createConfig) createNewOSDsFromStatus(
|
||||
) {
|
||||
if !c.awaitingStatusConfigMaps.Has(statusConfigMapName(nodeOrPVCName)) {
|
||||
// If there is a dangling OSD prepare configmap from another reconcile, don't process it
|
||||
logger.Infof("not creating deployments for OSD prepare results found in ConfigMap %q which was not created for the latest storage spec", statusConfigMapName(nodeOrPVCName))
|
||||
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "not creating deployments for OSD prepare results found in ConfigMap %q which was not created for the latest storage spec", statusConfigMapName(nodeOrPVCName))
|
||||
return
|
||||
}
|
||||
|
||||
if c.finishedStatusConfigMaps.Has(statusConfigMapName(nodeOrPVCName)) {
|
||||
// If we have already processed this configmap, don't process it again
|
||||
logger.Infof("not creating deployments for OSD prepare results found in ConfigMap %q which was already processed", statusConfigMapName(nodeOrPVCName))
|
||||
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "not creating deployments for OSD prepare results found in ConfigMap %q which was already processed", statusConfigMapName(nodeOrPVCName))
|
||||
return
|
||||
}
|
||||
|
||||
for i, osd := range status.OSDs {
|
||||
if c.deployments.Exists(osd.ID) {
|
||||
// This OSD will be handled by the updater
|
||||
logger.Debugf("not creating deployment for OSD %d which already exists", osd.ID)
|
||||
log.NamespacedDebug(c.cluster.clusterInfo.Namespace, logger, "not creating deployment for OSD %d which already exists", osd.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -102,13 +103,13 @@ func (c *createConfig) createNewOSDsFromStatus(
|
||||
c.cluster.clusterInfo.CephVersion, keyring.UninitializedCephxStatus())
|
||||
|
||||
if status.PvcBackedOSD {
|
||||
logger.Infof("creating OSD %d on PVC %q", osd.ID, nodeOrPVCName)
|
||||
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "creating OSD %d on PVC %q", osd.ID, nodeOrPVCName)
|
||||
err := createDaemonOnPVCFunc(c.cluster, &status.OSDs[i], nodeOrPVCName, c.provisionConfig)
|
||||
if err != nil {
|
||||
errs.addError("%v", errors.Wrapf(err, "failed to create OSD %d on PVC %q", osd.ID, nodeOrPVCName))
|
||||
}
|
||||
} else {
|
||||
logger.Infof("creating OSD %d on node %q", osd.ID, nodeOrPVCName)
|
||||
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "creating OSD %d on node %q", osd.ID, nodeOrPVCName)
|
||||
err := createDaemonOnNodeFunc(c.cluster, &status.OSDs[i], nodeOrPVCName, c.provisionConfig)
|
||||
if err != nil {
|
||||
errs.addError("%v", errors.Wrapf(err, "failed to create OSD %d on node %q", osd.ID, nodeOrPVCName))
|
||||
@@ -136,7 +137,7 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig, errs *provi
|
||||
|
||||
// no valid VolumeSource is ready to run an osd
|
||||
if len(c.deviceSets) == 0 {
|
||||
logger.Info("no storageClassDeviceSets defined to configure OSDs on PVCs")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "no storageClassDeviceSets defined to configure OSDs on PVCs")
|
||||
return sets.New[string](), nil
|
||||
}
|
||||
|
||||
@@ -161,14 +162,14 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig, errs *provi
|
||||
|
||||
metadataSource, metadataOK := volume.PVCSources[bluestorePVCMetadata]
|
||||
if metadataOK {
|
||||
logger.Infof("OSD will have its main bluestore block on %q and its metadata device on %q", dataSource.ClaimName, metadataSource.ClaimName)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its main bluestore block on %q and its metadata device on %q", dataSource.ClaimName, metadataSource.ClaimName)
|
||||
} else {
|
||||
logger.Infof("OSD will have its main bluestore block on %q", dataSource.ClaimName)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its main bluestore block on %q", dataSource.ClaimName)
|
||||
}
|
||||
|
||||
walSource, walOK := volume.PVCSources[bluestorePVCWal]
|
||||
if walOK {
|
||||
logger.Infof("OSD will have its wal device on %q", walSource.ClaimName)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its wal device on %q", walSource.ClaimName)
|
||||
}
|
||||
|
||||
osdProps := osdProperties{
|
||||
@@ -196,13 +197,13 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig, errs *provi
|
||||
// Allow updating OSD prepare pod if the OSD needs migration
|
||||
if c.migrateOSD != nil {
|
||||
if strings.Contains(c.migrateOSD.BlockPath, dataSource.ClaimName) {
|
||||
logger.Infof("updating OSD prepare pod to replace OSD.%d", c.migrateOSD.ID)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "updating OSD prepare pod to replace OSD.%d", c.migrateOSD.ID)
|
||||
skipPreparePod = false
|
||||
}
|
||||
}
|
||||
|
||||
if skipPreparePod {
|
||||
logger.Infof("skipping OSD prepare job creation for PVC %q because OSD daemon using the PVC already exists", osdProps.crushHostname)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "skipping OSD prepare job creation for PVC %q because OSD daemon using the PVC already exists", osdProps.crushHostname)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -264,13 +265,13 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig, errs *provi
|
||||
// usage of awaitingStatusConfigMaps in this file.
|
||||
func (c *Cluster) startProvisioningOverNodes(config *provisionConfig, errs *provisionErrors) (sets.Set[string], error) {
|
||||
if !c.spec.Storage.UseAllNodes && len(c.spec.Storage.Nodes) == 0 {
|
||||
logger.Info("no nodes are defined for configuring OSDs on raw devices")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "no nodes are defined for configuring OSDs on raw devices")
|
||||
return sets.New[string](), nil
|
||||
}
|
||||
|
||||
if c.spec.Storage.UseAllNodes {
|
||||
if len(c.spec.Storage.Nodes) > 0 {
|
||||
logger.Warningf("useAllNodes is TRUE, but nodes are specified. NODES in the cluster CR will be IGNORED unless useAllNodes is FALSE.")
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "useAllNodes is TRUE, but nodes are specified. NODES in the cluster CR will be IGNORED unless useAllNodes is FALSE.")
|
||||
}
|
||||
|
||||
// Get the list of all nodes in the cluster. The placement settings will be applied below.
|
||||
@@ -286,19 +287,19 @@ func (c *Cluster) startProvisioningOverNodes(config *provisionConfig, errs *prov
|
||||
}
|
||||
c.spec.Storage.Nodes = append(c.spec.Storage.Nodes, storageNode)
|
||||
}
|
||||
logger.Debugf("storage nodes: %+v", c.spec.Storage.Nodes)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "storage nodes: %+v", c.spec.Storage.Nodes)
|
||||
}
|
||||
// generally speaking, this finds nodes which are capable of running new osds
|
||||
validNodes := k8sutil.GetValidNodes(c.clusterInfo.Context, c.spec.Storage, c.context.Clientset, cephv1.GetOSDPlacement(c.spec.Placement))
|
||||
|
||||
logger.Infof("%d of the %d storage nodes are valid", len(validNodes), len(c.spec.Storage.Nodes))
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "%d of the %d storage nodes are valid", len(validNodes), len(c.spec.Storage.Nodes))
|
||||
|
||||
c.ValidStorage = *c.spec.Storage.DeepCopy()
|
||||
c.ValidStorage.Nodes = validNodes
|
||||
|
||||
// no valid node is ready to run an osd
|
||||
if len(validNodes) == 0 {
|
||||
logger.Warningf("no valid nodes available to run osds on nodes in namespace %q", c.clusterInfo.Namespace)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "no valid nodes available to run osds on nodes in namespace %q", c.clusterInfo.Namespace)
|
||||
return sets.New[string](), nil
|
||||
}
|
||||
|
||||
@@ -316,12 +317,12 @@ func (c *Cluster) startProvisioningOverNodes(config *provisionConfig, errs *prov
|
||||
// don't care about osd device class resources since it will be overwritten later for prepareosd resources
|
||||
n := c.resolveNode(node.Name, "")
|
||||
if n == nil {
|
||||
logger.Warningf("node %q did not resolve", node.Name)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "node %q did not resolve", node.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
if n.Name == "" {
|
||||
logger.Warningf("skipping node with a blank name! %+v", n)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "skipping node with a blank name! %+v", n)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -372,7 +373,7 @@ func (c *Cluster) runPrepareJob(osdProps *osdProperties, config *provisionConfig
|
||||
return errors.Wrapf(err, "failed to run osd provisioning job for %s %q", nodeOrPVC, nodeOrPVCName)
|
||||
}
|
||||
|
||||
logger.Infof("started OSD provisioning job for %s %q", nodeOrPVC, nodeOrPVCName)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "started OSD provisioning job for %s %q", nodeOrPVC, nodeOrPVCName)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -109,7 +110,7 @@ func (c *Cluster) prepareStorageClassDeviceSets(errs *provisionErrors) {
|
||||
highestExistingID := -1
|
||||
countInDeviceSet := 0
|
||||
if existingIDs, ok := uniqueOSDsPerDeviceSet[deviceSet.Name]; ok {
|
||||
logger.Infof("verifying PVCs exist for %d OSDs in device set %q", existingIDs.Len(), deviceSet.Name)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "verifying PVCs exist for %d OSDs in device set %q", existingIDs.Len(), deviceSet.Name)
|
||||
for existingID := range existingIDs {
|
||||
pvcID, err := strconv.Atoi(existingID)
|
||||
if err != nil {
|
||||
@@ -129,7 +130,7 @@ func (c *Cluster) prepareStorageClassDeviceSets(errs *provisionErrors) {
|
||||
// No new PVCs will be created if we have too many
|
||||
pvcsToCreate := deviceSet.Count - countInDeviceSet
|
||||
if pvcsToCreate > 0 {
|
||||
logger.Infof("creating %d new PVCs for device set %q", pvcsToCreate, deviceSet.Name)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "creating %d new PVCs for device set %q", pvcsToCreate, deviceSet.Name)
|
||||
}
|
||||
for i := 0; i < pvcsToCreate; i++ {
|
||||
pvcID := highestExistingID + i + 1
|
||||
@@ -234,12 +235,12 @@ func (c *Cluster) createDeviceSetPVC(existingPVCs map[string]*v1.PersistentVolum
|
||||
}
|
||||
|
||||
if existingPVC != nil {
|
||||
logger.Infof("OSD PVC %q already exists", existingPVC.Name)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD PVC %q already exists", existingPVC.Name)
|
||||
|
||||
desiredPvcSize, desiredOK := pvc.Spec.Resources.Requests[v1.ResourceStorage]
|
||||
actualPvcSize, currentOK := existingPVC.Spec.Resources.Requests[v1.ResourceStorage]
|
||||
if !desiredOK || !currentOK {
|
||||
logger.Debugf("desired or current size are not specified for PVC %q", existingPVC.Name)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "desired or current size are not specified for PVC %q", existingPVC.Name)
|
||||
return existingPVC, nil
|
||||
}
|
||||
|
||||
@@ -260,7 +261,7 @@ func (c *Cluster) createDeviceSetPVC(existingPVCs map[string]*v1.PersistentVolum
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to create PVC %q for device set %q", pvc.Name, deviceSetName)
|
||||
}
|
||||
logger.Infof("successfully provisioned PVC %q", deployedPVC.Name)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "successfully provisioned PVC %q", deployedPVC.Name)
|
||||
|
||||
return deployedPVC, nil
|
||||
}
|
||||
@@ -345,13 +346,13 @@ func waitForPvcToExpandWithTimeout(ctx context.Context, client client.Client, pv
|
||||
timeout := currentTime.Add(pvcResizeWaitTimeOut)
|
||||
for {
|
||||
if time.Now().UTC().After(timeout) {
|
||||
logger.Warning("timed out waiting for PVC size to be updated, osds may need to be restarted manually after resize")
|
||||
log.NamespacedWarning(namespace, logger, "timed out waiting for PVC size to be updated, osds may need to be restarted manually after resize")
|
||||
return
|
||||
}
|
||||
|
||||
allPvcResized := checkAllPvcResize(ctx, client, namespace, pvcResizeMap)
|
||||
if allPvcResized {
|
||||
logger.Info("all osd pvcs resized")
|
||||
log.NamespacedInfo(namespace, logger, "all osd pvcs resized")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -373,7 +374,7 @@ func checkAllPvcResize(ctx context.Context, client client.Client, namespace stri
|
||||
currentPVC := &v1.PersistentVolumeClaim{}
|
||||
err := client.Get(ctx, types.NamespacedName{Name: pvcName, Namespace: namespace}, currentPVC)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get PVC %q. %v", currentPVC.Name, err)
|
||||
log.NamespacedError(namespace, logger, "failed to get PVC %q. %v", currentPVC.Name, err)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -385,7 +386,7 @@ func checkAllPvcResize(ctx context.Context, client client.Client, namespace stri
|
||||
|
||||
if currentPVC.Status.Capacity != nil {
|
||||
if currentPVC.Status.Capacity[v1.ResourceStorage] == pvcSize {
|
||||
logger.Infof("PVC %q size is updated to %s", currentPVC.Name, pvcSize.String())
|
||||
log.NamespacedInfo(namespace, logger, "PVC %q size is updated to %s", currentPVC.Name, pvcSize.String())
|
||||
numberOfPvcResized++
|
||||
pvcResizeMap[pvcName] = pvcResize{
|
||||
desiredSize: pvc.desiredSize,
|
||||
@@ -393,11 +394,11 @@ func checkAllPvcResize(ctx context.Context, client client.Client, namespace stri
|
||||
resizeConfirmed: true,
|
||||
}
|
||||
} else {
|
||||
logger.Debugf("PVC %q size is not updated to %v", currentPVC.Name, pvcSize.String())
|
||||
log.NamespacedDebug(namespace, logger, "PVC %q size is not updated to %v", currentPVC.Name, pvcSize.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("%d of osd pvcs resized and %d remain to be resized", numberOfPvcResized, len(pvcResizeMap)-numberOfPvcResized)
|
||||
log.NamespacedInfo(namespace, logger, "%d of osd pvcs resized and %d remain to be resized", numberOfPvcResized, len(pvcResizeMap)-numberOfPvcResized)
|
||||
return numberOfPvcResized == len(pvcResizeMap)
|
||||
}
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
)
|
||||
|
||||
@@ -58,7 +59,7 @@ func NewOSDHealthMonitor(context *clusterd.Context, clusterInfo *client.ClusterI
|
||||
// allow overriding the check interval
|
||||
checkInterval := healthCheck.DaemonHealth.ObjectStorageDaemon.Interval
|
||||
if checkInterval != nil {
|
||||
logger.Infof("ceph osd status in namespace %q check interval %q", h.clusterInfo.Namespace, checkInterval.Duration.String())
|
||||
log.NamespacedInfo(h.clusterInfo.Namespace, logger, "ceph osd status in namespace %q check interval %q", h.clusterInfo.Namespace, checkInterval.Duration.String())
|
||||
h.interval = &checkInterval.Duration
|
||||
}
|
||||
|
||||
@@ -72,17 +73,17 @@ func (m *OSDHealthMonitor) Start(monitoringRoutines *sync.Map, daemon string) {
|
||||
// we will get an invalid pointer error and the go routine will panic
|
||||
v, ok := monitoringRoutines.Load(daemon)
|
||||
if !ok {
|
||||
logger.Infof("ceph cluster %q has been deleted. stopping monitoring of OSDs", m.clusterInfo.Namespace)
|
||||
log.NamespacedInfo(m.clusterInfo.Namespace, logger, "ceph cluster %q has been deleted. stopping monitoring of OSDs", m.clusterInfo.Namespace)
|
||||
return
|
||||
}
|
||||
health := v.(*opcontroller.ClusterHealth)
|
||||
select {
|
||||
case <-time.After(*m.interval):
|
||||
logger.Debug("checking osd processes status.")
|
||||
log.NamespacedDebug(m.clusterInfo.Namespace, logger, "checking osd processes status.")
|
||||
m.checkOSDHealth()
|
||||
|
||||
case <-health.InternalCtx.Done():
|
||||
logger.Infof("stopping monitoring of OSDs in namespace %q", m.clusterInfo.Namespace)
|
||||
log.NamespacedInfo(m.clusterInfo.Namespace, logger, "stopping monitoring of OSDs in namespace %q", m.clusterInfo.Namespace)
|
||||
monitoringRoutines.Delete(daemon)
|
||||
return
|
||||
}
|
||||
@@ -98,7 +99,7 @@ func (m *OSDHealthMonitor) Update(removeOSDsIfOUTAndSafeToRemove bool) {
|
||||
func (m *OSDHealthMonitor) checkOSDHealth() {
|
||||
err := m.checkOSDDump()
|
||||
if err != nil {
|
||||
logger.Debugf("failed to check OSD Dump. %v", err)
|
||||
log.NamespacedDebug(m.clusterInfo.Namespace, logger, "failed to check OSD Dump. %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,7 +116,7 @@ func (m *OSDHealthMonitor) checkOSDDump() error {
|
||||
}
|
||||
id := int(id64)
|
||||
|
||||
logger.Debugf("validating status of osd.%d", id)
|
||||
log.NamespacedDebug(m.clusterInfo.Namespace, logger, "validating status of osd.%d", id)
|
||||
|
||||
status, in, err := osdDump.StatusByID(int64(id))
|
||||
if err != nil {
|
||||
@@ -123,17 +124,17 @@ func (m *OSDHealthMonitor) checkOSDDump() error {
|
||||
}
|
||||
|
||||
if status == upStatus {
|
||||
logger.Debugf("osd.%d is healthy.", id)
|
||||
log.NamespacedDebug(m.clusterInfo.Namespace, logger, "osd.%d is healthy.", id)
|
||||
continue
|
||||
}
|
||||
|
||||
logger.Debugf("osd.%d is marked 'DOWN'", id)
|
||||
log.NamespacedDebug(m.clusterInfo.Namespace, logger, "osd.%d is marked 'DOWN'", id)
|
||||
|
||||
if in != inStatus {
|
||||
logger.Debugf("osd.%d is marked 'OUT'", id)
|
||||
log.NamespacedDebug(m.clusterInfo.Namespace, logger, "osd.%d is marked 'OUT'", id)
|
||||
if m.removeOSDsIfOUTAndSafeToRemove {
|
||||
if err := m.removeOSDDeploymentIfSafeToDestroy(id); err != nil {
|
||||
logger.Errorf("error handling marked out osd osd.%d. %v", id, err)
|
||||
log.NamespacedError(m.clusterInfo.Namespace, logger, "error handling marked out osd osd.%d. %v", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -162,7 +163,7 @@ func (m *OSDHealthMonitor) removeOSDDeploymentIfSafeToDestroy(outOSDid int) erro
|
||||
podDeletionTimeStamp := podCreationTimestamp.Add(graceTime)
|
||||
currentTime := time.Now().UTC()
|
||||
if podDeletionTimeStamp.Before(currentTime) {
|
||||
logger.Infof("osd.%d is 'safe-to-destroy'. removing the osd deployment.", outOSDid)
|
||||
log.NamespacedInfo(m.clusterInfo.Namespace, logger, "osd.%d is 'safe-to-destroy'. removing the osd deployment.", outOSDid)
|
||||
if err := k8sutil.DeleteDeployment(m.clusterInfo.Context, m.context.Clientset, dp.Items[0].Namespace, dp.Items[0].Name); err != nil {
|
||||
return errors.Wrapf(err, "failed to delete osd deployment %s", dp.Items[0].Name)
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
batch "k8s.io/api/batch/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -247,7 +248,7 @@ func (c *Cluster) reconcileKeyRotationCronJob() error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
logger.Debugf("successfully deleted key rotation cron jobs")
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "successfully deleted key rotation cron jobs")
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -259,7 +260,7 @@ func (c *Cluster) reconcileKeyRotationCronJob() error {
|
||||
return errors.Wrap(err, "failed to query existing OSD deployments")
|
||||
}
|
||||
|
||||
logger.Debugf("found %d osd deployments", len(deployments.Items))
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "found %d osd deployments", len(deployments.Items))
|
||||
for i := range deployments.Items {
|
||||
osdDep := deployments.Items[i]
|
||||
osd, err := c.getOSDInfo(&osdDep)
|
||||
@@ -279,7 +280,7 @@ func (c *Cluster) reconcileKeyRotationCronJob() error {
|
||||
continue
|
||||
}
|
||||
|
||||
logger.Infof("starting OSD key rotation cron job for osd %d", osd.ID)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "starting OSD key rotation cron job for osd %d", osd.ID)
|
||||
cj, err := c.makeKeyRotationCronJob(pvcName, osd, osdProps)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to make key rotation cron job")
|
||||
@@ -294,9 +295,9 @@ func (c *Cluster) reconcileKeyRotationCronJob() error {
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to create or update key rotation cron job %q", cj.Name)
|
||||
}
|
||||
logger.Infof("started OSD key rotation cron job %q", cj.Name)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "started OSD key rotation cron job %q", cj.Name)
|
||||
}
|
||||
logger.Infof("successfully started OSD key rotation cron jobs")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "successfully started OSD key rotation cron jobs")
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
@@ -94,7 +95,7 @@ func (m *migrationConfig) migrateForEncryption(c *Cluster, osdDeployments *appsv
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to details about the OSD %q", osdDeployments.Items[i].Name)
|
||||
}
|
||||
logger.Infof("migration is required for OSD.%d due to change in encryption settings from %t to %t in storageClassDeviceSet %q", osdInfo.ID, actualEncryptedSetting, requestedEncryptionSetting, osdDeviceSetName)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "migration is required for OSD.%d due to change in encryption settings from %t to %t in storageClassDeviceSet %q", osdInfo.ID, actualEncryptedSetting, requestedEncryptionSetting, osdDeviceSetName)
|
||||
if _, exists := m.osds[osdInfo.ID]; !exists {
|
||||
m.osds[osdInfo.ID] = &osdInfo
|
||||
}
|
||||
@@ -113,7 +114,7 @@ func (m *migrationConfig) migrateForOSDStore(c *Cluster, osdDeployments *appsv1.
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to details about the OSD %q", osdDeployments.Items[i].Name)
|
||||
}
|
||||
logger.Infof("migration is required for OSD.%d to update storeType from %q to %q", osdInfo.ID, osdStore, desiredOSDStore)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "migration is required for OSD.%d to update storeType from %q to %q", osdInfo.ID, osdStore, desiredOSDStore)
|
||||
if _, exists := m.osds[osdInfo.ID]; !exists {
|
||||
m.osds[osdInfo.ID] = &osdInfo
|
||||
}
|
||||
@@ -185,12 +186,12 @@ func isLastOSDMigrationComplete(c *Cluster) (bool, error) {
|
||||
_, err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Get(c.clusterInfo.Context, deploymentName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Infof("deployment for the last migrated OSD with ID - %d is not found.", migratedOSDId)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "deployment for the last migrated OSD with ID - %d is not found.", migratedOSDId)
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("migration of OSD.%d was successful", migratedOSDId)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "migration of OSD.%d was successful", migratedOSDId)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -205,7 +206,7 @@ func getLastMigratedOSDId(context *clusterd.Context, clusterInfo *cephclient.Clu
|
||||
|
||||
osdID, ok := cm.Data[OSDIdKey]
|
||||
if !ok || osdID == "" {
|
||||
logger.Debugf("empty config map %q", OSDMigrationConfigName)
|
||||
log.NamespacedDebug(clusterInfo.Namespace, logger, "empty config map %q", OSDMigrationConfigName)
|
||||
return -1, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -38,6 +38,7 @@ import (
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/pkg/errors"
|
||||
@@ -206,47 +207,47 @@ func (c *Cluster) validateOSDSettings() error {
|
||||
}
|
||||
|
||||
func (c *Cluster) validateTopologyAcrossNodes() error {
|
||||
logger.Info("Starting validateTopologyAcrossNodes()")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Starting validateTopologyAcrossNodes()")
|
||||
if os.Getenv("ROOK_SKIP_OSD_TOPOLOGY_CHECK") == "true" {
|
||||
logger.Debugf("Skipping topology validation due to ROOK_SKIP_OSD_TOPOLOGY_CHECK=true")
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "Skipping topology validation due to ROOK_SKIP_OSD_TOPOLOGY_CHECK=true")
|
||||
return nil
|
||||
}
|
||||
if topologyValidated {
|
||||
logger.Debug("Skipping topology validation because it was already validated")
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "Skipping topology validation because it was already validated")
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Info("Validating node topology across all cluster nodes")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Validating node topology across all cluster nodes")
|
||||
|
||||
nodelist, err := c.context.Clientset.CoreV1().Nodes().List(c.clusterInfo.Context, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
logger.Errorf("Failed to list nodes for topology validation: %v", err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "Failed to list nodes for topology validation: %v", err)
|
||||
return errors.Wrap(err, "failed to list nodes for topology validation")
|
||||
}
|
||||
logger.Infof("Fetched nodelist with %d nodes", len(nodelist.Items))
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Fetched nodelist with %d nodes", len(nodelist.Items))
|
||||
|
||||
if err := topology.CheckTopologyConflicts(&nodelist.Items); err != nil {
|
||||
logger.Infof("Topology conflict detected: %v", err)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Topology conflict detected: %v", err)
|
||||
// Check if there are any existing OSDs
|
||||
osdRunning, errPods := k8sutil.PodsRunningWithLabel(c.clusterInfo.Context, c.context.Clientset, c.clusterInfo.Namespace, "app=rook-ceph-osd")
|
||||
if errPods != nil {
|
||||
logger.Errorf("Failed to list OSD pods: %v", errPods)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "Failed to list OSD pods: %v", errPods)
|
||||
return errPods // fail safe
|
||||
}
|
||||
logger.Infof("Found %d existing OSDs", osdRunning)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Found %d existing OSDs", osdRunning)
|
||||
if osdRunning == 0 {
|
||||
logger.Errorf("Topology conflict detected in new cluster: %v", err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "Topology conflict detected in new cluster: %v", err)
|
||||
return errors.Wrap(err, "topology conflict detected in new cluster")
|
||||
}
|
||||
|
||||
// Existing OSDs found — log and continue
|
||||
logger.Warningf("Topology conflict detected, but skipping failure since OSDs already exist. %v", err)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "Topology conflict detected, but skipping failure since OSDs already exist. %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Info("Node topology validation passed without conflicts")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "Node topology validation passed without conflicts")
|
||||
topologyValidated = true
|
||||
logger.Info("validateTopologyAcrossNodes() completed successfully")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "validateTopologyAcrossNodes() completed successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -265,10 +266,10 @@ func (c *Cluster) Start() error {
|
||||
if err := c.initializeNodeConfigmaps(); err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Infof("start running osds in namespace %q", namespace)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "start running osds in namespace %q", namespace)
|
||||
|
||||
if !c.spec.Storage.UseAllNodes && len(c.spec.Storage.Nodes) == 0 && len(c.spec.Storage.StorageClassDeviceSets) == 0 {
|
||||
logger.Warningf("useAllNodes is set to false and no nodes, storageClassDevicesets or volumeSources are specified, no OSD pods are going to be created")
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "useAllNodes is set to false and no nodes, storageClassDevicesets or volumeSources are specified, no OSD pods are going to be created")
|
||||
}
|
||||
|
||||
if c.spec.WaitTimeoutForHealthyOSDInMinutes != 0 {
|
||||
@@ -276,11 +277,11 @@ func (c *Cluster) Start() error {
|
||||
} else {
|
||||
c.clusterInfo.OsdUpgradeTimeout = defaultWaitTimeoutForHealthyOSD
|
||||
}
|
||||
logger.Infof("wait timeout for healthy OSDs during upgrade or restart is %q", c.clusterInfo.OsdUpgradeTimeout)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "wait timeout for healthy OSDs during upgrade or restart is %q", c.clusterInfo.OsdUpgradeTimeout)
|
||||
|
||||
osdsToSkipReconcile, err := controller.GetDaemonsToSkipReconcile(c.clusterInfo.Context, c.context, c.clusterInfo.Namespace, OsdIdLabelKey, AppName)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to get osds to skip reconcile. %v", err)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to get osds to skip reconcile. %v", err)
|
||||
}
|
||||
|
||||
migrationConfig, err := c.startOSDMigration()
|
||||
@@ -301,20 +302,20 @@ func (c *Cluster) Start() error {
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debugf("%d of %d OSD Deployments need update", updateQueue.Len(), deployments.Len())
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "%d of %d OSD Deployments need update", updateQueue.Len(), deployments.Len())
|
||||
updateConfig := c.newUpdateConfig(config, updateQueue, deployments, osdsToSkipReconcile)
|
||||
|
||||
// prepare for creating new OSDs
|
||||
statusConfigMaps := sets.New[string]()
|
||||
|
||||
logger.Info("start provisioning the OSDs on PVCs, if needed")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "start provisioning the OSDs on PVCs, if needed")
|
||||
pvcConfigMaps, err := c.startProvisioningOverPVCs(config, errs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
statusConfigMaps = statusConfigMaps.Union(pvcConfigMaps)
|
||||
|
||||
logger.Info("start provisioning the OSDs on nodes, if needed")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "start provisioning the OSDs on nodes, if needed")
|
||||
nodeConfigMaps, err := c.startProvisioningOverNodes(config, errs)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -360,26 +361,26 @@ func (c *Cluster) Start() error {
|
||||
// remove the osdBootstrapKey as the osds are running
|
||||
c.deleteOsdBootstrapKeyring()
|
||||
|
||||
logger.Infof("finished running OSDs in namespace %q", namespace)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "finished running OSDs in namespace %q", namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) deleteOsdBootstrapKeyring() {
|
||||
err := cephclient.AuthDelete(c.context, c.clusterInfo, "client.bootstrap-osd")
|
||||
if err != nil {
|
||||
logger.Debugf("failed to delete client.bootstrap-osd bootstrap key in the namespace %q. Error: %v", c.clusterInfo.Namespace, err)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "failed to delete client.bootstrap-osd bootstrap key in the namespace %q. Error: %v", c.clusterInfo.Namespace, err)
|
||||
} else {
|
||||
logger.Debugf("successfully deleted client.bootstrap-osd bootstrap key in the namespace %q", c.clusterInfo.Namespace)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "successfully deleted client.bootstrap-osd bootstrap key in the namespace %q", c.clusterInfo.Namespace)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) startOSDMigration() (*migrationConfig, error) {
|
||||
if !c.isMigrationRequested() {
|
||||
logger.Debug("no OSD migration is requested")
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "no OSD migration is requested")
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
logger.Info("osd migration is requested")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "osd migration is requested")
|
||||
|
||||
// start migration only if PGs are active+clean
|
||||
pgsHealhty, err := c.waitForHealthyPGs()
|
||||
@@ -409,7 +410,7 @@ func (c *Cluster) startOSDMigration() (*migrationConfig, error) {
|
||||
// delete deployment of the osd that needs migration
|
||||
if migrationConfig != nil && len(migrationConfig.osds) > 0 {
|
||||
osdToMigrate := migrationConfig.getOSDToMigrate()
|
||||
logger.Infof("deleting OSD.%d deployment for migration ", osdToMigrate.ID)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "deleting OSD.%d deployment for migration ", osdToMigrate.ID)
|
||||
err = c.deleteOSDDeployment(osdToMigrate.ID)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to delete deployment for osd.%d that needs migration %q", osdToMigrate.ID, c.clusterInfo.Namespace)
|
||||
@@ -437,13 +438,13 @@ func (c *Cluster) postReconcileUpdateOSDProperties(desiredOSDs map[int]*OSDInfo)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to get osd usage")
|
||||
}
|
||||
logger.Debugf("post processing osd properties with %d actual osds from ceph osd df and %d existing osds found during reconcile", len(osdUsage.OSDNodes), len(desiredOSDs))
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "post processing osd properties with %d actual osds from ceph osd df and %d existing osds found during reconcile", len(osdUsage.OSDNodes), len(desiredOSDs))
|
||||
for _, actualOSD := range osdUsage.OSDNodes {
|
||||
if c.spec.Storage.AllowOsdCrushWeightUpdate {
|
||||
_, err := cephclient.ResizeOsdCrushWeight(actualOSD, c.context, c.clusterInfo)
|
||||
if err != nil {
|
||||
// Log the error and allow other updates to continue
|
||||
logger.Errorf("failed to resize osd crush weight on cluster in namespace %s: %v", c.clusterInfo.Namespace, err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to resize osd crush weight on cluster in namespace %s: %v", c.clusterInfo.Namespace, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -453,7 +454,7 @@ func (c *Cluster) postReconcileUpdateOSDProperties(desiredOSDs map[int]*OSDInfo)
|
||||
}
|
||||
if err := c.updateDeviceClassIfChanged(actualOSD.ID, desiredOSD.DeviceClass, actualOSD.DeviceClass); err != nil {
|
||||
// Log the error and allow other updates to continue
|
||||
logger.Errorf("failed to update device class on cluster in namespace %s: %v", c.clusterInfo.Namespace, err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to update device class on cluster in namespace %s: %v", c.clusterInfo.Namespace, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -466,14 +467,14 @@ func (c *Cluster) updateDeviceClassIfChanged(osdID int, desiredDeviceClass, actu
|
||||
return nil
|
||||
}
|
||||
if desiredDeviceClass != "" && desiredDeviceClass != actualDeviceClass {
|
||||
logger.Infof("updating osd.%d device class from %q to %q", osdID, actualDeviceClass, desiredDeviceClass)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "updating osd.%d device class from %q to %q", osdID, actualDeviceClass, desiredDeviceClass)
|
||||
err := cephclient.SetDeviceClass(c.context, c.clusterInfo, osdID, desiredDeviceClass)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to set device class on osd %d", osdID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
logger.Debugf("no device class change needed for osd.%d. desired=%q, actual=%q", osdID, desiredDeviceClass, actualDeviceClass)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "no device class change needed for osd.%d. desired=%q, actual=%q", osdID, desiredDeviceClass, actualDeviceClass)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -584,21 +585,21 @@ func (c *Cluster) getOSDPropsForPVC(pvcName string) (osdProperties, error) {
|
||||
// The data PVC template is required.
|
||||
dataSource, dataOK := deviceSet.PVCSources[bluestorePVCData]
|
||||
if !dataOK {
|
||||
logger.Warningf("failed to find data source daemon for device set %q, missing the data template", deviceSet.Name)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to find data source daemon for device set %q, missing the data template", deviceSet.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
if pvcName == dataSource.ClaimName {
|
||||
metadataSource, metadataOK := deviceSet.PVCSources[bluestorePVCMetadata]
|
||||
if metadataOK {
|
||||
logger.Infof("OSD will have its main bluestore block on %q and its metadata device on %q", dataSource.ClaimName, metadataSource.ClaimName)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its main bluestore block on %q and its metadata device on %q", dataSource.ClaimName, metadataSource.ClaimName)
|
||||
} else {
|
||||
logger.Infof("OSD will have its main bluestore block on %q", dataSource.ClaimName)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its main bluestore block on %q", dataSource.ClaimName)
|
||||
}
|
||||
|
||||
walSource, walOK := deviceSet.PVCSources[bluestorePVCWal]
|
||||
if walOK {
|
||||
logger.Infof("OSD will have its wal device on %q", walSource.ClaimName)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD will have its wal device on %q", walSource.ClaimName)
|
||||
}
|
||||
|
||||
if deviceSet.Resources.Limits == nil && deviceSet.Resources.Requests == nil {
|
||||
@@ -670,7 +671,7 @@ func (c *Cluster) getPVCHostName(pvcName string) (string, error) {
|
||||
for _, pod := range pods.Items {
|
||||
name, err := k8sutil.GetNodeHostName(c.clusterInfo.Context, c.context.Clientset, pod.Spec.NodeName)
|
||||
if err != nil {
|
||||
logger.Warningf("falling back to node name %s since hostname not found for node", pod.Spec.NodeName)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "falling back to node name %s since hostname not found for node", pod.Spec.NodeName)
|
||||
name = pod.Spec.NodeName
|
||||
}
|
||||
if name == "" {
|
||||
@@ -754,7 +755,7 @@ func (c *Cluster) getOSDInfo(d *appsv1.Deployment) (OSDInfo, error) {
|
||||
// If CVMode is empty, this likely means we upgraded Rook
|
||||
// This property did not exist before so we need to initialize it
|
||||
if osd.CVMode == "" {
|
||||
logger.Infof("required CVMode for OSD %d was not found. assuming this is an LVM OSD", osd.ID)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "required CVMode for OSD %d was not found. assuming this is an LVM OSD", osd.ID)
|
||||
osd.CVMode = "lvm"
|
||||
}
|
||||
|
||||
@@ -762,7 +763,7 @@ func (c *Cluster) getOSDInfo(d *appsv1.Deployment) (OSDInfo, error) {
|
||||
if isPVC && osd.TopologyAffinity == "" {
|
||||
osd.TopologyAffinity, err = getTopologyFromNode(c.clusterInfo.Context, c.context.Clientset, d, osd)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get topology affinity for osd %d. %v", osd.ID, err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get topology affinity for osd %d. %v", osd.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -772,7 +773,7 @@ func (c *Cluster) getOSDInfo(d *appsv1.Deployment) (OSDInfo, error) {
|
||||
if !locationFound {
|
||||
location, _, err := getLocationFromPod(c.clusterInfo.Context, c.context.Clientset, d, cephclient.GetCrushRootFromSpec(&c.spec))
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get location. %v", err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to get location. %v", err)
|
||||
} else {
|
||||
osd.Location = location
|
||||
}
|
||||
@@ -884,7 +885,7 @@ func getTopologyFromNode(ctx context.Context, clientset kubernetes.Interface, d
|
||||
// osd is not portable, no need to load the topology affinity
|
||||
return "", nil
|
||||
}
|
||||
logger.Infof("detecting topology affinity for osd %d after upgrade", osd.ID)
|
||||
log.NamespacedInfo(d.Namespace, logger, "detecting topology affinity for osd %d after upgrade", osd.ID)
|
||||
|
||||
// Get the osd pod and its assigned node, then look up the node labels
|
||||
pods, err := clientset.CoreV1().Pods(d.Namespace).List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", OsdIdLabelKey, d.Labels[OsdIdLabelKey])})
|
||||
@@ -903,7 +904,7 @@ func getTopologyFromNode(ctx context.Context, clientset kubernetes.Interface, d
|
||||
return "", errors.Wrap(err, "failed to get the node for topology affinity")
|
||||
}
|
||||
_, topologyAffinity := topology.ExtractOSDTopologyFromLabels(node.Labels)
|
||||
logger.Infof("found osd %d topology affinity at %q", osd.ID, topologyAffinity)
|
||||
log.NamespacedInfo(d.Namespace, logger, "found osd %d topology affinity at %q", osd.ID, topologyAffinity)
|
||||
return topologyAffinity, nil
|
||||
}
|
||||
|
||||
@@ -984,7 +985,7 @@ func (c *Cluster) applyUpgradeOSDFunctionality() {
|
||||
// Get all the daemons versions
|
||||
versions, err := cephclient.GetAllCephDaemonVersions(c.context, c.clusterInfo)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to get ceph daemons versions; this likely means there are no osds yet. %v", err)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to get ceph daemons versions; this likely means there are no osds yet. %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -997,7 +998,7 @@ func (c *Cluster) applyUpgradeOSDFunctionality() {
|
||||
for v := range versions.Osd {
|
||||
osdVersion, err = cephver.ExtractCephVersion(v)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to extract ceph version. %v", err)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to extract ceph version. %v", err)
|
||||
return
|
||||
}
|
||||
// Ensure the required version of OSDs is set to the current consistent version,
|
||||
@@ -1005,7 +1006,7 @@ func (c *Cluster) applyUpgradeOSDFunctionality() {
|
||||
// previous major ceph version.
|
||||
err = cephclient.EnableReleaseOSDFunctionality(c.context, c.clusterInfo, osdVersion.ReleaseName())
|
||||
if err != nil {
|
||||
logger.Warningf("failed to enable new osd functionality. %v", err)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to enable new osd functionality. %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -1016,12 +1017,12 @@ func (c *Cluster) applyUpgradeOSDFunctionality() {
|
||||
func (c *Cluster) deleteOSDDeployment(osdID int) error {
|
||||
// Delete the OSD deployment
|
||||
deploymentName := fmt.Sprintf("rook-ceph-osd-%d", osdID)
|
||||
logger.Infof("removing the OSD deployment %q", deploymentName)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "removing the OSD deployment %q", deploymentName)
|
||||
if err := k8sutil.DeleteDeployment(c.clusterInfo.Context, c.context.Clientset, c.clusterInfo.Namespace, deploymentName); err != nil {
|
||||
if !kerrors.IsNotFound(err) {
|
||||
return errors.Wrapf(err, "failed to delete OSD deployment %q.", deploymentName)
|
||||
}
|
||||
logger.Debugf("osd deployment %q not found. Ignoring since object must be deleted.", deploymentName)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "osd deployment %q not found. Ignoring since object must be deleted.", deploymentName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1035,7 +1036,7 @@ func (c *Cluster) waitForHealthyPGs() (bool, error) {
|
||||
if pgClean {
|
||||
return true, nil
|
||||
}
|
||||
logger.Infof("waiting for PGs to be healthy. PG status: %q", pgHealthMsg)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "waiting for PGs to be healthy. PG status: %q", pgHealthMsg)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
@@ -1083,7 +1084,7 @@ func (c *Cluster) updateCephOsdStorageStatus() error {
|
||||
err := c.context.Client.Get(c.clusterInfo.Context, c.clusterInfo.NamespacedName(), &cephCluster)
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Debug("CephCluster resource not found. Ignoring since object must be deleted.")
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "CephCluster resource not found. Ignoring since object must be deleted.")
|
||||
return nil
|
||||
}
|
||||
return errors.Wrapf(err, "failed to retrieve ceph cluster %q to update ceph Storage", c.clusterInfo.NamespacedName().Name)
|
||||
@@ -1177,7 +1178,7 @@ func (c *Cluster) initializeNodeConfigmaps() error {
|
||||
for _, configmap := range configmaps.Items {
|
||||
if strings.HasPrefix(configmap.Name, prefix) {
|
||||
nodeName := strings.TrimPrefix(configmap.Name, prefix)
|
||||
logger.Infof("found node %q configmap, will mount it for alternate ceph conf overrides for osds on this node", nodeName)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "found node %q configmap, will mount it for alternate ceph conf overrides for osds on this node", nodeName)
|
||||
c.nodeConfigmaps[nodeName] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
|
||||
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
batch "k8s.io/api/batch/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -365,13 +366,13 @@ func (c *Cluster) deleteAllOrphanedPrepareJobs() {
|
||||
}
|
||||
jobs, err := c.context.Clientset.BatchV1().Jobs(c.clusterInfo.Namespace).List(c.clusterInfo.Context, listOpts)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to clean up any orphaned OSD prepare jobs. failed to list OSD prepare jobs. %v", err)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to clean up any orphaned OSD prepare jobs. failed to list OSD prepare jobs. %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
nodes, err := c.context.Clientset.CoreV1().Nodes().List(c.clusterInfo.Context, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
logger.Warningf("failed to clean up any orphaned OSD prepare jobs. failed to list nodes. %v", err)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to clean up any orphaned OSD prepare jobs. failed to list nodes. %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -387,9 +388,9 @@ func (c *Cluster) deleteAllOrphanedPrepareJobs() {
|
||||
nodeSelector := job.Spec.Template.Spec.NodeSelector
|
||||
if jobHostName, ok := nodeSelector[k8sutil.LabelHostname()]; ok {
|
||||
if !hostNames.Has(jobHostName) {
|
||||
logger.Infof("cleaning up orphaned OSD prepare job %q.", job.Name)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "cleaning up orphaned OSD prepare job %q.", job.Name)
|
||||
if err := c.context.Clientset.BatchV1().Jobs(c.clusterInfo.Namespace).Delete(c.clusterInfo.Context, job.Name, metav1.DeleteOptions{}); err != nil {
|
||||
logger.Warningf("failed to clean up OSD prepare job %q. %v", job.Name, err)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to clean up OSD prepare job %q. %v", job.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
cephkey "github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -312,14 +313,14 @@ func deploymentName(osdID int) string {
|
||||
|
||||
func (c *Cluster) updateCephConfigVolume(volumes []v1.Volume, nodeName string) []v1.Volume {
|
||||
if _, ok := c.nodeConfigmaps[nodeName]; !ok {
|
||||
logger.Debugf("no configmap to override for node %q", nodeName)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "no configmap to override for node %q", nodeName)
|
||||
return volumes
|
||||
}
|
||||
|
||||
updatedVolumes := []v1.Volume{}
|
||||
for _, volume := range volumes {
|
||||
if volume.Name == k8sutil.ConfigOverrideName {
|
||||
logger.Debugf("found configmap volume to override for node %q", nodeName)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "found configmap volume to override for node %q", nodeName)
|
||||
name := fmt.Sprintf("%s-%s", k8sutil.ConfigOverrideName, nodeName)
|
||||
volume.VolumeSource.Projected.Sources[0].ConfigMap.LocalObjectReference.Name = name
|
||||
}
|
||||
@@ -351,12 +352,12 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provision
|
||||
}
|
||||
|
||||
if c.spec.Storage.AllowDeviceClassUpdate && osdProps.storeConfig.DeviceClass != "" && osdProps.storeConfig.DeviceClass != osd.DeviceClass {
|
||||
logger.Infof("The device class for osd %d is changing from %q to %q", osd.ID, osd.DeviceClass, osdProps.storeConfig.DeviceClass)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "The device class for osd %d is changing from %q to %q", osd.ID, osd.DeviceClass, osdProps.storeConfig.DeviceClass)
|
||||
osd.DeviceClass = osdProps.storeConfig.DeviceClass
|
||||
}
|
||||
// Assign the resources specific to this device class
|
||||
if resources, ok := cephv1.GetOSDResourcesForDeviceClass(c.spec.Resources, osd.DeviceClass); ok {
|
||||
logger.Debugf("assigning resources for device class %q to osd %d: %+v", osd.DeviceClass, osd.ID, resources)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "assigning resources for device class %q to osd %d: %+v", osd.DeviceClass, osd.ID, resources)
|
||||
osdProps.resources = resources
|
||||
}
|
||||
|
||||
@@ -560,7 +561,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provision
|
||||
return nil, errors.Wrapf(err, "failed to export service %q", osdService.Name)
|
||||
}
|
||||
|
||||
logger.Infof("osd.%d exported IP is %q", osd.ID, exportedIP)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "osd.%d exported IP is %q", osd.ID, exportedIP)
|
||||
|
||||
args = append(args, []string{
|
||||
fmt.Sprintf("--public-addr=%s", exportedIP),
|
||||
@@ -793,7 +794,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provision
|
||||
}
|
||||
if osdProps.portable {
|
||||
// portable OSDs must have affinity to the topology where the osd prepare job was executed
|
||||
if err := applyTopologyAffinity(&deployment.Spec.Template.Spec, *osd); err != nil {
|
||||
if err := c.applyTopologyAffinity(&deployment.Spec.Template.Spec, *osd); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
@@ -848,7 +849,7 @@ func (c *Cluster) createOSDService(osd OSDInfo, labels map[string]string) (*v1.S
|
||||
return nil, err
|
||||
}
|
||||
|
||||
logger.Infof("osd.%d cluster IP is %q", osd.ID, svc.Spec.ClusterIP)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "osd.%d cluster IP is %q", osd.ID, svc.Spec.ClusterIP)
|
||||
|
||||
return svc, nil
|
||||
}
|
||||
@@ -869,12 +870,12 @@ func (c *Cluster) applyAllPlacementIfNeeded(d *v1.PodSpec) {
|
||||
}
|
||||
}
|
||||
|
||||
func applyTopologyAffinity(spec *v1.PodSpec, osd OSDInfo) error {
|
||||
func (c *Cluster) applyTopologyAffinity(spec *v1.PodSpec, osd OSDInfo) error {
|
||||
if osd.TopologyAffinity == "" {
|
||||
logger.Debugf("no topology affinity to set for osd %d", osd.ID)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "no topology affinity to set for osd %d", osd.ID)
|
||||
return nil
|
||||
}
|
||||
logger.Infof("assigning osd %d topology affinity to %q", osd.ID, osd.TopologyAffinity)
|
||||
log.NamespacedInfo(osd.Cluster, logger, "assigning osd %d topology affinity to %q", osd.ID, osd.TopologyAffinity)
|
||||
nodeAffinity, err := k8sutil.GenerateNodeAffinity(osd.TopologyAffinity)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to generate osd %d topology affinity", osd.ID)
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -176,7 +177,7 @@ func (c *Cluster) updateAndCreateOSDs(
|
||||
if !doLoop {
|
||||
return err
|
||||
}
|
||||
logger.Errorf("%v", err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -248,20 +249,20 @@ func (c *Cluster) updateAndCreateOSDsLoop(
|
||||
select {
|
||||
case event, ok := <-watcher.ResultChan():
|
||||
if !ok {
|
||||
logger.Infof("restarting watcher for OSD provisioning status ConfigMaps. the watcher closed the channel")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "restarting watcher for OSD provisioning status ConfigMaps. the watcher closed the channel")
|
||||
return true, nil
|
||||
}
|
||||
|
||||
if !isAddOrModifyEvent(event.Type) {
|
||||
// We don't want to process delete events when we delete configmaps after having
|
||||
// processed them. We also don't want to process BOOKMARK or ERROR events.
|
||||
logger.Debugf("not processing %s event for object %q", event.Type, eventObjectName(event))
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "not processing %s event for object %q", event.Type, eventObjectName(event))
|
||||
break // case
|
||||
}
|
||||
|
||||
configMap, ok := event.Object.(*corev1.ConfigMap)
|
||||
if !ok {
|
||||
logger.Errorf("recovering. %s. expected type ConfigMap but found %T", watchErrMsg, configMap)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "recovering. %s. expected type ConfigMap but found %T", watchErrMsg, configMap)
|
||||
break // case
|
||||
}
|
||||
|
||||
@@ -275,11 +276,11 @@ func (c *Cluster) updateAndCreateOSDsLoop(
|
||||
// Log progress
|
||||
c, cExp := createConfig.progress()
|
||||
u, uExp := updateConfig.progress()
|
||||
logger.Infof("waiting... %d of %d OSD prepare jobs have finished processing and %d of %d OSDs have been updated", c, cExp, u, uExp)
|
||||
log.NamespacedInfo(namespace, logger, "waiting... %d of %d OSD prepare jobs have finished processing and %d of %d OSDs have been updated", c, cExp, u, uExp)
|
||||
|
||||
// keep track of each time progress is made
|
||||
if c != prevCreatedCount || u != prevUpdatedCount {
|
||||
logger.Debugf("cluster in namespace %q, progress was made processing OSDs: %d nodes/OSDs were processed for creation and %d updated", namespace, c-prevCreatedCount, u-prevUpdatedCount)
|
||||
log.NamespacedDebug(namespace, logger, "cluster in namespace %q, progress was made processing OSDs: %d nodes/OSDs were processed for creation and %d updated", namespace, c-prevCreatedCount, u-prevUpdatedCount)
|
||||
prevCreatedCount = c
|
||||
prevUpdatedCount = u
|
||||
prevChangeTime = time.Now().UTC()
|
||||
@@ -292,7 +293,7 @@ func (c *Cluster) updateAndCreateOSDsLoop(
|
||||
}
|
||||
|
||||
case <-c.clusterInfo.Context.Done():
|
||||
logger.Infof("context cancelled, exiting OSD update and create loop")
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "context cancelled, exiting OSD update and create loop")
|
||||
return false, c.clusterInfo.Context.Err()
|
||||
}
|
||||
}
|
||||
@@ -327,7 +328,7 @@ func (c *Cluster) createOSDsForStatusMap(
|
||||
) {
|
||||
nodeOrPVCName, ok := configMap.Labels[nodeLabelKey]
|
||||
if !ok {
|
||||
logger.Warningf("missing node label on configmap %s", configMap.Name)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "missing node label on configmap %s", configMap.Name)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -340,7 +341,7 @@ func (c *Cluster) createOSDsForStatusMap(
|
||||
nodeOrPVC = "PVC"
|
||||
}
|
||||
|
||||
logger.Infof("OSD orchestration status for %s %s is %q", nodeOrPVC, nodeOrPVCName, status.Status)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "OSD orchestration status for %s %s is %q", nodeOrPVC, nodeOrPVCName, status.Status)
|
||||
|
||||
if status.Status == OrchestrationStatusCompleted {
|
||||
createConfig.createNewOSDsFromStatus(status, nodeOrPVCName, errs)
|
||||
@@ -362,7 +363,7 @@ func statusConfigMapName(nodeOrPVCName string) string {
|
||||
|
||||
func (c *Cluster) deleteStatusConfigMap(nodeOrPVCName string) {
|
||||
if err := c.kv.ClearStore(c.clusterInfo.Context, statusConfigMapName(nodeOrPVCName)); err != nil {
|
||||
logger.Errorf("failed to remove the status configmap %q. %v", statusConfigMapName(nodeOrPVCName), err)
|
||||
log.NamespacedError(c.clusterInfo.Namespace, logger, "failed to remove the status configmap %q. %v", statusConfigMapName(nodeOrPVCName), err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -373,14 +374,14 @@ func (c *Cluster) deleteAllStatusConfigMaps() {
|
||||
cmClientset := c.context.Clientset.CoreV1().ConfigMaps(c.clusterInfo.Namespace)
|
||||
cms, err := cmClientset.List(c.clusterInfo.Context, listOpts)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to clean up any dangling OSD prepare status configmaps. failed to list OSD prepare status configmaps. %v", err)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to clean up any dangling OSD prepare status configmaps. failed to list OSD prepare status configmaps. %v", err)
|
||||
return
|
||||
}
|
||||
for _, cm := range cms.Items {
|
||||
logger.Debugf("cleaning up dangling OSD prepare status configmap %q", cm.Name)
|
||||
log.NamespacedDebug(c.clusterInfo.Namespace, logger, "cleaning up dangling OSD prepare status configmap %q", cm.Name)
|
||||
err := cmClientset.Delete(c.clusterInfo.Context, cm.Name, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
logger.Warningf("failed to clean up dangling OSD prepare status configmap %q. %v", cm.Name, err)
|
||||
log.NamespacedWarning(c.clusterInfo.Namespace, logger, "failed to clean up dangling OSD prepare status configmap %q. %v", cm.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -85,14 +86,14 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
|
||||
if !c.cluster.spec.SkipUpgradeChecks && c.cluster.spec.UpgradeOSDRequiresHealthyPGs {
|
||||
pgHealthMsg, pgClean, err := cephclient.IsClusterClean(c.cluster.context, c.cluster.clusterInfo, c.cluster.spec.DisruptionManagement.PGHealthyRegex)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to check PGs status to update OSDs, will try updating it again later. %v", err)
|
||||
log.NamespacedWarning(c.cluster.clusterInfo.Namespace, logger, "failed to check PGs status to update OSDs, will try updating it again later. %v", err)
|
||||
return
|
||||
}
|
||||
if !pgClean {
|
||||
logger.Infof("PGs are not healthy to update OSDs, will try updating it again later. PGs status: %q", pgHealthMsg)
|
||||
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "PGs are not healthy to update OSDs, will try updating it again later. PGs status: %q", pgHealthMsg)
|
||||
return
|
||||
}
|
||||
logger.Infof("PGs are healthy to proceed updating OSDs. %v", pgHealthMsg)
|
||||
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "PGs are healthy to proceed updating OSDs. %v", pgHealthMsg)
|
||||
}
|
||||
|
||||
osdIDQuery, _ := c.queue.Pop()
|
||||
@@ -102,7 +103,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
|
||||
if c.cluster.spec.SkipUpgradeChecks || !shouldCheckOkToStopFunc(c.cluster.context, c.cluster.clusterInfo) {
|
||||
// If we should not check ok-to-stop, then only process one OSD at a time. There are likely
|
||||
// less than 3 OSDs in the cluster or the cluster is on a single node. E.g., in CI :wink:.
|
||||
logger.Infof("skipping osd checks for ok-to-stop")
|
||||
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "skipping osd checks for ok-to-stop")
|
||||
osdIDs = []int{osdIDQuery}
|
||||
} else {
|
||||
osdMaxUpdatesInParallel := c.cluster.spec.Storage.OSDMaxUpdatesInParallel
|
||||
@@ -112,29 +113,29 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
|
||||
osdIDs, err = cephclient.OSDOkToStop(c.cluster.context, c.cluster.clusterInfo, osdIDQuery, osdMaxUpdatesInParallel)
|
||||
if err != nil {
|
||||
if c.cluster.spec.ContinueUpgradeAfterChecksEvenIfNotHealthy {
|
||||
logger.Infof("OSD %d is not ok-to-stop but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so continuing to update it", osdIDQuery)
|
||||
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "OSD %d is not ok-to-stop but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so continuing to update it", osdIDQuery)
|
||||
osdIDs = []int{osdIDQuery} // make sure to update the queried OSD
|
||||
} else {
|
||||
logger.Infof("OSD %d is not ok-to-stop. will try updating it again later", osdIDQuery)
|
||||
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "OSD %d is not ok-to-stop. will try updating it again later", osdIDQuery)
|
||||
c.queue.Push(osdIDQuery) // push back onto queue to make sure we retry it later
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debugf("updating OSDs: %v", osdIDs)
|
||||
log.NamespacedDebug(c.cluster.clusterInfo.Namespace, logger, "updating OSDs: %v", osdIDs)
|
||||
|
||||
updatedDeployments := make([]*appsv1.Deployment, 0, len(osdIDs))
|
||||
listIDs := []string{} // use this to build the k8s api selector query
|
||||
for _, osdID := range osdIDs {
|
||||
if !c.deployments.Exists(osdID) {
|
||||
logger.Debugf("not updating deployment for OSD %d that is newly created", osdID)
|
||||
log.NamespacedDebug(c.cluster.clusterInfo.Namespace, logger, "not updating deployment for OSD %d that is newly created", osdID)
|
||||
continue
|
||||
}
|
||||
|
||||
// osdIDQuery which has been popped off the queue but it does need to be updated
|
||||
if osdID != osdIDQuery && !c.queue.Exists(osdID) {
|
||||
logger.Debugf("not updating deployment for OSD %d that is not in the update queue. the OSD has already been updated", osdID)
|
||||
log.NamespacedDebug(c.cluster.clusterInfo.Namespace, logger, "not updating deployment for OSD %d that is not in the update queue. the OSD has already been updated", osdID)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -152,7 +153,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
|
||||
c.osdDesiredState[osdID] = &osdInfo
|
||||
|
||||
if c.osdsToSkipReconcile.Has(strconv.Itoa(osdID)) {
|
||||
logger.Warningf("Skipping update for OSD %d since labeled with %s", osdID, cephv1.SkipReconcileLabelKey)
|
||||
log.NamespacedWarning(c.cluster.clusterInfo.Namespace, logger, "Skipping update for OSD %d since labeled with %s", osdID, cephv1.SkipReconcileLabelKey)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -162,7 +163,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
|
||||
if osdInfo.DeviceClass == "" || osdInfo.DeviceClass == "None" {
|
||||
deviceClassInfo, err := cephclient.OSDDeviceClasses(c.cluster.context, c.cluster.clusterInfo, []string{strconv.Itoa(osdID)})
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get device class for existing deployment %q. %v", depName, err)
|
||||
log.NamespacedError(c.cluster.clusterInfo.Namespace, logger, "failed to get device class for existing deployment %q. %v", depName, err)
|
||||
} else {
|
||||
osdInfo.DeviceClass = deviceClassInfo[0].DeviceClass
|
||||
}
|
||||
@@ -188,14 +189,14 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
|
||||
}
|
||||
|
||||
if osdIsOnPVC(dep) {
|
||||
logger.Infof("updating OSD %d on PVC %q", osdID, nodeOrPVCName)
|
||||
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "updating OSD %d on PVC %q", osdID, nodeOrPVCName)
|
||||
updatedDep, err = deploymentOnPVCFunc(c.cluster, &osdInfo, nodeOrPVCName, c.provisionConfig)
|
||||
message := fmt.Sprintf("Processing OSD %d on PVC %q", osdID, nodeOrPVCName)
|
||||
updateConditionFunc(c.cluster.clusterInfo.Context, c.cluster.context, c.cluster.clusterInfo.NamespacedName(), k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, message)
|
||||
} else {
|
||||
if !c.cluster.ValidStorage.NodeExists(nodeOrPVCName) {
|
||||
// node will not reconcile, so don't update the deployment
|
||||
logger.Warningf(
|
||||
log.NamespacedWarning(c.cluster.clusterInfo.Namespace, logger,
|
||||
"not updating OSD %d on node %q. node no longer exists in the storage spec. "+
|
||||
"if the user wishes to remove OSDs from the node, they must do so manually. "+
|
||||
"Rook will not remove OSDs from nodes that are removed from the storage spec in order to prevent accidental data loss",
|
||||
@@ -203,7 +204,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
|
||||
continue
|
||||
}
|
||||
|
||||
logger.Infof("updating OSD %d on node %q", osdID, nodeOrPVCName)
|
||||
log.NamespacedInfo(c.cluster.clusterInfo.Namespace, logger, "updating OSD %d on node %q", osdID, nodeOrPVCName)
|
||||
updatedDep, err = deploymentOnNodeFunc(c.cluster, &osdInfo, nodeOrPVCName, c.provisionConfig)
|
||||
message := fmt.Sprintf("Processing OSD %d on node %q", osdID, nodeOrPVCName)
|
||||
updateConditionFunc(c.cluster.clusterInfo.Context, c.cluster.context, c.cluster.clusterInfo.NamespacedName(), k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, message)
|
||||
@@ -424,7 +425,7 @@ func (c *Cluster) rotateCephxKey(osdInfo OSDInfo) (cephv1.CephxStatus, error) {
|
||||
return didNotRotateCephxStatus, nil
|
||||
}
|
||||
|
||||
logger.Infof("rotating cephx key of OSD %d for CephCluster in namespace %q", osdInfo.ID, c.clusterInfo.Namespace)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "rotating cephx key of OSD %d for CephCluster in namespace %q", osdInfo.ID, c.clusterInfo.Namespace)
|
||||
user := fmt.Sprintf("osd.%d", osdInfo.ID)
|
||||
// Note: OSD key is not stored in k8s secret; rotated key is picked up by OSD init container
|
||||
_, err = cephclient.AuthRotate(c.context, c.clusterInfo, user)
|
||||
@@ -435,7 +436,7 @@ func (c *Cluster) rotateCephxKey(osdInfo OSDInfo) (cephv1.CephxStatus, error) {
|
||||
// rotating the `client.osd-lockbox.$OSD_UUID` keys created for luks-encrypted OSDs by ceph-volume
|
||||
if osdInfo.Encrypted {
|
||||
osdLockBoxUser := fmt.Sprintf("client.osd-lockbox.%s", osdInfo.UUID)
|
||||
logger.Infof("rotating osd-lockbox cephx key of encrypted OSD %d for CephCluster in namespace %q", osdInfo.ID, c.clusterInfo.Namespace)
|
||||
log.NamespacedInfo(c.clusterInfo.Namespace, logger, "rotating osd-lockbox cephx key of encrypted OSD %d for CephCluster in namespace %q", osdInfo.ID, c.clusterInfo.Namespace)
|
||||
_, err = cephclient.AuthRotate(c.context, c.clusterInfo, osdLockBoxUser)
|
||||
if err != nil {
|
||||
return didNotRotateCephxStatus, errors.Wrapf(err, "failed to rotate osd-lockbox cephx key for the encrypted OSD %d", osdInfo.ID)
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
|
||||
kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
@@ -169,7 +170,7 @@ func getPVCOSDVolumes(osdProps *osdProperties, configDir string, namespace strin
|
||||
volumes = append(volumes, walPVCVolume...)
|
||||
}
|
||||
|
||||
logger.Debugf("volumes are %+v", volumes)
|
||||
log.NamespacedDebug(namespace, logger, "volumes are %+v", volumes)
|
||||
|
||||
return volumes
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
discoverDaemon "github.com/rook/rook/pkg/daemon/discover"
|
||||
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -139,7 +140,7 @@ func predicateForOperatorConfigCMWatcher[T *corev1.ConfigMap]() predicate.TypedF
|
||||
}
|
||||
|
||||
if objOld.Data["ROOK_USE_CSI_OPERATOR"] != objNew.Data["ROOK_USE_CSI_OPERATOR"] {
|
||||
logger.Infof("ROOK_USE_CSI_OPERATOR changed from %q to %q", objOld.Data["ROOK_USE_CSI_OPERATOR"], objNew.Data["ROOK_USE_CSI_OPERATOR"])
|
||||
log.NamespacedInfo(objNew.Namespace, logger, "ROOK_USE_CSI_OPERATOR changed from %q to %q", objOld.Data["ROOK_USE_CSI_OPERATOR"], objNew.Data["ROOK_USE_CSI_OPERATOR"])
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -196,18 +197,18 @@ func watchControllerPredicate[T *cephv1.CephCluster](ctx context.Context, c clie
|
||||
objOld := (*cephv1.CephCluster)(e.ObjectOld)
|
||||
objNew := (*cephv1.CephCluster)(e.ObjectNew)
|
||||
|
||||
logger.Debug("update event on CephCluster CR")
|
||||
log.NamespacedDebug(objNew.Namespace, logger, "update event on CephCluster CR")
|
||||
// If the labels "do_not_reconcile" is set on the object, let's not reconcile that request
|
||||
if opcontroller.IsDoNotReconcile(objNew.GetLabels()) {
|
||||
logger.Debugf("object %q matched on update but %q label is set, doing nothing", opcontroller.DoNotReconcileLabelName, objNew.Name)
|
||||
log.NamespacedDebug(objNew.Namespace, logger, "object %q matched on update but %q label is set, doing nothing", opcontroller.DoNotReconcileLabelName, objNew.Name)
|
||||
return false
|
||||
}
|
||||
diff := cmp.Diff(objOld.Spec, objNew.Spec, resourceQtyComparer)
|
||||
if diff != "" {
|
||||
logger.Infof("CR has changed for %q. diff=%s", objNew.Name, diff)
|
||||
log.NamespacedInfo(objNew.Namespace, logger, "CR has changed for %q. diff=%s", objNew.Name, diff)
|
||||
|
||||
if objNew.Spec.CleanupPolicy.HasDataDirCleanPolicy() {
|
||||
logger.Infof("skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set. not reloading the manager", objNew.GetName(), objNew.GetNamespace())
|
||||
log.NamespacedInfo(objNew.Namespace, logger, "skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set. not reloading the manager", objNew.GetName(), objNew.GetNamespace())
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -217,7 +218,7 @@ func watchControllerPredicate[T *cephv1.CephCluster](ctx context.Context, c clie
|
||||
return false
|
||||
|
||||
} else if !objOld.GetDeletionTimestamp().Equal(objNew.GetDeletionTimestamp()) {
|
||||
logger.Infof("CR %q is going be deleted, cancelling any ongoing orchestration", objNew.Name)
|
||||
log.NamespacedInfo(objNew.Namespace, logger, "CR %q is going be deleted, cancelling any ongoing orchestration", objNew.Name)
|
||||
|
||||
// Stop any ongoing orchestration
|
||||
opcontroller.ReloadManager()
|
||||
@@ -225,7 +226,7 @@ func watchControllerPredicate[T *cephv1.CephCluster](ctx context.Context, c clie
|
||||
return false
|
||||
|
||||
} else if objOld.GetGeneration() != objNew.GetGeneration() {
|
||||
logger.Debugf("reconciling CephCluster %q with changed generation", objNew.Name)
|
||||
log.NamespacedDebug(objNew.Namespace, logger, "reconciling CephCluster %q with changed generation", objNew.Name)
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
@@ -59,7 +60,7 @@ func (r *ReconcileCephRBDMirror) generateKeyring(clusterInfo *client.ClusterInfo
|
||||
}
|
||||
|
||||
if r.shouldRotateCephxKeys {
|
||||
logger.Infof("rotating cephx key for CephRBDMirror %q in the namespace %q", daemonConfig.ResourceName, clusterInfo.Namespace)
|
||||
log.NamespacedInfo(clusterInfo.Namespace, logger, "rotating cephx key for CephRBDMirror %q in the namespace %q", daemonConfig.ResourceName, clusterInfo.Namespace)
|
||||
newKey, err := s.RotateKey(user)
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "failed to rotate cephx key for CephRDBMirror %q in the namespace %q", daemonConfig.ResourceName, clusterInfo.Namespace)
|
||||
@@ -79,9 +80,9 @@ func (r *ReconcileCephRBDMirror) reconcileAddBootstrapPeer(cephRBDMirror *cephv1
|
||||
// List all the peers secret, we can have more than one peer we might want to configure
|
||||
// For each, get the Kubernetes Secret and import the "peer token" so that we can configure the mirroring
|
||||
|
||||
logger.Warning("(DEPRECATED) use of peer secret names in CephRBDMirror is deprecated. Please use CephBlockPool CR to configure peer secret names and import peers.")
|
||||
log.NamespacedWarning(cephRBDMirror.Namespace, logger, "(DEPRECATED) use of peer secret names in CephRBDMirror is deprecated. Please use CephBlockPool CR to configure peer secret names and import peers.")
|
||||
for _, peerSecret := range cephRBDMirror.Spec.Peers.SecretNames {
|
||||
logger.Debugf("fetching bootstrap peer kubernetes secret %q", peerSecret)
|
||||
log.NamespacedDebug(cephRBDMirror.Namespace, logger, "fetching bootstrap peer kubernetes secret %q", peerSecret)
|
||||
s, err := r.context.Clientset.CoreV1().Secrets(r.clusterInfo.Namespace).Get(r.opManagerContext, peerSecret, metav1.GetOptions{})
|
||||
// We don't care about IsNotFound here, we still need to fail
|
||||
if err != nil {
|
||||
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/ceph/reporting"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
@@ -168,7 +169,7 @@ func (r *ReconcileCephRBDMirror) Reconcile(context context.Context, request reco
|
||||
if err != nil {
|
||||
return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed set ready failure status to the cephRBDMirror %q", request.NamespacedName)
|
||||
}
|
||||
logger.Errorf("failed to reconcile %v", err)
|
||||
log.NamespacedError(request.Namespace, logger, "failed to reconcile %v", err)
|
||||
}
|
||||
|
||||
return reporting.ReportReconcileResult(logger, r.recorder, request, &cephRBDMirror, reconcileResponse, err)
|
||||
@@ -180,7 +181,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
|
||||
err := r.client.Get(r.opManagerContext, request.NamespacedName, cephRBDMirror)
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Debug("cephRBDMirror resource not found. Ignoring since object must be deleted.")
|
||||
log.NamespacedDebug(request.Namespace, logger, "cephRBDMirror resource not found. Ignoring since object must be deleted.")
|
||||
return reconcile.Result{}, *cephRBDMirror, nil
|
||||
}
|
||||
// Error reading the object - requeue the request.
|
||||
@@ -213,7 +214,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
|
||||
// Make sure a CephCluster is present otherwise do nothing
|
||||
cephCluster, isReadyToReconcile, _, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName)
|
||||
if !isReadyToReconcile {
|
||||
logger.Debugf("CephCluster resource not ready in namespace %q, retrying in %q.", request.NamespacedName.Namespace, reconcileResponse.RequeueAfter.String())
|
||||
log.NamedDebug(request.NamespacedName, logger, "CephCluster resource not ready, retrying in %q.", reconcileResponse.RequeueAfter.String())
|
||||
return reconcileResponse, *cephRBDMirror, nil
|
||||
}
|
||||
r.cephClusterSpec = &cephCluster.Spec
|
||||
@@ -238,7 +239,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
|
||||
)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
|
||||
logger.Info(opcontroller.OperatorNotInitializedMessage)
|
||||
log.NamespacedInfo(request.Namespace, logger, opcontroller.OperatorNotInitializedMessage)
|
||||
return opcontroller.WaitForRequeueIfOperatorNotInitialized, *cephRBDMirror, nil
|
||||
}
|
||||
return reconcile.Result{}, *cephRBDMirror, errors.Wrap(err, "failed to detect running and desired ceph version")
|
||||
@@ -256,7 +257,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
|
||||
r.clusterInfo.CephVersion = *runningCephVersion
|
||||
|
||||
// Add bootstrap peer if any
|
||||
logger.Debug("reconciling ceph rbd mirror peers addition")
|
||||
log.NamespacedDebug(request.Namespace, logger, "reconciling ceph rbd mirror peers addition")
|
||||
reconcileResponse, err = r.reconcileAddBootstrapPeer(cephRBDMirror, request.NamespacedName)
|
||||
if err != nil {
|
||||
return opcontroller.ImmediateRetryResult, *cephRBDMirror, errors.Wrap(err, "failed to add ceph rbd mirror peer")
|
||||
@@ -268,11 +269,11 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
|
||||
return reconcile.Result{}, *cephRBDMirror, errors.Wrapf(err, "failed to determine if cephx keys should be rotated for the cephRBDMirror %q", request.NamespacedName)
|
||||
}
|
||||
if r.shouldRotateCephxKeys {
|
||||
logger.Infof("cephx keys for CephRBDMirror %q will be rotated", request.NamespacedName)
|
||||
log.NamedInfo(request.NamespacedName, logger, "cephx keys for CephRBDMirror will be rotated")
|
||||
}
|
||||
|
||||
// CREATE/UPDATE
|
||||
logger.Debug("reconciling ceph rbd mirror deployments")
|
||||
log.NamespacedDebug(request.Namespace, logger, "reconciling ceph rbd mirror deployments")
|
||||
reconcileResponse, err = r.reconcileCreateCephRBDMirror(cephRBDMirror)
|
||||
if err != nil {
|
||||
return opcontroller.ImmediateRetryResult, *cephRBDMirror, errors.Wrap(err, "failed to create ceph rbd mirror deployments")
|
||||
@@ -288,7 +289,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
|
||||
}
|
||||
|
||||
// Return and do not requeue
|
||||
logger.Debug("done reconciling ceph rbd mirror")
|
||||
log.NamespacedDebug(request.Namespace, logger, "done reconciling ceph rbd mirror")
|
||||
return reconcile.Result{}, *cephRBDMirror, nil
|
||||
}
|
||||
|
||||
@@ -318,7 +319,7 @@ func (r *ReconcileCephRBDMirror) updateStatus(observedGeneration int64, namespac
|
||||
err := r.client.Get(r.opManagerContext, namespacedName, rbdMirror)
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Debug("CephRBDMirror resource not found. Ignoring since object must be deleted.")
|
||||
log.NamespacedDebug(namespacedName.Namespace, logger, "CephRBDMirror resource not found. Ignoring since object must be deleted.")
|
||||
return nil
|
||||
}
|
||||
return errors.Wrapf(err, "failed to retrieve CephRBDMirror %q to update status to %q", namespacedName, status)
|
||||
@@ -347,6 +348,6 @@ func (r *ReconcileCephRBDMirror) updateStatus(observedGeneration int64, namespac
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Debugf("successfully updated cephRBDMirror %q status updated to %+v", namespacedName, rbdMirror.Status)
|
||||
log.NamedDebug(namespacedName, logger, "successfully updated cephRBDMirror status updated to %+v", rbdMirror.Status)
|
||||
return nil
|
||||
}
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
@@ -51,7 +52,7 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro
|
||||
return errors.Wrap(err, "error checking pod memory")
|
||||
}
|
||||
|
||||
logger.Infof("configure rbd-mirroring with %d workers", cephRBDMirror.Spec.Count)
|
||||
log.NamespacedInfo(cephRBDMirror.Namespace, logger, "configure rbd-mirroring with %d workers", cephRBDMirror.Spec.Count)
|
||||
|
||||
ownerInfo := k8sutil.NewOwnerInfo(cephRBDMirror, r.scheme)
|
||||
daemonID := k8sutil.IndexToName(0)
|
||||
@@ -68,7 +69,7 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro
|
||||
return errors.Wrap(err, "failed to check for RBD Mirror to skip reconcile")
|
||||
}
|
||||
if rbdMirrorToSkipReconcile.Len() > 0 {
|
||||
logger.Warningf("skipping RBD mirror reconcile since RBD mirror daemons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(rbdMirrorToSkipReconcile))
|
||||
log.NamespacedWarning(cephRBDMirror.Namespace, logger, "skipping RBD mirror reconcile since RBD mirror daemons are labeled with %s: %v", cephv1.SkipReconcileLabelKey, sets.List(rbdMirrorToSkipReconcile))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -102,11 +103,11 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro
|
||||
if !kerrors.IsAlreadyExists(err) {
|
||||
return errors.Wrapf(err, "failed to create %q deployment", resourceName)
|
||||
}
|
||||
logger.Infof("deployment for rbd-mirror %q already exists. updating if needed", resourceName)
|
||||
log.NamespacedInfo(cephRBDMirror.Namespace, logger, "deployment for rbd-mirror %q already exists. updating if needed", resourceName)
|
||||
|
||||
if err := updateDeploymentAndWait(r.context, r.clusterInfo, d, config.RbdMirrorType, daemonConf.DaemonID, r.cephClusterSpec.SkipUpgradeChecks, false); err != nil {
|
||||
// fail could be an issue updating label selector (immutable), so try del and recreate
|
||||
logger.Debugf("updateDeploymentAndWait failed for rbd-mirror %q. Attempting del-and-recreate. %v", resourceName, err)
|
||||
log.NamespacedDebug(cephRBDMirror.Namespace, logger, "updateDeploymentAndWait failed for rbd-mirror %q. Attempting del-and-recreate. %v", resourceName, err)
|
||||
err = r.context.Clientset.AppsV1().Deployments(cephRBDMirror.Namespace).Delete(r.opManagerContext, d.Name, metav1.DeleteOptions{})
|
||||
if err != nil && !kerrors.IsNotFound(err) {
|
||||
return errors.Wrapf(err, "failed to delete rbd-mirror %q during del-and-recreate update attempt", resourceName)
|
||||
@@ -117,6 +118,6 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("%q deployment started", resourceName)
|
||||
log.NamespacedInfo(cephRBDMirror.Namespace, logger, "%q deployment started", resourceName)
|
||||
return nil
|
||||
}
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
)
|
||||
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "telemetry")
|
||||
@@ -53,8 +54,8 @@ var CSIVersion string
|
||||
func ReportKeyValue(context *clusterd.Context, clusterInfo *client.ClusterInfo, key, value string) {
|
||||
ms := config.GetMonStore(context, clusterInfo)
|
||||
if err := ms.SetKeyValue(key, value); err != nil {
|
||||
logger.Warningf("failed to set telemetry key %q. %v", key, err)
|
||||
log.NamespacedWarning(clusterInfo.Namespace, logger, "failed to set telemetry key %q. %v", key, err)
|
||||
return
|
||||
}
|
||||
logger.Debugf("set telemetry key: %s=%s", key, value)
|
||||
log.NamespacedDebug(clusterInfo.Namespace, logger, "set telemetry key: %s=%s", key, value)
|
||||
}
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -36,7 +37,7 @@ func populateConfigOverrideConfigMap(clusterdContext *clusterd.Context, namespac
|
||||
existingCM, err := clusterdContext.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, k8sutil.ConfigOverrideName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if !kerrors.IsNotFound(err) {
|
||||
logger.Warningf("failed to get cm %q to check labels and annotations", k8sutil.ConfigOverrideName)
|
||||
log.NamespacedWarning(namespace, logger, "failed to get cm %q to check labels and annotations", k8sutil.ConfigOverrideName)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -66,7 +67,7 @@ func populateConfigOverrideConfigMap(clusterdContext *clusterd.Context, namespac
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to create override configmap %s", namespace)
|
||||
}
|
||||
logger.Infof("created placeholder configmap for ceph overrides %q", cm.Name)
|
||||
log.NamespacedInfo(namespace, logger, "created placeholder configmap for ceph overrides %q", cm.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -83,9 +84,9 @@ func populateConfigOverrideConfigMap(clusterdContext *clusterd.Context, namespac
|
||||
if updateRequired {
|
||||
_, err = clusterdContext.Clientset.CoreV1().ConfigMaps(namespace).Update(ctx, existingCM, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
logger.Warningf("failed to add recommended labels and annotations to configmap %q. %v", existingCM.Name, err)
|
||||
log.NamespacedWarning(namespace, logger, "failed to add recommended labels and annotations to configmap %q. %v", existingCM.Name, err)
|
||||
} else {
|
||||
logger.Infof("added expected labels and annotations to configmap %q", existingCM.Name)
|
||||
log.NamespacedInfo(namespace, logger, "added expected labels and annotations to configmap %q", existingCM.Name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -96,15 +97,15 @@ func initRequiredMetadata(metadata metav1.ObjectMeta, labels, annotations map[st
|
||||
releaseNameAttr := "meta.helm.sh/release-name"
|
||||
chartName, ok := metadata.Annotations[releaseNameAttr]
|
||||
if !ok {
|
||||
logger.Debug("cluster helm chart is not configured, not adding helm annotations to configmap")
|
||||
log.NamespacedDebug(metadata.Namespace, logger, "cluster helm chart is not configured, not adding helm annotations to configmap")
|
||||
return false
|
||||
}
|
||||
if _, ok := annotations[releaseNameAttr]; ok {
|
||||
logger.Debug("cluster helm chart helm annotations already added to configmap")
|
||||
log.NamespacedDebug(metadata.Namespace, logger, "cluster helm chart helm annotations already added to configmap")
|
||||
return false
|
||||
}
|
||||
|
||||
logger.Infof("adding helm chart name %q annotation to configmap", chartName)
|
||||
log.NamespacedInfo(metadata.Namespace, logger, "adding helm chart name %q annotation to configmap", chartName)
|
||||
labels["app.kubernetes.io/managed-by"] = "Helm"
|
||||
annotations[releaseNameAttr] = chartName
|
||||
annotations["meta.helm.sh/release-namespace"] = metadata.Namespace
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/util/log"
|
||||
)
|
||||
|
||||
func (c *ClusterController) detectAndValidateCephVersion(cluster *cluster) (*cephver.CephVersion, bool, error) {
|
||||
@@ -40,7 +41,7 @@ func (c *ClusterController) detectAndValidateCephVersion(cluster *cluster) (*cep
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
logger.Info("validating ceph version from provided image")
|
||||
log.NamespacedInfo(cluster.Namespace, logger, "validating ceph version from provided image")
|
||||
if err := cluster.validateCephVersion(version); err != nil {
|
||||
return nil, cluster.isUpgrade, err
|
||||
}
|
||||
@@ -54,7 +55,7 @@ func (c *ClusterController) detectAndValidateCephVersion(cluster *cluster) (*cep
|
||||
func (c *cluster) printOverallCephVersion() {
|
||||
versions, err := daemonclient.GetAllCephDaemonVersions(c.context, c.ClusterInfo)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get ceph daemons versions. %v", err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to get ceph daemons versions. %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -62,21 +63,21 @@ func (c *cluster) printOverallCephVersion() {
|
||||
for v := range versions.Overall {
|
||||
version, err := cephver.ExtractCephVersion(v)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to extract ceph version. %v", err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to extract ceph version. %v", err)
|
||||
return
|
||||
}
|
||||
vv := *version
|
||||
logger.Infof("successfully upgraded cluster to version: %q", vv.String())
|
||||
log.NamespacedInfo(c.Namespace, logger, "successfully upgraded cluster to version: %q", vv.String())
|
||||
}
|
||||
} else {
|
||||
// This shouldn't happen, but let's log just in case
|
||||
logger.Warningf("upgrade orchestration completed but somehow we still have more than one Ceph version running. %v:", versions.Overall)
|
||||
log.NamespacedWarning(c.Namespace, logger, "upgrade orchestration completed but somehow we still have more than one Ceph version running. %v:", versions.Overall)
|
||||
}
|
||||
}
|
||||
|
||||
// This function compare the Ceph spec image and the cluster running version
|
||||
// It returns true if the image is different and false if identical
|
||||
func diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion, runningVersions cephv1.CephDaemonsVersions) (bool, error) {
|
||||
func (c *cluster) diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion, runningVersions cephv1.CephDaemonsVersions) (bool, error) {
|
||||
numberOfCephVersions := len(runningVersions.Overall)
|
||||
if numberOfCephVersions == 0 {
|
||||
// let's return immediately
|
||||
@@ -85,7 +86,7 @@ func diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion,
|
||||
|
||||
if numberOfCephVersions > 1 {
|
||||
// let's return immediately
|
||||
logger.Warningf("it looks like we have more than one ceph version running. triggering upgrade. %+v:", runningVersions.Overall)
|
||||
log.NamespacedWarning(c.Namespace, logger, "it looks like we have more than one ceph version running. triggering upgrade. %+v:", runningVersions.Overall)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -93,24 +94,24 @@ func diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion,
|
||||
for v := range runningVersions.Overall {
|
||||
version, err := cephver.ExtractCephVersion(v)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to extract ceph version. %v", err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to extract ceph version. %v", err)
|
||||
return false, err
|
||||
}
|
||||
clusterRunningVersion := *version
|
||||
|
||||
// If this is the same version
|
||||
if cephver.IsIdentical(clusterRunningVersion, imageSpecVersion) {
|
||||
logger.Debugf("both cluster and image spec versions are identical, doing nothing %s", imageSpecVersion.String())
|
||||
log.NamespacedDebug(c.Namespace, logger, "both cluster and image spec versions are identical, doing nothing %s", imageSpecVersion.String())
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if cephver.IsSuperior(imageSpecVersion, clusterRunningVersion) {
|
||||
logger.Infof("image spec version %s is higher than the running cluster version %s, upgrading", imageSpecVersion.String(), clusterRunningVersion.String())
|
||||
log.NamespacedInfo(c.Namespace, logger, "image spec version %s is higher than the running cluster version %s, upgrading", imageSpecVersion.String(), clusterRunningVersion.String())
|
||||
return true, nil
|
||||
}
|
||||
|
||||
if cephver.IsInferior(imageSpecVersion, clusterRunningVersion) {
|
||||
logger.Warningf("image spec version %s is lower than the running cluster version %s, downgrading is not supported", imageSpecVersion.String(), clusterRunningVersion.String())
|
||||
log.NamespacedWarning(c.Namespace, logger, "image spec version %s is lower than the running cluster version %s, downgrading is not supported", imageSpecVersion.String(), clusterRunningVersion.String())
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
@@ -129,11 +130,11 @@ func (c *cluster) validateCephVersion(version *cephver.CephVersion) error {
|
||||
if !c.Spec.CephVersion.AllowUnsupported {
|
||||
return errors.Errorf("allowUnsupported must be set to true to run with this version %q", version.String())
|
||||
}
|
||||
logger.Warningf("unsupported ceph version detected: %q, pursuing", version)
|
||||
log.NamespacedWarning(c.Namespace, logger, "unsupported ceph version detected: %q, pursuing", version)
|
||||
}
|
||||
|
||||
if version.Unsupported() {
|
||||
logger.Errorf("UNSUPPORTED: ceph version %q detected, it is recommended to rollback to the previous pin-point stable release, pursuing anyways", version)
|
||||
log.NamespacedError(c.Namespace, logger, "UNSUPPORTED: ceph version %q detected, it is recommended to rollback to the previous pin-point stable release, pursuing anyways", version)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,13 +150,13 @@ func (c *cluster) validateCephVersion(version *cephver.CephVersion) error {
|
||||
// Write connection info (ceph config file and keyring) for ceph commands
|
||||
err = mon.WriteConnectionConfig(c.context, clusterInfo)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to write config. attempting to continue. %v", err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to write config. attempting to continue. %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := clusterInfo.IsInitialized(); err != nil {
|
||||
// If not initialized, this is likely a new cluster so there is nothing to do
|
||||
logger.Debugf("cluster not initialized, nothing to validate. %v", err)
|
||||
log.NamespacedDebug(c.Namespace, logger, "cluster not initialized, nothing to validate. %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -170,21 +171,21 @@ func (c *cluster) validateCephVersion(version *cephver.CephVersion) error {
|
||||
// On external cluster setup, if we don't bootstrap any resources in the Kubernetes cluster then
|
||||
// there is no need to validate the Ceph image further
|
||||
if c.Spec.External.Enable && c.Spec.CephVersion.Image == "" {
|
||||
logger.Debug("no spec image specified on external cluster, not validating Ceph version.")
|
||||
log.NamespacedDebug(c.Namespace, logger, "no spec image specified on external cluster, not validating Ceph version.")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get cluster running versions
|
||||
versions, err := daemonclient.GetAllCephDaemonVersions(c.context, c.ClusterInfo)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get ceph daemons versions, this typically happens during the first cluster initialization. %v", err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to get ceph daemons versions, this typically happens during the first cluster initialization. %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
runningVersions := *versions
|
||||
differentImages, err := diffImageSpecAndClusterRunningVersion(*version, runningVersions)
|
||||
differentImages, err := c.diffImageSpecAndClusterRunningVersion(*version, runningVersions)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to determine if we should upgrade or not. %v", err)
|
||||
log.NamespacedError(c.Namespace, logger, "failed to determine if we should upgrade or not. %v", err)
|
||||
// we shouldn't block the orchestration if we can't determine the version of the image spec, we proceed anyway in best effort
|
||||
// we won't be able to check if there is an update or not and what to do, so we don't check the cluster status either
|
||||
// This will happen if someone uses ceph/daemon:latest-master for instance
|
||||
@@ -197,13 +198,13 @@ func (c *cluster) validateCephVersion(version *cephver.CephVersion) error {
|
||||
cephHealthy := daemonclient.IsCephHealthy(c.context, c.ClusterInfo)
|
||||
if !cephHealthy {
|
||||
if c.Spec.SkipUpgradeChecks {
|
||||
logger.Warning("ceph is not healthy but SkipUpgradeChecks is set, forcing upgrade.")
|
||||
log.NamespacedWarning(c.Namespace, logger, "ceph is not healthy but SkipUpgradeChecks is set, forcing upgrade.")
|
||||
} else {
|
||||
return errors.Errorf("ceph status in namespace %s is not healthy, refusing to upgrade. Either fix the health issue or force an update by setting skipUpgradeChecks to true in the cluster CR", c.Namespace)
|
||||
}
|
||||
}
|
||||
// This is an upgrade
|
||||
logger.Infof("upgrading ceph cluster to %q", version.String())
|
||||
log.NamespacedInfo(c.Namespace, logger, "upgrading ceph cluster to %q", version.String())
|
||||
c.isUpgrade = true
|
||||
}
|
||||
|
||||
|
||||
@@ -44,7 +44,8 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
|
||||
err := json.Unmarshal(fakeRunningVersions, &dummyRunningVersions)
|
||||
assert.NoError(t, err)
|
||||
|
||||
m, err := diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions)
|
||||
c := testSpec(t)
|
||||
m, err := c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions)
|
||||
assert.Error(t, err) // Overall is absent
|
||||
assert.False(t, m)
|
||||
|
||||
@@ -60,7 +61,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
|
||||
err = json.Unmarshal(fakeRunningVersions, &dummyRunningVersions2)
|
||||
assert.NoError(t, err)
|
||||
|
||||
m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions2)
|
||||
m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions2)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, m)
|
||||
|
||||
@@ -76,7 +77,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Allow the downgrade
|
||||
m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions3)
|
||||
m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions3)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, m)
|
||||
|
||||
@@ -92,7 +93,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
|
||||
err = json.Unmarshal(fakeRunningVersions, &dummyRunningVersions4)
|
||||
assert.NoError(t, err)
|
||||
|
||||
m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions4)
|
||||
m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions4)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, m)
|
||||
|
||||
@@ -111,7 +112,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
|
||||
err = json.Unmarshal(fakeRunningVersions, &dummyRunningVersions5)
|
||||
assert.NoError(t, err)
|
||||
|
||||
m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions5)
|
||||
m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions5)
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, m)
|
||||
|
||||
@@ -130,7 +131,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
|
||||
err = json.Unmarshal(fakeRunningVersions, &dummyRunningVersions6)
|
||||
assert.NoError(t, err)
|
||||
|
||||
m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions6)
|
||||
m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions6)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, m)
|
||||
|
||||
@@ -149,7 +150,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
|
||||
err = json.Unmarshal(fakeRunningVersions, &dummyRunningVersions7)
|
||||
assert.NoError(t, err)
|
||||
|
||||
m, err = diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions7)
|
||||
m, err = c.diffImageSpecAndClusterRunningVersion(fakeImageVersion, dummyRunningVersions7)
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, m)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
/*
|
||||
Copyright 2025 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package log
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
)
|
||||
|
||||
func NamedInfo(name types.NamespacedName, logger *capnslog.PackageLogger, message string, args ...interface{}) {
|
||||
NamespacedInfo(name.String(), logger, message, args...)
|
||||
}
|
||||
|
||||
func NamedWarning(name types.NamespacedName, logger *capnslog.PackageLogger, message string, args ...interface{}) {
|
||||
NamespacedWarning(name.String(), logger, message, args...)
|
||||
}
|
||||
|
||||
func NamedError(name types.NamespacedName, logger *capnslog.PackageLogger, message string, args ...interface{}) {
|
||||
NamespacedError(name.String(), logger, message, args...)
|
||||
}
|
||||
|
||||
func NamedDebug(name types.NamespacedName, logger *capnslog.PackageLogger, message string, args ...interface{}) {
|
||||
NamespacedDebug(name.String(), logger, message, args...)
|
||||
}
|
||||
|
||||
func NamedTrace(name types.NamespacedName, logger *capnslog.PackageLogger, message string, args ...interface{}) {
|
||||
NamespacedTrace(name.String(), logger, message, args...)
|
||||
}
|
||||
|
||||
func NamespacedInfo(namespace string, logger *capnslog.PackageLogger, message string, args ...interface{}) {
|
||||
logger.Infof("[%s] %s", namespace, fmt.Sprintf(message, args...))
|
||||
}
|
||||
|
||||
func NamespacedWarning(namespace string, logger *capnslog.PackageLogger, message string, args ...interface{}) {
|
||||
logger.Warningf("[%s] %s", namespace, fmt.Sprintf(message, args...))
|
||||
}
|
||||
|
||||
func NamespacedError(namespace string, logger *capnslog.PackageLogger, message string, args ...interface{}) {
|
||||
logger.Errorf("[%s] %s", namespace, fmt.Sprintf(message, args...))
|
||||
}
|
||||
|
||||
func NamespacedDebug(namespace string, logger *capnslog.PackageLogger, message string, args ...interface{}) {
|
||||
logger.Debugf("[%s] %s", namespace, fmt.Sprintf(message, args...))
|
||||
}
|
||||
|
||||
func NamespacedTrace(namespace string, logger *capnslog.PackageLogger, message string, args ...interface{}) {
|
||||
logger.Tracef("[%s] %s", namespace, fmt.Sprintf(message, args...))
|
||||
}
|
||||
Reference in new issue
Block a user