Merge pull request #5256 from travisn/review-cleanup

Reduce verbose logging for ok-to-stop failure
This commit is contained in:
Sébastien Han authored and GitHub committed 2020-04-15 10:30:08 +02:00
commit 2ad0d344b7
4 files changed
+19 -8

No files matched your search

+1 -1
View File
@@ -105,7 +105,7 @@ Settings can be specified at the global level to apply to the cluster as a whole
To ensure a consistent version of the image is running across all nodes in the cluster, it is recommended to use a very specific image version.
Tags also exist that would give the latest version, but they are only recommended for test environments. For example, the tag `v14` will be updated each time a new nautilus build is released.
Using the `v14` or similar tag is not recommended in production because it may lead to inconsistent versions of the image running across different nodes in the cluster.
* `allowUnsupported`: If `true`, allow an unsupported major version of the Ceph release. Currently `mimic` and `nautilus` are supported, so `octopus` would require this to be set to `true`. Should be set to `false` in production.
* `allowUnsupported`: If `true`, allow an unsupported major version of the Ceph release. Currently `nautilus` and `octopus` are supported. Future versions such as `pacific` would require this to be set to `true`. Should be set to `false` in production.
* `dataDirHostPath`: The path on the host ([hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath)) where config and data should be stored for each of the services. If the directory does not exist, it will be created. Because this directory persists on the host, it will remain after pods are deleted. Following paths and any of their subpaths **must not be used**: `/etc/ceph`, `/rook` or `/var/log/ceph`.
* On **Minikube** environments, use `/data/rook`. Minikube boots into a tmpfs but it provides some [directories](https://github.com/kubernetes/minikube/blob/master/docs/persistent_volumes.md) where files can be persisted across reboots. Using one of these directories will ensure that Rook's data and configuration files are persisted and that enough storage space is available.
* **WARNING**: For test scenarios, if you delete a cluster and start a new cluster on the same hosts, the path used by `dataDirHostPath` must be deleted. Otherwise, stale keys and other config will remain from the previous cluster and the new mons will fail to start.
+15 -4
View File
@@ -546,6 +546,15 @@ Here are some key lines to look for in the log:
```console
# A device will be skipped if Rook sees it has partitions or a filesystem
2019-05-30 19:02:57.353171 W | cephosd: skipping device sda that is in use
2019-05-30 19:02:57.452168 W | skipping device "sdb5": ["Used by ceph-disk"]
# Other messages about a disk being unusable by ceph include:
Insufficient space (<5GB) on vgs
Insufficient space (<5GB)
LVM detected
Has BlueStore device label
locked
read-only
# A device is going to be configured
2019-05-30 19:02:57.535598 I | cephosd: device sdc to be configured by ceph-volume
@@ -558,10 +567,12 @@ Here are some key lines to look for in the log:
### Solution
After you have either updated the CRD with the correct settings, or you have cleaned the partitions or filesystem from your devices,
you can trigger the operator to analyze the devices again by restarting the operator. Each time the operator starts, it
will ensure all the desired devices are configured. The operator does automatically deploy OSDs in most scenarios, but an operator restart
will cover any scenarios that the operator doesn't detect automatically.
Either update the CR with the correct settings, or clean the partitions or filesystem from your devices.
To clean devices from a previous install see the [cleanup guide](ceph-teardown.md#zapping-devices).
After the settings are updated or the devices are cleaned, trigger the operator to analyze the devices again by restarting the operator.
Each time the operator starts, it will ensure all the desired devices are configured. The operator does automatically
deploy OSDs in most scenarios, but an operator restart will cover any scenarios that the operator doesn't detect automatically.
```console
# Restart the operator to ensure devices are configured. A new pod will automatically be started when the current operator pod is deleted.
+1 -1
View File
@@ -183,7 +183,7 @@ func (c *Cluster) Start() error {
logger.Infof("deployment for mgr %s already exists. updating if needed", resourceName)
if err := updateDeploymentAndWait(c.context, d, c.Namespace, config.MgrType, mgrConfig.DaemonID, c.skipUpgradeChecks, false); err != nil {
logger.Errorf("failed to update mgr deployment %q. %+v", resourceName, err)
logger.Errorf("failed to update mgr deployment %q. %v", resourceName, err)
}
}
if existingDeployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(d.GetName(), metav1.GetOptions{}); err != nil {
+2 -2
View File
@@ -493,7 +493,7 @@ func (c *Cluster) startOSDDaemonsOnPVC(pvcName string, config *provisionConfig,
if createErr != nil && kerrors.IsAlreadyExists(createErr) {
if err = updateDeploymentAndWait(c.context, dp, c.Namespace, opconfig.OsdType, strconv.Itoa(osd.ID), c.skipUpgradeChecks, c.continueUpgradeAfterChecksEvenIfNotHealthy); err != nil {
logger.Errorf("failed to update osd deployment %d. %+v", osd.ID, err)
logger.Errorf("failed to update osd deployment %d. %v", osd.ID, err)
}
}
logger.Infof("started deployment for osd %d on pvc", osd.ID)
@@ -573,7 +573,7 @@ func (c *Cluster) startOSDDaemonsOnNode(nodeName string, config *provisionConfig
if createErr != nil && kerrors.IsAlreadyExists(createErr) {
if err = updateDeploymentAndWait(c.context, dp, c.Namespace, opconfig.OsdType, strconv.Itoa(osd.ID), c.skipUpgradeChecks, c.continueUpgradeAfterChecksEvenIfNotHealthy); err != nil {
logger.Errorf("failed to update osd deployment %d. %+v", osd.ID, err)
logger.Errorf("failed to update osd deployment %d. %v", osd.ID, err)
}
}
logger.Infof("started deployment for osd %d", osd.ID)