Merge pull request #5256 from travisn/review-cleanup
Reduce verbose logging for ok-to-stop failure
This commit is contained in:
4 files changed
+19
-8
No files matched your search
@@ -105,7 +105,7 @@ Settings can be specified at the global level to apply to the cluster as a whole
|
||||
To ensure a consistent version of the image is running across all nodes in the cluster, it is recommended to use a very specific image version.
|
||||
Tags also exist that would give the latest version, but they are only recommended for test environments. For example, the tag `v14` will be updated each time a new nautilus build is released.
|
||||
Using the `v14` or similar tag is not recommended in production because it may lead to inconsistent versions of the image running across different nodes in the cluster.
|
||||
* `allowUnsupported`: If `true`, allow an unsupported major version of the Ceph release. Currently `mimic` and `nautilus` are supported, so `octopus` would require this to be set to `true`. Should be set to `false` in production.
|
||||
* `allowUnsupported`: If `true`, allow an unsupported major version of the Ceph release. Currently `nautilus` and `octopus` are supported. Future versions such as `pacific` would require this to be set to `true`. Should be set to `false` in production.
|
||||
* `dataDirHostPath`: The path on the host ([hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath)) where config and data should be stored for each of the services. If the directory does not exist, it will be created. Because this directory persists on the host, it will remain after pods are deleted. Following paths and any of their subpaths **must not be used**: `/etc/ceph`, `/rook` or `/var/log/ceph`.
|
||||
* On **Minikube** environments, use `/data/rook`. Minikube boots into a tmpfs but it provides some [directories](https://github.com/kubernetes/minikube/blob/master/docs/persistent_volumes.md) where files can be persisted across reboots. Using one of these directories will ensure that Rook's data and configuration files are persisted and that enough storage space is available.
|
||||
* **WARNING**: For test scenarios, if you delete a cluster and start a new cluster on the same hosts, the path used by `dataDirHostPath` must be deleted. Otherwise, stale keys and other config will remain from the previous cluster and the new mons will fail to start.
|
||||
|
||||
@@ -546,6 +546,15 @@ Here are some key lines to look for in the log:
|
||||
```console
|
||||
# A device will be skipped if Rook sees it has partitions or a filesystem
|
||||
2019-05-30 19:02:57.353171 W | cephosd: skipping device sda that is in use
|
||||
2019-05-30 19:02:57.452168 W | skipping device "sdb5": ["Used by ceph-disk"]
|
||||
|
||||
# Other messages about a disk being unusable by ceph include:
|
||||
Insufficient space (<5GB) on vgs
|
||||
Insufficient space (<5GB)
|
||||
LVM detected
|
||||
Has BlueStore device label
|
||||
locked
|
||||
read-only
|
||||
|
||||
# A device is going to be configured
|
||||
2019-05-30 19:02:57.535598 I | cephosd: device sdc to be configured by ceph-volume
|
||||
@@ -558,10 +567,12 @@ Here are some key lines to look for in the log:
|
||||
|
||||
### Solution
|
||||
|
||||
After you have either updated the CRD with the correct settings, or you have cleaned the partitions or filesystem from your devices,
|
||||
you can trigger the operator to analyze the devices again by restarting the operator. Each time the operator starts, it
|
||||
will ensure all the desired devices are configured. The operator does automatically deploy OSDs in most scenarios, but an operator restart
|
||||
will cover any scenarios that the operator doesn't detect automatically.
|
||||
Either update the CR with the correct settings, or clean the partitions or filesystem from your devices.
|
||||
To clean devices from a previous install see the [cleanup guide](ceph-teardown.md#zapping-devices).
|
||||
|
||||
After the settings are updated or the devices are cleaned, trigger the operator to analyze the devices again by restarting the operator.
|
||||
Each time the operator starts, it will ensure all the desired devices are configured. The operator does automatically
|
||||
deploy OSDs in most scenarios, but an operator restart will cover any scenarios that the operator doesn't detect automatically.
|
||||
|
||||
```console
|
||||
# Restart the operator to ensure devices are configured. A new pod will automatically be started when the current operator pod is deleted.
|
||||
|
||||
@@ -183,7 +183,7 @@ func (c *Cluster) Start() error {
|
||||
logger.Infof("deployment for mgr %s already exists. updating if needed", resourceName)
|
||||
|
||||
if err := updateDeploymentAndWait(c.context, d, c.Namespace, config.MgrType, mgrConfig.DaemonID, c.skipUpgradeChecks, false); err != nil {
|
||||
logger.Errorf("failed to update mgr deployment %q. %+v", resourceName, err)
|
||||
logger.Errorf("failed to update mgr deployment %q. %v", resourceName, err)
|
||||
}
|
||||
}
|
||||
if existingDeployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(d.GetName(), metav1.GetOptions{}); err != nil {
|
||||
|
||||
@@ -493,7 +493,7 @@ func (c *Cluster) startOSDDaemonsOnPVC(pvcName string, config *provisionConfig,
|
||||
|
||||
if createErr != nil && kerrors.IsAlreadyExists(createErr) {
|
||||
if err = updateDeploymentAndWait(c.context, dp, c.Namespace, opconfig.OsdType, strconv.Itoa(osd.ID), c.skipUpgradeChecks, c.continueUpgradeAfterChecksEvenIfNotHealthy); err != nil {
|
||||
logger.Errorf("failed to update osd deployment %d. %+v", osd.ID, err)
|
||||
logger.Errorf("failed to update osd deployment %d. %v", osd.ID, err)
|
||||
}
|
||||
}
|
||||
logger.Infof("started deployment for osd %d on pvc", osd.ID)
|
||||
@@ -573,7 +573,7 @@ func (c *Cluster) startOSDDaemonsOnNode(nodeName string, config *provisionConfig
|
||||
|
||||
if createErr != nil && kerrors.IsAlreadyExists(createErr) {
|
||||
if err = updateDeploymentAndWait(c.context, dp, c.Namespace, opconfig.OsdType, strconv.Itoa(osd.ID), c.skipUpgradeChecks, c.continueUpgradeAfterChecksEvenIfNotHealthy); err != nil {
|
||||
logger.Errorf("failed to update osd deployment %d. %+v", osd.ID, err)
|
||||
logger.Errorf("failed to update osd deployment %d. %v", osd.ID, err)
|
||||
}
|
||||
}
|
||||
logger.Infof("started deployment for osd %d", osd.ID)
|
||||
|
||||
Reference in new issue
Block a user