ceph: gave up to run the operator if the controller-runtime failed to start

The operator continues to run even though the controller-runtime
failed to start. Since the controller-runtime is essential, the operator
is non-functional after that. It makes troubleshooting more difficult.

Closes: https://github.com/rook/rook/issues/5434

Signed-off-by: Satoru Takeuchi <satoru.takeuchi@gmail.com>
This commit is contained in:
Satoru Takeuchi committed 2020-05-18 17:38:35 +00:00
1 parent 9b4edc2036
commit f6a3da2760
2 files changed
+20 -8

No files matched your search

+8 -5
View File
@@ -17,6 +17,7 @@ limitations under the License.
package operator
import (
"github.com/pkg/errors"
"github.com/rook/rook/pkg/operator/ceph/cluster"
"github.com/rook/rook/pkg/operator/ceph/disruption/controllerconfig"
@@ -24,7 +25,8 @@ import (
"sigs.k8s.io/controller-runtime/pkg/manager"
)
func (o *Operator) startManager(namespaceToWatch string, stopCh <-chan struct{}) {
func (o *Operator) startManager(namespaceToWatch string, stopCh <-chan struct{},
mgrErrorCh chan error) {
// Set up a manager
mgrOpts := manager.Options{
LeaderElection: false,
@@ -34,13 +36,13 @@ func (o *Operator) startManager(namespaceToWatch string, stopCh <-chan struct{})
logger.Info("setting up the controller-runtime manager")
kubeConfig, err := config.GetConfig()
if err != nil {
logger.Errorf("failed to get client config for controller-runtime manager. %v", err)
mgrErrorCh <- errors.Wrap(err, "failed to get client config for controller-runtime manager")
return
}
mgr, err := manager.New(kubeConfig, mgrOpts)
if err != nil {
logger.Errorf("failed to set up overall controller-runtime manager. %v", err)
mgrErrorCh <- errors.Wrap(err, "failed to set up overall controller-runtime manager")
return
}
@@ -54,12 +56,13 @@ func (o *Operator) startManager(namespaceToWatch string, stopCh <-chan struct{})
// Add the registered controllers to the manager (entrypoint for controllers)
err = cluster.AddToManager(mgr, controllerOpts, o.clusterController)
if err != nil {
logger.Errorf("failed to add controllers to controller-runtime manager. %v", err)
mgrErrorCh <- errors.Wrap(err, "failed to add controllers to controller-runtime manager")
return
}
logger.Info("starting the controller-runtime manager")
if err := mgr.Start(stopCh); err != nil {
logger.Errorf("unable to run the controller-runtime manager. %v", err)
mgrErrorCh <- errors.Wrap(err, "unable to run the controller-runtime manager")
return
}
}
+12 -3
View File
@@ -100,6 +100,11 @@ func New(context *clusterd.Context, volumeAttachmentWrapper attachment.Attachmen
return o
}
func (o *Operator) cleanup(stopCh chan struct{}) {
close(stopCh)
o.clusterController.StopWatch()
}
// Run the operator instance
func (o *Operator) Run() error {
@@ -149,7 +154,8 @@ func (o *Operator) Run() error {
}
// Start the controller-runtime Manager.
go o.startManager(namespaceToWatch, stopChan)
mgrErrorChan := make(chan error)
go o.startManager(namespaceToWatch, stopChan, mgrErrorChan)
// Start the operator setting watcher
go o.clusterController.StartOperatorSettingsWatch(namespaceToWatch, stopChan)
@@ -159,9 +165,12 @@ func (o *Operator) Run() error {
select {
case <-signalChan:
logger.Info("shutdown signal received, exiting...")
close(stopChan)
o.clusterController.StopWatch()
o.cleanup(stopChan)
return nil
case err := <-mgrErrorChan:
logger.Errorf("gave up to run the operator. %v", err)
o.cleanup(stopChan)
return err
}
}
}