ceph: gave up to run the operator if the controller-runtime failed to start
The operator continues to run even though the controller-runtime failed to start. Since the controller-runtime is essential, the operator is non-functional after that. It makes troubleshooting more difficult. Closes: https://github.com/rook/rook/issues/5434 Signed-off-by: Satoru Takeuchi <satoru.takeuchi@gmail.com>
This commit is contained in:
1 parent
9b4edc2036
commit
f6a3da2760
2 files changed
+20
-8
No files matched your search
@@ -17,6 +17,7 @@ limitations under the License.
|
||||
package operator
|
||||
|
||||
import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster"
|
||||
"github.com/rook/rook/pkg/operator/ceph/disruption/controllerconfig"
|
||||
|
||||
@@ -24,7 +25,8 @@ import (
|
||||
"sigs.k8s.io/controller-runtime/pkg/manager"
|
||||
)
|
||||
|
||||
func (o *Operator) startManager(namespaceToWatch string, stopCh <-chan struct{}) {
|
||||
func (o *Operator) startManager(namespaceToWatch string, stopCh <-chan struct{},
|
||||
mgrErrorCh chan error) {
|
||||
// Set up a manager
|
||||
mgrOpts := manager.Options{
|
||||
LeaderElection: false,
|
||||
@@ -34,13 +36,13 @@ func (o *Operator) startManager(namespaceToWatch string, stopCh <-chan struct{})
|
||||
logger.Info("setting up the controller-runtime manager")
|
||||
kubeConfig, err := config.GetConfig()
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get client config for controller-runtime manager. %v", err)
|
||||
mgrErrorCh <- errors.Wrap(err, "failed to get client config for controller-runtime manager")
|
||||
return
|
||||
}
|
||||
|
||||
mgr, err := manager.New(kubeConfig, mgrOpts)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to set up overall controller-runtime manager. %v", err)
|
||||
mgrErrorCh <- errors.Wrap(err, "failed to set up overall controller-runtime manager")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -54,12 +56,13 @@ func (o *Operator) startManager(namespaceToWatch string, stopCh <-chan struct{})
|
||||
// Add the registered controllers to the manager (entrypoint for controllers)
|
||||
err = cluster.AddToManager(mgr, controllerOpts, o.clusterController)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to add controllers to controller-runtime manager. %v", err)
|
||||
mgrErrorCh <- errors.Wrap(err, "failed to add controllers to controller-runtime manager")
|
||||
return
|
||||
}
|
||||
|
||||
logger.Info("starting the controller-runtime manager")
|
||||
if err := mgr.Start(stopCh); err != nil {
|
||||
logger.Errorf("unable to run the controller-runtime manager. %v", err)
|
||||
mgrErrorCh <- errors.Wrap(err, "unable to run the controller-runtime manager")
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -100,6 +100,11 @@ func New(context *clusterd.Context, volumeAttachmentWrapper attachment.Attachmen
|
||||
return o
|
||||
}
|
||||
|
||||
func (o *Operator) cleanup(stopCh chan struct{}) {
|
||||
close(stopCh)
|
||||
o.clusterController.StopWatch()
|
||||
}
|
||||
|
||||
// Run the operator instance
|
||||
func (o *Operator) Run() error {
|
||||
|
||||
@@ -149,7 +154,8 @@ func (o *Operator) Run() error {
|
||||
}
|
||||
|
||||
// Start the controller-runtime Manager.
|
||||
go o.startManager(namespaceToWatch, stopChan)
|
||||
mgrErrorChan := make(chan error)
|
||||
go o.startManager(namespaceToWatch, stopChan, mgrErrorChan)
|
||||
|
||||
// Start the operator setting watcher
|
||||
go o.clusterController.StartOperatorSettingsWatch(namespaceToWatch, stopChan)
|
||||
@@ -159,9 +165,12 @@ func (o *Operator) Run() error {
|
||||
select {
|
||||
case <-signalChan:
|
||||
logger.Info("shutdown signal received, exiting...")
|
||||
close(stopChan)
|
||||
o.clusterController.StopWatch()
|
||||
o.cleanup(stopChan)
|
||||
return nil
|
||||
case err := <-mgrErrorChan:
|
||||
logger.Errorf("gave up to run the operator. %v", err)
|
||||
o.cleanup(stopChan)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user