From f25e4a6fbc4bf18c3e7981f751acff6c9185da76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Wed, 21 Aug 2019 14:06:20 +0200 Subject: [PATCH] ceph: re-adapt controller code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-adapt controller code with the latest changes. Split External and not external more accuratly in initializeCluster. Signed-off-by: Sébastien Han --- pkg/operator/ceph/cluster/cluster.go | 91 ++++++++-------- pkg/operator/ceph/cluster/controller.go | 132 +++++++++++++----------- 2 files changed, 114 insertions(+), 109 deletions(-) diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 2846b094f..d161e92a0 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -230,53 +230,52 @@ func (c *cluster) doOrchestration(rookImage string, cephVersion cephver.CephVers } // This gets triggered on CR update so let's not run that (mon/mgr/osd daemons) - if !spec.External.Enable { - // Start the mon pods - clusterInfo, err := c.mons.Start(c.Info, rookImage, cephVersion, *c.Spec) - if err != nil { - return fmt.Errorf("failed to start the mons. %+v", err) - } - c.Info = clusterInfo // mons return the cluster's info - - // The cluster Identity must be established at this point - if !c.Info.IsInitialized() { - return fmt.Errorf("the cluster identity was not established: %+v", c.Info) - } - - mgrs := mgr.New(c.Info, c.context, c.Namespace, rookImage, - spec.CephVersion, cephv1.GetMgrPlacement(spec.Placement), cephv1.GetMgrAnnotations(c.Spec.Annotations), - spec.Network.HostNetwork, spec.Dashboard, spec.Monitoring, cephv1.GetMgrResources(spec.Resources), c.ownerRef, c.Spec.DataDirHostPath) - err = mgrs.Start() - if err != nil { - return fmt.Errorf("failed to start the ceph mgr. %+v", err) - } - - // Start the OSDs - osds := osd.New(c.Info, c.context, c.Namespace, rookImage, spec.CephVersion, spec.Storage, spec.DataDirHostPath, - cephv1.GetOSDPlacement(spec.Placement), cephv1.GetOSDAnnotations(spec.Annotations), spec.Network.HostNetwork, - cephv1.GetOSDResources(spec.Resources), c.ownerRef) - err = osds.Start() - if err != nil { - return fmt.Errorf("failed to start the osds. %+v", err) - } - - // Start the rbd mirroring daemon(s) - rbdmirror := rbd.New(c.Info, c.context, c.Namespace, rookImage, spec.CephVersion, cephv1.GetRBDMirrorPlacement(spec.Placement), - cephv1.GetRBDMirrorAnnotations(spec.Annotations), spec.Network.HostNetwork, spec.RBDMirroring, - cephv1.GetRBDMirrorResources(spec.Resources), c.ownerRef, c.Spec.DataDirHostPath) - err = rbdmirror.Start() - if err != nil { - return fmt.Errorf("failed to start the rbd mirrors. %+v", err) - } - - logger.Infof("Done creating rook instance in namespace %s", c.Namespace) - c.initCompleted = true - - // Notify the child controllers that the cluster spec might have changed - for _, child := range c.childControllers { - child.ParentClusterChanged(*c.Spec, clusterInfo) - } + // Start the mon pods + clusterInfo, err := c.mons.Start(c.Info, rookImage, cephVersion, *c.Spec) + if err != nil { + return fmt.Errorf("failed to start the mons. %+v", err) } + c.Info = clusterInfo // mons return the cluster's info + + // The cluster Identity must be established at this point + if !c.Info.IsInitialized() { + return fmt.Errorf("the cluster identity was not established: %+v", c.Info) + } + + mgrs := mgr.New(c.Info, c.context, c.Namespace, rookImage, + spec.CephVersion, cephv1.GetMgrPlacement(spec.Placement), cephv1.GetMgrAnnotations(c.Spec.Annotations), + spec.Network.HostNetwork, spec.Dashboard, spec.Monitoring, cephv1.GetMgrResources(spec.Resources), c.ownerRef, c.Spec.DataDirHostPath) + err = mgrs.Start() + if err != nil { + return fmt.Errorf("failed to start the ceph mgr. %+v", err) + } + + // Start the OSDs + osds := osd.New(c.Info, c.context, c.Namespace, rookImage, spec.CephVersion, spec.Storage, spec.DataDirHostPath, + cephv1.GetOSDPlacement(spec.Placement), cephv1.GetOSDAnnotations(spec.Annotations), spec.Network.HostNetwork, + cephv1.GetOSDResources(spec.Resources), c.ownerRef) + err = osds.Start() + if err != nil { + return fmt.Errorf("failed to start the osds. %+v", err) + } + + // Start the rbd mirroring daemon(s) + rbdmirror := rbd.New(c.Info, c.context, c.Namespace, rookImage, spec.CephVersion, cephv1.GetRBDMirrorPlacement(spec.Placement), + cephv1.GetRBDMirrorAnnotations(spec.Annotations), spec.Network.HostNetwork, spec.RBDMirroring, + cephv1.GetRBDMirrorResources(spec.Resources), c.ownerRef, c.Spec.DataDirHostPath) + err = rbdmirror.Start() + if err != nil { + return fmt.Errorf("failed to start the rbd mirrors. %+v", err) + } + + logger.Infof("Done creating rook instance in namespace %s", c.Namespace) + c.initCompleted = true + + // Notify the child controllers that the cluster spec might have changed + for _, child := range c.childControllers { + child.ParentClusterChanged(*c.Spec, clusterInfo) + } + return nil } diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 44b224b60..09438bd69 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -265,57 +265,71 @@ func (c *ClusterController) configureExternalCephCluster(namespace, name string, logger.Warningf("waiting for the connection info of the external cluster. %+v", err) time.Sleep(10 * time.Second) continue + } else { + break } - logger.Infof("found the cluster info to connect to the external cluster. mons=%+v", cluster.Info.Monitors) - - // Write the rook-ceph-config configmap (used by various daemons to apply config overrides) - // If we don't do this, daemons will never start, waiting forever for this configmap to be present - err = config.GetStore(cluster.context, namespace, &cluster.ownerRef).CreateOrUpdate(cluster.Info) - if err != nil { - return err - } - - // Let's write connection info (ceph config file and keyring) to the operator for health checks - err = mon.WriteConnectionConfig(cluster.context, cluster.Info) - if err != nil { - return fmt.Errorf("failed to write connection info %+v", err) - } - - // Get Ceph monitors version on the external cluster - cephMonVersion, err := client.GetCephMonVersion(c.context, namespace) - if err != nil { - return fmt.Errorf("failed to get ceph mon version. %+v", err) - } - - logger.Infof("detecting the image version provided for the external cluster...") - specCephVersionImage, err := cluster.detectCephVersion(c.rookImage, cluster.Spec.CephVersion.Image, detectCephVersionTimeout) - if err != nil { - return fmt.Errorf("unknown ceph major version. %+v", err) - } - specCephVersion := *specCephVersionImage - - // Populate clusterInfo with the external cluster version - cluster.Info.CephVersion = *cephMonVersion - - // Make sure the external cluster version and the ceph version in the provided image are identical - if !cephver.IsIdentical(specCephVersion, cluster.Info.CephVersion) { - return fmt.Errorf("wrong ceph version %s, external cluster version is %s, they must match", specCephVersion.String(), cephMonVersion.String()) - } - - // The cluster Identity must be established at this point - if !cluster.Info.IsInitialized() { - return fmt.Errorf("the cluster identity was not established: %+v", cluster.Info) - } - logger.Info("cluster identity established.") - - // Everything went well so let's update the CR's status to "connected" - c.updateClusterStatus(namespace, name, cephv1.ClusterStateConnected, "") - - return nil } + + logger.Infof("found the cluster info to connect to the external cluster. mons=%+v", cluster.Info.Monitors) + + // Write the rook-ceph-config configmap (used by various daemons to apply config overrides) + // If we don't do this, daemons will never start, waiting forever for this configmap to be present + err = config.GetStore(cluster.context, namespace, &cluster.ownerRef).CreateOrUpdate(cluster.Info) + if err != nil { + return err + } + + // Let's write connection info (ceph config file and keyring) to the operator for health checks + err = mon.WriteConnectionConfig(cluster.context, cluster.Info) + if err != nil { + return fmt.Errorf("failed to write connection info %+v", err) + } + + // Get Ceph monitors version on the external cluster + cephMonVersion, err := client.GetCephMonVersion(c.context, namespace) + if err != nil { + return fmt.Errorf("failed to get ceph mon version. %+v", err) + } + + logger.Infof("detecting the image version provided for the external cluster...") + specCephVersionImage, err := cluster.detectCephVersion(c.rookImage, cluster.Spec.CephVersion.Image, detectCephVersionTimeout) + if err != nil { + return fmt.Errorf("unknown ceph major version. %+v", err) + } + specCephVersion := *specCephVersionImage + + // Populate clusterInfo with the external cluster version + cluster.Info.CephVersion = *cephMonVersion + + // Make sure the external cluster version and the ceph version in the provided image are identical + if !cephver.IsIdentical(specCephVersion, cluster.Info.CephVersion) { + return fmt.Errorf("wrong ceph version %s, external cluster version is %s, they must match", specCephVersion.String(), cephMonVersion.String()) + } + + // The cluster Identity must be established at this point + if !cluster.Info.IsInitialized() { + return fmt.Errorf("the cluster identity was not established: %+v", cluster.Info) + } + logger.Info("cluster identity established.") + + // Everything went well so let's update the CR's status to "connected" + c.updateClusterStatus(namespace, name, cephv1.ClusterStateConnected, "") + + // Mark initialization has done + cluster.initCompleted = true + + return nil } func (c *ClusterController) configureLocalCephCluster(namespace, name string, cluster *cluster, clusterObj *cephv1.CephCluster) error { + if cluster.Spec.Mon.Count == 0 { + logger.Warningf("mon count should be at least 1, will use default value of %d", mon.DefaultMonCount) + cluster.Spec.Mon.Count = mon.DefaultMonCount + } + if cluster.Spec.Mon.Count%2 == 0 { + logger.Warningf("mon count is even (given: %d), should be uneven, continuing", cluster.Spec.Mon.Count) + } + if c.devicesInUse && cluster.Spec.Storage.AnyUseAllDevices() { c.updateClusterStatus(clusterObj.Namespace, clusterObj.Name, cephv1.ClusterStateError, "using all devices in more than one namespace is not supported") return fmt.Errorf("using all devices in more than one namespace is not supported") @@ -325,19 +339,6 @@ func (c *ClusterController) configureLocalCephCluster(namespace, name string, cl c.devicesInUse = true } - return nil -} - -func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *cephv1.CephCluster) { - cluster.Spec = &clusterObj.Spec - if cluster.Spec.Mon.Count == 0 { - logger.Warningf("mon count should be at least 1, will use default value of %d", mon.DefaultMonCount) - cluster.Spec.Mon.Count = mon.DefaultMonCount - } - if cluster.Spec.Mon.Count%2 == 0 { - logger.Warningf("mon count is even (given: %d), should be uneven, continuing", cluster.Spec.Mon.Count) - } - // Start the Rook cluster components. Retry several times in case of failure. failedMessage := "" state := cephv1.ClusterStateError @@ -349,7 +350,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *ceph // Let's write connection info (ceph config file and keyring) to the operator for health checks err = mon.WriteConnectionConfig(cluster.context, cluster.Info) if err != nil { - return + return err } } @@ -384,9 +385,15 @@ func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *ceph if state == cephv1.ClusterStateError { // the cluster could not be initialized - return + return fmt.Errorf("failed to initialized the cluster") } + return nil +} + +func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *cephv1.CephCluster) { + cluster.Spec = &clusterObj.Spec + if !cluster.Spec.External.Enable { if err := c.configureLocalCephCluster(clusterObj.Namespace, clusterObj.Name, cluster, clusterObj); err != nil { logger.Errorf("failed to configure local ceph cluster. %+v", err) @@ -439,7 +446,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *ceph go cephChecker.checkCephStatus(cluster.stopCh) // add the finalizer to the crd - err = c.addFinalizer(clusterObj.Namespace, clusterObj.Name) + err := c.addFinalizer(clusterObj.Namespace, clusterObj.Name) if err != nil { logger.Errorf("failed to add finalizer to cluster crd. %+v", err) } @@ -622,7 +629,6 @@ func (c *ClusterController) onUpdate(oldObj, newObj interface{}) { // Display success after upgrade if versionChanged { printOverallCephVersion(c.context, cluster.Namespace) - // TODO: Update all the crd controllers that there is a new version of the ceph image to deploy } }