From 2d1c150539ddbfbf08c6535faca8a749f676d0a0 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Tue, 21 Mar 2017 10:26:51 -0700 Subject: [PATCH] detect if mons are healthy --- pkg/cephmgr/mon/config.go | 1 + pkg/cephmgr/mon/leader.go | 5 +- pkg/operator/{ => cluster}/cluster.go | 75 +++++++------ pkg/operator/{ => cluster}/cluster_list.go | 2 +- pkg/operator/{ => cluster}/cluster_test.go | 6 +- pkg/operator/cluster/spec.go | 41 +++++++ pkg/operator/event.go | 6 +- pkg/operator/mon/mon.go | 120 ++++++++++++++++----- pkg/operator/mon/pod.go | 5 +- pkg/operator/operator.go | 35 +++--- 10 files changed, 206 insertions(+), 90 deletions(-) rename pkg/operator/{ => cluster}/cluster.go (74%) rename pkg/operator/{ => cluster}/cluster_list.go (99%) rename pkg/operator/{ => cluster}/cluster_test.go (93%) create mode 100644 pkg/operator/cluster/spec.go diff --git a/pkg/cephmgr/mon/config.go b/pkg/cephmgr/mon/config.go index 3f961f511..88ad8f206 100644 --- a/pkg/cephmgr/mon/config.go +++ b/pkg/cephmgr/mon/config.go @@ -251,6 +251,7 @@ func ConnectToCluster(context *clusterd.Context, factory client.ConnectionFactor return nil, fmt.Errorf("failed to connect to cluster %s: %+v", cluster.Name, err) } + logger.Infof("successfully connected to cluster %s", cluster.Name) return conn, nil } diff --git a/pkg/cephmgr/mon/leader.go b/pkg/cephmgr/mon/leader.go index 22ec0813e..b7c2189cb 100644 --- a/pkg/cephmgr/mon/leader.go +++ b/pkg/cephmgr/mon/leader.go @@ -42,7 +42,7 @@ type Leader struct { } func NewLeader() *Leader { - return &Leader{waitForQuorum: waitForQuorum} + return &Leader{waitForQuorum: WaitForQuorum} } // Apply the desired state to the cluster. The context provides all the information needed to make changes to the service. @@ -356,7 +356,8 @@ func calculateMonitorCount(nodeCount int) int { } } -func waitForQuorum(factory client.ConnectionFactory, context *clusterd.Context, cluster *ClusterInfo) error { +func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context, cluster *ClusterInfo) error { + logger.Infof("waiting for mon quorum...") // open an admin connection to the cluster adminConn, err := ConnectToClusterAsAdmin(context, factory, cluster) diff --git a/pkg/operator/cluster.go b/pkg/operator/cluster/cluster.go similarity index 74% rename from pkg/operator/cluster.go rename to pkg/operator/cluster/cluster.go index 938a49a83..11d4df34e 100644 --- a/pkg/operator/cluster.go +++ b/pkg/operator/cluster/cluster.go @@ -16,13 +16,15 @@ limitations under the License. Some of the code below came from https://github.com/coreos/etcd-operator which also has the apache 2.0 license. */ -package operator +package cluster import ( "fmt" + "time" "k8s.io/client-go/pkg/api/errors" + "github.com/coreos/pkg/capnslog" "github.com/rook/rook/pkg/cephmgr/client" cephmon "github.com/rook/rook/pkg/cephmgr/mon" "github.com/rook/rook/pkg/clusterd" @@ -30,41 +32,33 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/mon" "github.com/rook/rook/pkg/operator/osd" + "github.com/rook/rook/pkg/operator/rgw" "k8s.io/client-go/kubernetes" "k8s.io/client-go/pkg/api/v1" ) +const ( + defaultPool = "rook" +) + +var ( + healthCheckInterval = 8 * time.Second + logger = capnslog.NewPackageLogger("github.com/rook/rook-operator", "op-cluster") +) + type Cluster struct { factory client.ConnectionFactory clientset kubernetes.Interface Metadata v1.ObjectMeta `json:"metadata,omitempty"` - Spec ClusterSpec `json:"spec"` + Spec `json:"spec"` dataDir string + mons *mon.Cluster + osds *osd.Cluster + apis *api.Cluster + rgws *rgw.Cluster } -type ClusterSpec struct { - // The namespace where the the rook resources will all be created. - Namespace string `json:"namespace"` - - // Version is the expected version of the rook container to run in the cluster. - // The rook-operator will eventually make the rook cluster version - // equal to the expected version. - Version string `json:"version"` - - // Paused is to pause the control of the operator for the rook cluster. - Paused bool `json:"paused,omitempty"` - - // Whether to consume all the storage devices found on a machine - UseAllDevices bool `json:"useAllDevices"` - - // A regular expression to allow more fine-grained selection of devices on nodes across the cluster - DeviceFilter string `json:"deviceFilter"` - - // The path on the host where config and data can be persisted. - DataDirHostPath string `json:"dataDirHostPath"` -} - -func newCluster(spec ClusterSpec, factory client.ConnectionFactory, clientset kubernetes.Interface) *Cluster { +func New(spec Spec, factory client.ConnectionFactory, clientset kubernetes.Interface) *Cluster { return &Cluster{ Spec: spec, factory: factory, @@ -85,26 +79,26 @@ func (c *Cluster) CreateInstance() error { } // Start the mon pods - m := mon.New(c.clientset, c.factory, c.Spec.Namespace, c.Spec.DataDirHostPath, c.Spec.Version) - cluster, err := m.Start() + c.mons = mon.New(c.clientset, c.factory, c.Spec.Namespace, c.Spec.DataDirHostPath, c.Spec.Version) + clusterInfo, err := c.mons.Start() if err != nil { return fmt.Errorf("failed to start the mons. %+v", err) } - a := api.New(c.clientset, c.Spec.Namespace, c.Spec.Version) - err = a.Start() + c.apis = api.New(c.clientset, c.Spec.Namespace, c.Spec.Version) + err = c.apis.Start() if err != nil { return fmt.Errorf("failed to start the REST api. %+v", err) } // Start the OSDs - osds := osd.New(c.clientset, c.Spec.Namespace, c.Spec.Version, c.Spec.DeviceFilter, c.Spec.DataDirHostPath, c.Spec.UseAllDevices) - err = osds.Start() + c.osds = osd.New(c.clientset, c.Spec.Namespace, c.Spec.Version, c.Spec.DeviceFilter, c.Spec.DataDirHostPath, c.Spec.UseAllDevices) + err = c.osds.Start() if err != nil { return fmt.Errorf("failed to start the osds. %+v", err) } - err = c.createClientAccess(cluster) + err = c.createClientAccess(clusterInfo) if err != nil { return fmt.Errorf("failed to create client access. %+v", err) } @@ -113,6 +107,23 @@ func (c *Cluster) CreateInstance() error { return nil } +func (c *Cluster) Monitor(stopCh <-chan struct{}) { + for { + select { + case <-stopCh: + logger.Infof("Stopping monitoring of cluster %s", c.Spec.Namespace) + return + + case <-time.After(healthCheckInterval): + logger.Infof("checking health of mons") + err := c.mons.CheckHealth() + if err != nil { + logger.Infof("failed to check mon health. %+v", err) + } + } + } +} + func (c *Cluster) createClientAccess(clusterInfo *cephmon.ClusterInfo) error { context := &clusterd.Context{ConfigDir: c.dataDir} conn, err := cephmon.ConnectToClusterAsAdmin(context, c.factory, clusterInfo) diff --git a/pkg/operator/cluster_list.go b/pkg/operator/cluster/cluster_list.go similarity index 99% rename from pkg/operator/cluster_list.go rename to pkg/operator/cluster/cluster_list.go index 4ee1cbe9a..33801791c 100644 --- a/pkg/operator/cluster_list.go +++ b/pkg/operator/cluster/cluster_list.go @@ -16,7 +16,7 @@ limitations under the License. Some of the code below came from https://github.com/coreos/etcd-operator which also has the apache 2.0 license. */ -package operator +package cluster import ( "encoding/json" diff --git a/pkg/operator/cluster_test.go b/pkg/operator/cluster/cluster_test.go similarity index 93% rename from pkg/operator/cluster_test.go rename to pkg/operator/cluster/cluster_test.go index e4d9cc6c0..77d34e888 100644 --- a/pkg/operator/cluster_test.go +++ b/pkg/operator/cluster/cluster_test.go @@ -16,7 +16,7 @@ limitations under the License. Some of the code below came from https://github.com/coreos/etcd-operator which also has the apache 2.0 license. */ -package operator +package cluster import ( "fmt" @@ -39,8 +39,8 @@ func TestCreateSecrets(t *testing.T) { response := "{\"key\":\"mysecurekey\"}" return []byte(response), "", nil } - spec := ClusterSpec{Namespace: "ns", Version: "myversion"} - c := newCluster(spec, factory, clientset) + spec := Spec{Namespace: "ns", Version: "myversion"} + c := New(spec, factory, clientset) c.dataDir = "/tmp/testdir" defer os.RemoveAll(c.dataDir) diff --git a/pkg/operator/cluster/spec.go b/pkg/operator/cluster/spec.go new file mode 100644 index 000000000..9d9a91cf2 --- /dev/null +++ b/pkg/operator/cluster/spec.go @@ -0,0 +1,41 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Some of the code below came from https://github.com/coreos/etcd-operator +which also has the apache 2.0 license. +*/ +package cluster + +type Spec struct { + // The namespace where the the rook resources will all be created. + Namespace string `json:"namespace"` + + // Version is the expected version of the rook container to run in the cluster. + // The rook-operator will eventually make the rook cluster version + // equal to the expected version. + Version string `json:"version"` + + // Paused is to pause the control of the operator for the rook cluster. + Paused bool `json:"paused,omitempty"` + + // Whether to consume all the storage devices found on a machine + UseAllDevices bool `json:"useAllDevices"` + + // A regular expression to allow more fine-grained selection of devices on nodes across the cluster + DeviceFilter string `json:"deviceFilter"` + + // The path on the host where config and data can be persisted. + DataDirHostPath string `json:"dataDirHostPath"` +} diff --git a/pkg/operator/event.go b/pkg/operator/event.go index 4db5a44ba..359ed7199 100644 --- a/pkg/operator/event.go +++ b/pkg/operator/event.go @@ -23,13 +23,15 @@ import ( "fmt" "io" + "github.com/rook/rook/pkg/operator/cluster" + "k8s.io/client-go/pkg/api/unversioned" kwatch "k8s.io/client-go/pkg/watch" ) type Event struct { Type kwatch.EventType - Object *Cluster + Object *cluster.Cluster } type rawEvent struct { @@ -58,7 +60,7 @@ func pollEvent(decoder *json.Decoder) (*Event, *unversioned.Status, error) { ev := &Event{ Type: re.Type, - Object: &Cluster{}, + Object: &cluster.Cluster{}, } err = json.Unmarshal(re.Object, ev.Object) if err != nil { diff --git a/pkg/operator/mon/mon.go b/pkg/operator/mon/mon.go index 847e6a31a..60e177df8 100644 --- a/pkg/operator/mon/mon.go +++ b/pkg/operator/mon/mon.go @@ -21,6 +21,7 @@ import ( "github.com/rook/rook/pkg/cephmgr/client" "github.com/rook/rook/pkg/cephmgr/mon" + "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/client-go/kubernetes" "k8s.io/client-go/pkg/api/errors" @@ -53,6 +54,7 @@ type Cluster struct { clientset kubernetes.Interface factory client.ConnectionFactory retryDelay int + clusterInfo *mon.ClusterInfo dataDirHostPath string } @@ -77,7 +79,7 @@ func New(clientset kubernetes.Interface, factory client.ConnectionFactory, names func (c *Cluster) Start() (*mon.ClusterInfo, error) { logger.Infof("start running mons") - clusterInfo, err := c.initClusterInfo() + err := c.initClusterInfo() if err != nil { return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err) } @@ -86,50 +88,110 @@ func (c *Cluster) Start() (*mon.ClusterInfo, error) { mons = append(mons, &MonConfig{Name: fmt.Sprintf("mon%d", i), Port: int32(mon.Port)}) } - err = c.startPods(clusterInfo, mons) + err = c.startPods(mons) if err != nil { return nil, fmt.Errorf("failed to start mon pods. %+v", err) } - return clusterInfo, nil + context := &clusterd.Context{ConfigDir: k8sutil.DataDir} + err = mon.WaitForQuorum(c.factory, context, c.clusterInfo) + if err != nil { + return nil, fmt.Errorf("failed to wait for mon quorum. %+v", err) + } + + return c.clusterInfo, nil +} + +func (c *Cluster) CheckHealth() error { + context := &clusterd.Context{ConfigDir: k8sutil.DataDir} + conn, err := mon.ConnectToClusterAsAdmin(context, c.factory, c.clusterInfo) + if err != nil { + return fmt.Errorf("cannot connect to cluster. %+v", err) + } + defer conn.Shutdown() + + status, err := client.GetMonStatus(conn) + if err != nil { + return fmt.Errorf("failed to get mon status. %+v", err) + } + + for _, monitor := range status.MonMap.Mons { + inQuorum := monInQuorum(monitor, status.Quorum) + if inQuorum { + logger.Infof("mon %s found in quorum", monitor.Name) + } else { + logger.Infof("mon %s NOT found in quroum. %+v", monitor.Name, status.Quorum) + } + + pod, err := c.clientset.CoreV1().Pods(c.Namespace).Get(monitor.Name) + if err != nil { + logger.Infof("Get mon %s pod failed. %+v", monitor.Name, err) + } else { + logger.Infof("mon %s pod status: %+v", monitor.Name, pod.Status.Phase) + } + + if !inQuorum { + err = c.failoverMon(monitor.Name) + if err != nil { + logger.Errorf("failed to failover mon %s. %+v", monitor.Name, err) + } + } + } + + return nil +} + +func monInQuorum(monitor client.MonMapEntry, quorum []int) bool { + for _, rank := range quorum { + if rank == monitor.Rank { + return true + } + } + return false +} + +func (c *Cluster) failoverMon(name string) error { + //logger.Infof("Failing over monitor %s", name) + return nil } // Retrieve the ceph cluster info if it already exists. // If a new cluster create new keys. -func (c *Cluster) initClusterInfo() (*mon.ClusterInfo, error) { +func (c *Cluster) initClusterInfo() error { secrets, err := c.clientset.CoreV1().Secrets(c.Namespace).Get(appName) if err != nil { if !errors.IsNotFound(err) { - return nil, fmt.Errorf("failed to get mon secrets. %+v", err) + return fmt.Errorf("failed to get mon secrets. %+v", err) } return c.createMonSecretsAndSave() } - info := &mon.ClusterInfo{ + c.clusterInfo = &mon.ClusterInfo{ Name: string(secrets.Data[clusterSecretName]), FSID: string(secrets.Data[fsidSecretName]), MonitorSecret: string(secrets.Data[monSecretName]), AdminSecret: string(secrets.Data[adminSecretName]), Monitors: map[string]*mon.CephMonitorConfig{}, } - logger.Infof("found existing monitor secrets for cluster %s with fsid %s", info.Name, info.FSID) - return info, nil + logger.Infof("found existing monitor secrets for cluster %s", c.clusterInfo.Name) + return nil } -func (c *Cluster) createMonSecretsAndSave() (*mon.ClusterInfo, error) { +func (c *Cluster) createMonSecretsAndSave() error { logger.Infof("creating mon secrets for a new cluster") - info, err := mon.CreateNamedClusterInfo(c.factory, "", c.Namespace) + var err error + c.clusterInfo, err = mon.CreateNamedClusterInfo(c.factory, "", c.Namespace) if err != nil { - return nil, fmt.Errorf("failed to create mon secrets. %+v", err) + return fmt.Errorf("failed to create mon secrets. %+v", err) } // store the secrets for internal usage of the rook pods secrets := map[string]string{ - clusterSecretName: info.Name, - fsidSecretName: info.FSID, - monSecretName: info.MonitorSecret, - adminSecretName: info.AdminSecret, + clusterSecretName: c.clusterInfo.Name, + fsidSecretName: c.clusterInfo.FSID, + monSecretName: c.clusterInfo.MonitorSecret, + adminSecretName: c.clusterInfo.AdminSecret, } secret := &v1.Secret{ ObjectMeta: v1.ObjectMeta{Name: appName, Namespace: c.Namespace}, @@ -138,12 +200,12 @@ func (c *Cluster) createMonSecretsAndSave() (*mon.ClusterInfo, error) { } _, err = c.clientset.CoreV1().Secrets(c.Namespace).Create(secret) if err != nil { - return nil, fmt.Errorf("failed to save mon secrets. %+v", err) + return fmt.Errorf("failed to save mon secrets. %+v", err) } // store the secret for usage by the storage class storageClassSecret := map[string]string{ - "key": info.AdminSecret, + "key": c.clusterInfo.AdminSecret, } secret = &v1.Secret{ ObjectMeta: v1.ObjectMeta{Name: "rook-admin", Namespace: c.Namespace}, @@ -153,17 +215,17 @@ func (c *Cluster) createMonSecretsAndSave() (*mon.ClusterInfo, error) { _, err = c.clientset.CoreV1().Secrets(c.Namespace).Create(secret) if err != nil { if !errors.IsAlreadyExists(err) { - return nil, fmt.Errorf("failed to save rook-admin secret. %+v", err) + return fmt.Errorf("failed to save rook-admin secret. %+v", err) } logger.Infof("rook-admin secret already exists") } else { logger.Infof("saved rook-admin secret") } - return info, nil + return nil } -func (c *Cluster) startPods(clusterInfo *mon.ClusterInfo, mons []*MonConfig) error { +func (c *Cluster) startPods(mons []*MonConfig) error { // schedule the mons on different nodes if we have enough nodes to be unique antiAffinity, err := c.getAntiAffinity() if err != nil { @@ -176,14 +238,14 @@ func (c *Cluster) startPods(clusterInfo *mon.ClusterInfo, mons []*MonConfig) err } logger.Infof("%d running, %d pending pods", len(running), len(pending)) - clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{} + c.clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{} for _, m := range running { - clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, m.Status.PodIP) + c.clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, m.Status.PodIP) } - err = c.saveMonEndpoints(clusterInfo) + err = c.saveMonEndpoints() if err != nil { - return fmt.Errorf("failed to save endpoints for %d running mons. %+v", len(clusterInfo.Monitors), err) + return fmt.Errorf("failed to save endpoints for %d running mons. %+v", len(c.clusterInfo.Monitors), err) } if len(running) == c.Size { @@ -194,7 +256,7 @@ func (c *Cluster) startPods(clusterInfo *mon.ClusterInfo, mons []*MonConfig) err started := 0 alreadyRunning := 0 for _, m := range mons { - monPod := c.makeMonPod(m, clusterInfo, antiAffinity) + monPod := c.makeMonPod(m, antiAffinity) name := monPod.Name logger.Debugf("Starting pod: %+v", monPod) _, err = c.clientset.CoreV1().Pods(c.Namespace).Create(monPod) @@ -212,9 +274,9 @@ func (c *Cluster) startPods(clusterInfo *mon.ClusterInfo, mons []*MonConfig) err if err != nil { return fmt.Errorf("failed to start pod %s. %+v", name, err) } - clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, podIP) + c.clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, podIP) - err = c.saveMonEndpoints(clusterInfo) + err = c.saveMonEndpoints() if err != nil { return fmt.Errorf("failed to save endpoints after starting mon %s. %+v", m.Name, err) } @@ -248,7 +310,7 @@ func (c *Cluster) waitForPodToStart(name string) (string, error) { return "", fmt.Errorf("timed out waiting for pod %s to start", name) } -func (c *Cluster) saveMonEndpoints(clusterInfo *mon.ClusterInfo) error { +func (c *Cluster) saveMonEndpoints() error { configMap := &v1.ConfigMap{ ObjectMeta: v1.ObjectMeta{ Name: monConfigMapName, @@ -257,7 +319,7 @@ func (c *Cluster) saveMonEndpoints(clusterInfo *mon.ClusterInfo) error { }, } configMap.Data = map[string]string{ - monEndpointKey: mon.FlattenMonEndpoints(clusterInfo.Monitors), + monEndpointKey: mon.FlattenMonEndpoints(c.clusterInfo.Monitors), } _, err := c.clientset.CoreV1().ConfigMaps(c.Namespace).Create(configMap) diff --git a/pkg/operator/mon/pod.go b/pkg/operator/mon/pod.go index de8ea488a..2305e23fb 100644 --- a/pkg/operator/mon/pod.go +++ b/pkg/operator/mon/pod.go @@ -18,7 +18,6 @@ package mon import ( "fmt" - "github.com/rook/rook/pkg/cephmgr/mon" "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/pkg/labels" @@ -51,7 +50,7 @@ func (c *Cluster) getLabels() map[string]string { } } -func (c *Cluster) makeMonPod(config *MonConfig, clusterInfo *mon.ClusterInfo, antiAffinity bool) *v1.Pod { +func (c *Cluster) makeMonPod(config *MonConfig, antiAffinity bool) *v1.Pod { container := c.monContainer(config, clusterInfo.FSID) dataDirSource := v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}} @@ -78,7 +77,7 @@ func (c *Cluster) makeMonPod(config *MonConfig, clusterInfo *mon.ClusterInfo, an k8sutil.SetPodVersion(pod, k8sutil.VersionAttr, c.Version) if antiAffinity { - k8sutil.PodWithAntiAffinity(pod, monClusterAttr, clusterInfo.Name) + k8sutil.PodWithAntiAffinity(pod, monClusterAttr, c.clusterInfo.Name) } return pod } diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 57f888b55..91e7fda33 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -32,6 +32,7 @@ import ( "k8s.io/client-go/rest" "github.com/rook/rook/pkg/cephmgr/client" + "github.com/rook/rook/pkg/operator/cluster" "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/client-go/pkg/api" "k8s.io/client-go/pkg/api/unversioned" @@ -48,10 +49,6 @@ var ( ErrVersionOutdated = errors.New("requested version is outdated in apiserver") ) -const ( - defaultPool = "rook" -) - type Operator struct { Namespace string MasterHost string @@ -61,7 +58,7 @@ type Operator struct { waitCluster sync.WaitGroup factory client.ConnectionFactory stopChMap map[string]chan struct{} - clusters map[string]*Cluster + clusters map[string]*cluster.Cluster kubeHttpCli *http.Client // Kubernetes resource version of the clusters clusterRVs map[string]string @@ -73,7 +70,7 @@ func New(host, namespace string, factory client.ConnectionFactory, clientset kub MasterHost: host, factory: factory, clientset: clientset, - clusters: make(map[string]*Cluster), + clusters: make(map[string]*cluster.Cluster), clusterRVs: make(map[string]string), stopChMap: map[string]chan struct{}{}, retryDelay: 3, @@ -147,7 +144,7 @@ func (o *Operator) watchTPR(watchVersion string) error { continue } - newCluster := newCluster(c.Spec, o.factory, o.clientset) + newCluster := cluster.New(c.Spec, o.factory, o.clientset) stopCh := make(chan struct{}) o.stopChMap[ns] = stopCh o.clusters[ns] = newCluster @@ -170,21 +167,23 @@ func (o *Operator) watchTPR(watchVersion string) error { } -func (o *Operator) startCluster(cluster *Cluster) { - if o.devicesInUse && cluster.Spec.UseAllDevices { - logger.Warningf("devices in more than one namespace not supported. ignoring devices in namespace %s", cluster.Spec.Namespace) - cluster.Spec.UseAllDevices = false +func (o *Operator) startCluster(c *cluster.Cluster) { + if o.devicesInUse && c.Spec.UseAllDevices { + logger.Warningf("devices in more than one namespace not supported. ignoring devices in namespace %s", c.Spec.Namespace) + c.Spec.UseAllDevices = false } - if cluster.Spec.UseAllDevices { + if c.Spec.UseAllDevices { o.devicesInUse = true } go func() { - err := cluster.CreateInstance() + err := c.CreateInstance() if err != nil { - logger.Errorf("failed to create cluster in namespace %s. %+v", cluster.Spec.Namespace, err) + logger.Errorf("failed to create cluster in namespace %s. %+v", c.Spec.Namespace, err) + return } + c.Monitor(o.stopChMap[c.Spec.Namespace]) }() } @@ -200,7 +199,7 @@ func (o *Operator) findAllClusters() (string, error) { stopCh := make(chan struct{}) ns := c.Spec.Namespace - existingCluster := newCluster(c.Spec, o.factory, o.clientset) + existingCluster := cluster.New(c.Spec, o.factory, o.clientset) o.stopChMap[ns] = stopCh o.clusters[ns] = existingCluster o.clusterRVs[ns] = c.Metadata.ResourceVersion @@ -285,7 +284,7 @@ func (o *Operator) watch(watchVersion string) (<-chan *Event, <-chan error) { return eventCh, errCh } -func (o *Operator) isClustersCacheStale(currentClusters []Cluster) bool { +func (o *Operator) isClustersCacheStale(currentClusters []cluster.Cluster) bool { if len(o.clusterRVs) != len(currentClusters) { return true } @@ -305,13 +304,13 @@ func watchClusters(host, ns string, httpClient *http.Client, resourceVersion str host, tprGroup, tprVersion, ns, resourceVersion)) } -func getClusterList(restcli rest.Interface, ns string) (*ClusterList, error) { +func getClusterList(restcli rest.Interface, ns string) (*cluster.ClusterList, error) { b, err := restcli.Get().RequestURI(listClustersURI(ns)).DoRaw() if err != nil { return nil, err } - clusters := &ClusterList{} + clusters := &cluster.ClusterList{} if err := json.Unmarshal(b, clusters); err != nil { return nil, err }