From 8967dcc4cbc2cc76a670f52dfe73a183b78d1186 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Fri, 24 Mar 2017 09:13:14 -0700 Subject: [PATCH] refactor tprs to support both clusters and pools --- pkg/operator/cluster.go | 250 +++++++++++++++++++++++++++++++ pkg/operator/cluster/spec.go | 17 +++ pkg/operator/event.go | 53 +++++-- pkg/operator/operator.go | 281 +++++------------------------------ pkg/operator/pool.go | 42 ++++++ pkg/operator/tpr.go | 127 +++++++++++++--- pkg/operator/tracker.go | 46 ++++++ 7 files changed, 537 insertions(+), 279 deletions(-) create mode 100644 pkg/operator/cluster.go create mode 100644 pkg/operator/pool.go create mode 100644 pkg/operator/tracker.go diff --git a/pkg/operator/cluster.go b/pkg/operator/cluster.go new file mode 100644 index 000000000..47d62e7f6 --- /dev/null +++ b/pkg/operator/cluster.go @@ -0,0 +1,250 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Some of the code below came from https://github.com/coreos/etcd-operator +which also has the apache 2.0 license. +*/ +package operator + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "time" + + kwatch "k8s.io/client-go/pkg/watch" + + "github.com/rook/rook/pkg/operator/cluster" + "github.com/rook/rook/pkg/operator/k8sutil" +) + +type clusterTPR struct { + context *context + watchVersion string + devicesInUse bool + clusters map[string]*cluster.Cluster + tracker *tprTracker +} + +func newClusterTPR(context *context) *clusterTPR { + return &clusterTPR{ + context: context, + clusters: make(map[string]*cluster.Cluster), + tracker: newTPRTracker(), + } +} + +func (t *clusterTPR) Name() string { + return "cluster" +} + +func (t *clusterTPR) Description() string { + return "Managed Rook clusters" +} + +func (t *clusterTPR) Load() error { + + // Check if there is an existing cluster to recover + err := t.findAllClusters() + if err != nil { + return fmt.Errorf("failed to find clusters. %+v", err) + } + return nil +} + +func (t *clusterTPR) findAllClusters() error { + logger.Info("finding existing clusters...") + clusterList, err := t.getClusterList() + if err != nil { + return err + } + logger.Infof("found %d clusters", len(clusterList.Items)) + for i := range clusterList.Items { + c := clusterList.Items[i] + + ns := c.Spec.Namespace + existingCluster := cluster.New(c.Spec, t.context.factory, t.context.clientset) + t.tracker.add(ns, c.Metadata.ResourceVersion) + t.clusters[ns] = existingCluster + + logger.Infof("resuming cluster %s in namespace %s", c.Metadata.Name, ns) + t.startCluster(existingCluster) + } + + t.watchVersion = clusterList.Metadata.ResourceVersion + return nil +} + +func (t *clusterTPR) startCluster(c *cluster.Cluster) { + if t.devicesInUse && c.Spec.UseAllDevices { + logger.Warningf("devices in more than one namespace not supported. ignoring devices in namespace %s", c.Spec.Namespace) + c.Spec.UseAllDevices = false + } + + if c.Spec.UseAllDevices { + t.devicesInUse = true + } + + go func() { + err := c.CreateInstance() + if err != nil { + logger.Errorf("failed to create cluster in namespace %s. %+v", c.Spec.Namespace, err) + return + } + c.Monitor(t.tracker.stopChMap[c.Spec.Namespace]) + }() +} + +func (t *clusterTPR) isClustersCacheStale(currentClusters []cluster.Cluster) bool { + if len(t.tracker.clusterRVs) != len(currentClusters) { + return true + } + + for _, cc := range currentClusters { + rv, ok := t.tracker.clusterRVs[cc.Metadata.Name] + if !ok || rv != cc.Metadata.ResourceVersion { + return true + } + } + + return false +} + +func (t *clusterTPR) getClusterList() (*cluster.ClusterList, error) { + b, err := getRawList(t.context, t) + if err != nil { + return nil, err + } + + clusters := &cluster.ClusterList{} + if err := json.Unmarshal(b, clusters); err != nil { + return nil, err + } + return clusters, nil +} + +func (t *clusterTPR) Watch() error { + logger.Infof("start watching %s tpr: %s", t.Name(), t.watchVersion) + defer t.tracker.stop() + + eventCh, errCh := t.watch() + + go func() { + timer := k8sutil.NewPanicTimer( + time.Minute, + fmt.Sprintf("unexpected long blocking (> 1 Minute) when handling %s event", t.Name())) + + for event := range eventCh { + timer.Start() + + c := event.Object + + switch event.Type { + case kwatch.Added: + ns := c.Spec.Namespace + if ns == "" { + logger.Errorf("missing namespace attribute in rook spec") + continue + } + + newCluster := cluster.New(c.Spec, t.context.factory, t.context.clientset) + t.tracker.add(ns, c.Metadata.ResourceVersion) + t.clusters[ns] = newCluster + + logger.Infof("starting new cluster %s in namespace %s", c.Metadata.Name, ns) + t.startCluster(newCluster) + + case kwatch.Modified: + logger.Infof("modifying a cluster not implemented") + + case kwatch.Deleted: + logger.Infof("deleting a cluster not implemented") + } + + timer.Stop() + } + }() + return <-errCh + +} + +// watch creates a go routine, and watches the cluster.rook kind resources from +// the given watch version. It emits events on the resources through the returned +// event chan. Errors will be reported through the returned error chan. The go routine +// exits on any error. +func (t *clusterTPR) watch() (<-chan *clusterEvent, <-chan error) { + eventCh := make(chan *clusterEvent) + // On unexpected error case, the operator should exit + errCh := make(chan error, 1) + + go func() { + defer close(eventCh) + + for { + cancel, err := t.watchOuterTPR(eventCh) + if err != nil { + logger.Errorf("failed to watch cluster tpr. %+v", err) + } + if cancel { + logger.Warningf("cancelling cluster tpr watch") + return + } + } + }() + + return eventCh, errCh +} + +func (t *clusterTPR) watchOuterTPR(eventCh chan *clusterEvent) (bool, error) { + resp, err := watchTPR(t.context, t.Name(), t.watchVersion) + if err != nil { + return false, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return false, errors.New("invalid status code: " + resp.Status) + } + + decoder := json.NewDecoder(resp.Body) + for { + ev, st, err := pollClusterEvent(decoder) + done, cancel, err := handlePollEventResult(st, err, t.checkStaleCache) + if err != nil { + return false, err + } + if done { + return false, nil + } + if cancel { + return true, nil + } + logger.Debugf("rook cluster event: %+v", ev) + + t.watchVersion = ev.Object.Metadata.ResourceVersion + eventCh <- ev + } +} + +func (t *clusterTPR) checkStaleCache() (bool, error) { + clusterList, err := t.getClusterList() + if err == nil && !t.isClustersCacheStale(clusterList.Items) { + t.watchVersion = clusterList.Metadata.ResourceVersion + return false, nil + } + + return true, err +} diff --git a/pkg/operator/cluster/spec.go b/pkg/operator/cluster/spec.go index 9d9a91cf2..089c16153 100644 --- a/pkg/operator/cluster/spec.go +++ b/pkg/operator/cluster/spec.go @@ -39,3 +39,20 @@ type Spec struct { // The path on the host where config and data can be persisted. DataDirHostPath string `json:"dataDirHostPath"` } + +type PoolSpec struct { + // The namespace where the pool will be created (required). A Rook cluster must be running in this namespace. + Namespace string `json:"namespace"` + + // Type of storage pool, 'replicated' or 'erasure-coded' (required) (default "replicated") + Type string `json:"type"` + + // Number of copies per object in a replicated storage pool, including the object itself (required for replicated pool type) + ReplicaCount int `json:"replicaCount"` + + // Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type) + ECCodingChunks int `json:"ecCodingChunks"` + + // Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type) + ECDataChunks int `json:"ecDataChunks"` +} diff --git a/pkg/operator/event.go b/pkg/operator/event.go index 359ed7199..1d8a9c0ea 100644 --- a/pkg/operator/event.go +++ b/pkg/operator/event.go @@ -29,17 +29,56 @@ import ( kwatch "k8s.io/client-go/pkg/watch" ) -type Event struct { +type clusterEvent struct { Type kwatch.EventType Object *cluster.Cluster } +type poolEvent struct { + Type kwatch.EventType + Object *cluster.PoolSpec +} + type rawEvent struct { Type kwatch.EventType Object json.RawMessage } -func pollEvent(decoder *json.Decoder) (*Event, *unversioned.Status, error) { +func pollClusterEvent(decoder *json.Decoder) (*clusterEvent, *unversioned.Status, error) { + re, status, err := pollEvent(decoder) + if err != nil { + return nil, nil, fmt.Errorf("failed to poll cluster event. %+v", err) + } + + ev := &clusterEvent{ + Type: re.Type, + Object: &cluster.Cluster{}, + } + err = json.Unmarshal(re.Object, ev.Object) + if err != nil { + return nil, nil, fmt.Errorf("fail to unmarshal Cluster object from data (%s): %v", re.Object, err) + } + return ev, status, nil +} + +func pollPoolEvent(decoder *json.Decoder) (*poolEvent, *unversioned.Status, error) { + re, status, err := pollEvent(decoder) + if err != nil { + return nil, status, fmt.Errorf("failed to poll cluster event. %+v", err) + } + + ev := &poolEvent{ + Type: re.Type, + Object: &cluster.PoolSpec{}, + } + err = json.Unmarshal(re.Object, ev.Object) + if err != nil { + return nil, nil, fmt.Errorf("fail to unmarshal Cluster object from data (%s): %v", re.Object, err) + } + return ev, nil, nil +} + +func pollEvent(decoder *json.Decoder) (*rawEvent, *unversioned.Status, error) { re := &rawEvent{} err := decoder.Decode(re) if err != nil { @@ -58,13 +97,5 @@ func pollEvent(decoder *json.Decoder) (*Event, *unversioned.Status, error) { return nil, status, nil } - ev := &Event{ - Type: re.Type, - Object: &cluster.Cluster{}, - } - err = json.Unmarshal(re.Object, ev.Object) - if err != nil { - return nil, nil, fmt.Errorf("fail to unmarshal Cluster object from data (%s): %v", re.Object, err) - } - return ev, nil, nil + return re, nil, nil } diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 91e7fda33..c51a037ba 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -19,12 +19,9 @@ which also has the apache 2.0 license. package operator import ( - "encoding/json" "errors" "fmt" - "io" "net/http" - "sync" "time" "k8s.io/client-go/kubernetes" @@ -32,13 +29,10 @@ import ( "k8s.io/client-go/rest" "github.com/rook/rook/pkg/cephmgr/client" - "github.com/rook/rook/pkg/operator/cluster" - "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/client-go/pkg/api" "k8s.io/client-go/pkg/api/unversioned" "k8s.io/client-go/pkg/runtime" "k8s.io/client-go/pkg/runtime/serializer" - kwatch "k8s.io/client-go/pkg/watch" ) const ( @@ -49,39 +43,39 @@ var ( ErrVersionOutdated = errors.New("requested version is outdated in apiserver") ) +type context struct { + clientset kubernetes.Interface + namespace string + retryDelay int + maxRetries int + masterHost string + kubeHttpCli *http.Client + factory client.ConnectionFactory +} + type Operator struct { - Namespace string - MasterHost string - devicesInUse bool - retryDelay int - clientset kubernetes.Interface - waitCluster sync.WaitGroup - factory client.ConnectionFactory - stopChMap map[string]chan struct{} - clusters map[string]*cluster.Cluster - kubeHttpCli *http.Client - // Kubernetes resource version of the clusters - clusterRVs map[string]string + context *context + tprs []TPR } func New(host, namespace string, factory client.ConnectionFactory, clientset kubernetes.Interface) *Operator { - return &Operator{ - Namespace: namespace, - MasterHost: host, + context := &context{ + namespace: namespace, + masterHost: host, factory: factory, clientset: clientset, - clusters: make(map[string]*cluster.Cluster), - clusterRVs: make(map[string]string), - stopChMap: map[string]chan struct{}{}, retryDelay: 3, + maxRetries: 30, + } + return &Operator{ + context: context, + tprs: []TPR{newClusterTPR(context), newPoolTPR(context)}, } } func (o *Operator) Run() error { - var watchVersion string - var err error for { - watchVersion, err = o.initResources() + err := o.initResources() if err == nil { break } @@ -90,235 +84,34 @@ func (o *Operator) Run() error { } // watch for changes to the rook clusters - return o.watchTPR(watchVersion) + for _, tpr := range o.tprs { + if err := tpr.Watch(); err != nil { + return fmt.Errorf("failed to watch tpr %s. %+v", tpr.Name(), err) + } + } + + return nil } -func (o *Operator) initResources() (string, error) { +func (o *Operator) initResources() error { httpCli, err := newHttpClient() if err != nil { - return "", fmt.Errorf("failed to get tpr client. %+v", err) + return fmt.Errorf("failed to get tpr client. %+v", err) } - o.kubeHttpCli = httpCli.Client + o.context.kubeHttpCli = httpCli.Client - err = o.createTPR() + err = createTPRs(o.context, o.tprs) if err != nil { - return "", fmt.Errorf("failed to create TPR. %+v", err) - } - err = o.waitForTPRInit(o.clientset.CoreV1().RESTClient(), 30, o.Namespace) - if err != nil { - return "", fmt.Errorf("failed to wait for TPR. %+v", err) + return fmt.Errorf("failed to create TPR. %+v", err) } - // Check if there is an existing cluster to recover - watchVersion, err := o.findAllClusters() - if err != nil { - return "", fmt.Errorf("failed to find clusters. %+v", err) - } - return watchVersion, nil -} - -func (o *Operator) watchTPR(watchVersion string) error { - logger.Infof("start watching rook tpr: %s", watchVersion) - defer func() { - for _, stop := range o.stopChMap { - close(stop) - } - o.waitCluster.Wait() - }() - - eventCh, errCh := o.watch(watchVersion) - - go func() { - pt := k8sutil.NewPanicTimer(time.Minute, "unexpected long blocking (> 1 Minute) when handling cluster event") - - for event := range eventCh { - pt.Start() - - c := event.Object - - switch event.Type { - case kwatch.Added: - ns := c.Spec.Namespace - if ns == "" { - logger.Errorf("missing namespace attribute in rook spec") - continue - } - - newCluster := cluster.New(c.Spec, o.factory, o.clientset) - stopCh := make(chan struct{}) - o.stopChMap[ns] = stopCh - o.clusters[ns] = newCluster - o.clusterRVs[ns] = c.Metadata.ResourceVersion - - logger.Infof("starting new cluster %s in namespace %s", c.Metadata.Name, ns) - o.startCluster(newCluster) - - case kwatch.Modified: - logger.Infof("modifying a cluster not implemented") - - case kwatch.Deleted: - logger.Infof("deleting a cluster not implemented") - } - - pt.Stop() - } - }() - return <-errCh - -} - -func (o *Operator) startCluster(c *cluster.Cluster) { - if o.devicesInUse && c.Spec.UseAllDevices { - logger.Warningf("devices in more than one namespace not supported. ignoring devices in namespace %s", c.Spec.Namespace) - c.Spec.UseAllDevices = false - } - - if c.Spec.UseAllDevices { - o.devicesInUse = true - } - - go func() { - err := c.CreateInstance() - if err != nil { - logger.Errorf("failed to create cluster in namespace %s. %+v", c.Spec.Namespace, err) - return - } - c.Monitor(o.stopChMap[c.Spec.Namespace]) - }() -} - -func (o *Operator) findAllClusters() (string, error) { - logger.Info("finding existing clusters...") - clusterList, err := getClusterList(o.clientset.CoreV1().RESTClient(), o.Namespace) - if err != nil { - return "", err - } - logger.Infof("found %d clusters", len(clusterList.Items)) - for i := range clusterList.Items { - c := clusterList.Items[i] - - stopCh := make(chan struct{}) - ns := c.Spec.Namespace - existingCluster := cluster.New(c.Spec, o.factory, o.clientset) - o.stopChMap[ns] = stopCh - o.clusters[ns] = existingCluster - o.clusterRVs[ns] = c.Metadata.ResourceVersion - - logger.Infof("resuming cluster %s in namespace %s", c.Metadata.Name, ns) - o.startCluster(existingCluster) - } - - return clusterList.Metadata.ResourceVersion, nil -} - -// watch creates a go routine, and watches the cluster.rook kind resources from -// the given watch version. It emits events on the resources through the returned -// event chan. Errors will be reported through the returned error chan. The go routine -// exits on any error. -func (o *Operator) watch(watchVersion string) (<-chan *Event, <-chan error) { - eventCh := make(chan *Event) - // On unexpected error case, the operator should exit - errCh := make(chan error, 1) - - go func() { - defer close(eventCh) - - for { - resp, err := watchClusters(o.MasterHost, o.Namespace, o.kubeHttpCli, watchVersion) - if err != nil { - errCh <- err - return - } - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - errCh <- errors.New("invalid status code: " + resp.Status) - return - } - - decoder := json.NewDecoder(resp.Body) - for { - ev, st, err := pollEvent(decoder) - if err != nil { - if err == io.EOF { // apiserver will close stream periodically - logger.Debug("apiserver closed stream") - break - } - - logger.Errorf("received invalid event from API server: %v", err) - errCh <- err - return - } - - if st != nil { - resp.Body.Close() - - if st.Code == http.StatusGone { - // event history is outdated. - // if nothing has changed, we can go back to watch again. - clusterList, err := getClusterList(o.clientset.CoreV1().RESTClient(), o.Namespace) - if err == nil && !o.isClustersCacheStale(clusterList.Items) { - watchVersion = clusterList.Metadata.ResourceVersion - break - } - - // if anything has changed (or error on relist), we have to rebuild the state. - // go to recovery path - errCh <- ErrVersionOutdated - return - } - - logger.Errorf("unexpected status response from API server: %v", st.Message) - break - } - - logger.Debugf("rook cluster event: %+v", ev) - - watchVersion = ev.Object.Metadata.ResourceVersion - eventCh <- ev - } - - resp.Body.Close() - } - }() - - return eventCh, errCh -} - -func (o *Operator) isClustersCacheStale(currentClusters []cluster.Cluster) bool { - if len(o.clusterRVs) != len(currentClusters) { - return true - } - - for _, cc := range currentClusters { - rv, ok := o.clusterRVs[cc.Metadata.Name] - if !ok || rv != cc.Metadata.ResourceVersion { - return true + for _, tpr := range o.tprs { + if err := tpr.Load(); err != nil { + return fmt.Errorf("failed to load tpr %s. %+v", tpr.Name(), err) } } - return false -} - -func watchClusters(host, ns string, httpClient *http.Client, resourceVersion string) (*http.Response, error) { - return httpClient.Get(fmt.Sprintf("%s/apis/%s/%s/namespaces/%s/clusters?watch=true&resourceVersion=%s", - host, tprGroup, tprVersion, ns, resourceVersion)) -} - -func getClusterList(restcli rest.Interface, ns string) (*cluster.ClusterList, error) { - b, err := restcli.Get().RequestURI(listClustersURI(ns)).DoRaw() - if err != nil { - return nil, err - } - - clusters := &cluster.ClusterList{} - if err := json.Unmarshal(b, clusters); err != nil { - return nil, err - } - return clusters, nil -} - -func listClustersURI(ns string) string { - return fmt.Sprintf("/apis/%s/%s/namespaces/%s/clusters", tprGroup, tprVersion, ns) + return nil } func newHttpClient() (*rest.RESTClient, error) { diff --git a/pkg/operator/pool.go b/pkg/operator/pool.go new file mode 100644 index 000000000..da662ca9f --- /dev/null +++ b/pkg/operator/pool.go @@ -0,0 +1,42 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Some of the code below came from https://github.com/coreos/etcd-operator +which also has the apache 2.0 license. +*/ +package operator + +type poolTPR struct { + context *context +} + +func newPoolTPR(context *context) *poolTPR { + return &poolTPR{context: context} +} + +func (t *poolTPR) Name() string { + return "pool" +} + +func (t *poolTPR) Description() string { + return "Managed Rook pools" +} + +func (t *poolTPR) Load() error { + return nil +} +func (t *poolTPR) Watch() error { + return nil +} diff --git a/pkg/operator/tpr.go b/pkg/operator/tpr.go index 5595c6d32..db5795733 100644 --- a/pkg/operator/tpr.go +++ b/pkg/operator/tpr.go @@ -20,53 +20,78 @@ package operator import ( "fmt" + "io" + "net/http" "time" "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/client-go/pkg/api/errors" + "k8s.io/client-go/pkg/api/unversioned" "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/pkg/apis/extensions/v1beta1" - "k8s.io/client-go/rest" ) const ( - tprKind = "cluster" - tprGroup = "rook.io" - tprVersion = "v1beta1" - tprDescription = "Managed rook clusters" + tprGroup = "rook.io" + tprVersion = "v1beta1" ) -func tprName() string { - return fmt.Sprintf("%s.%s", tprKind, tprGroup) +type TPR interface { + Name() string + Description() string + Load() error + Watch() error } -func (o *Operator) createTPR() error { - logger.Info("creating rook TPR") - tpr := &v1beta1.ThirdPartyResource{ - ObjectMeta: v1.ObjectMeta{ - Name: tprName(), - }, - Versions: []v1beta1.APIVersion{ - {Name: tprVersion}, - }, - Description: tprDescription, +func qualifiedName(tpr TPR) string { + return fmt.Sprintf("%s.%s", tpr.Name(), tprGroup) +} + +func createTPRs(context *context, tprs []TPR) error { + for _, tpr := range tprs { + if err := createTPR(context, tpr); err != nil { + return fmt.Errorf("failed to init tpr %s. %+v", tpr.Name(), err) + } } - _, err := o.clientset.ExtensionsV1beta1().ThirdPartyResources().Create(tpr) - if err != nil { - if !errors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create rook third party resources. %+v", err) + + for _, tpr := range tprs { + if err := waitForTPRInit(context, tpr); err != nil { + return fmt.Errorf("failed to complete init %s. %+v", tpr.Name(), err) } } return nil } -func (o *Operator) waitForTPRInit(restcli rest.Interface, maxRetries int, ns string) error { - uri := fmt.Sprintf("/apis/%s/%s/namespaces/%s/clusters", tprGroup, tprVersion, ns) - return k8sutil.Retry(time.Duration(o.retryDelay)*time.Second, maxRetries, func() (bool, error) { +func createTPR(context *context, tpr TPR) error { + logger.Infof("creating %s TPR", tpr.Name()) + r := &v1beta1.ThirdPartyResource{ + ObjectMeta: v1.ObjectMeta{ + Name: qualifiedName(tpr), + }, + Versions: []v1beta1.APIVersion{ + {Name: tprVersion}, + }, + Description: tpr.Description(), + } + _, err := context.clientset.ExtensionsV1beta1().ThirdPartyResources().Create(r) + if err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create %s TPR. %+v", tpr.Name(), err) + } + } + + return nil +} + +func waitForTPRInit(context *context, tpr TPR) error { + restcli := context.clientset.CoreV1().RESTClient() + uri := tprURI(context, tpr.Name()) + return k8sutil.Retry(time.Duration(context.retryDelay)*time.Second, context.maxRetries, func() (bool, error) { _, err := restcli.Get().RequestURI(uri).DoRaw() if err != nil { + logger.Infof("failed to query for tpr %s. %+v", tpr.Name(), err) if errors.IsNotFound(err) { return false, nil } @@ -75,3 +100,57 @@ func (o *Operator) waitForTPRInit(restcli rest.Interface, maxRetries int, ns str return true, nil }) } + +func watchTPR(context *context, name string, resourceVersion string) (*http.Response, error) { + return context.kubeHttpCli.Get(fmt.Sprintf("%s/%s?watch=true&resourceVersion=%s", + context.masterHost, tprURI(context, name), resourceVersion)) +} + +func tprURI(context *context, name string) string { + return fmt.Sprintf("/apis/%s/%s/namespaces/%s/%ss", tprGroup, tprVersion, context.namespace, name) +} + +func getRawList(context *context, tpr TPR) ([]byte, error) { + restcli := context.clientset.CoreV1().RESTClient() + return restcli.Get().RequestURI(tprURI(context, tpr.Name())).DoRaw() +} + +func handlePollEventResult(status *unversioned.Status, errIn error, checkStaleCache func() (bool, error)) (done, cancel bool, err error) { + if errIn != nil { + if errIn == io.EOF { // apiserver will close stream periodically + logger.Debug("apiserver closed stream") + done = true + return + } + + logger.Errorf("received invalid event from API server: %v", err) + err = errIn + cancel = true + return + } + + if status != nil { + + if status.Code == http.StatusGone { + // event history is outdated. + // if nothing has changed, we can go back to watch again. + var stale bool + stale, err = checkStaleCache() + if err == nil && !stale { + done = true + return + } + + // if anything has changed (or error on relist), we have to rebuild the state. + // go to recovery path + err = ErrVersionOutdated + cancel = true + return + } + + logger.Errorf("unexpected status response from API server: %v", status.Message) + done = true + return + } + return +} diff --git a/pkg/operator/tracker.go b/pkg/operator/tracker.go new file mode 100644 index 000000000..f571dc316 --- /dev/null +++ b/pkg/operator/tracker.go @@ -0,0 +1,46 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Some of the code below came from https://github.com/coreos/etcd-operator +which also has the apache 2.0 license. +*/ +package operator + +import "sync" + +type tprTracker struct { + stopChMap map[string]chan struct{} + clusterRVs map[string]string + waitCluster sync.WaitGroup +} + +func newTPRTracker() *tprTracker { + return &tprTracker{ + clusterRVs: make(map[string]string), + stopChMap: map[string]chan struct{}{}, + } +} + +func (t *tprTracker) add(name, version string) { + t.stopChMap[name] = make(chan struct{}) + t.clusterRVs[name] = version +} + +func (t *tprTracker) stop() { + for _, stop := range t.stopChMap { + close(stop) + } + t.waitCluster.Wait() +}