refactor tprs to support both clusters and pools

This commit is contained in:
Travis Nielsen committed 2017-03-27 09:47:27 -07:00
1 parent 2f6e4ea2a7
commit 8967dcc4cb
7 files changed
+537 -279

No files matched your search

+250
View File
@@ -0,0 +1,250 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Some of the code below came from https://github.com/coreos/etcd-operator
which also has the apache 2.0 license.
*/
package operator
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"time"
kwatch "k8s.io/client-go/pkg/watch"
"github.com/rook/rook/pkg/operator/cluster"
"github.com/rook/rook/pkg/operator/k8sutil"
)
type clusterTPR struct {
context *context
watchVersion string
devicesInUse bool
clusters map[string]*cluster.Cluster
tracker *tprTracker
}
func newClusterTPR(context *context) *clusterTPR {
return &clusterTPR{
context: context,
clusters: make(map[string]*cluster.Cluster),
tracker: newTPRTracker(),
}
}
func (t *clusterTPR) Name() string {
return "cluster"
}
func (t *clusterTPR) Description() string {
return "Managed Rook clusters"
}
func (t *clusterTPR) Load() error {
// Check if there is an existing cluster to recover
err := t.findAllClusters()
if err != nil {
return fmt.Errorf("failed to find clusters. %+v", err)
}
return nil
}
func (t *clusterTPR) findAllClusters() error {
logger.Info("finding existing clusters...")
clusterList, err := t.getClusterList()
if err != nil {
return err
}
logger.Infof("found %d clusters", len(clusterList.Items))
for i := range clusterList.Items {
c := clusterList.Items[i]
ns := c.Spec.Namespace
existingCluster := cluster.New(c.Spec, t.context.factory, t.context.clientset)
t.tracker.add(ns, c.Metadata.ResourceVersion)
t.clusters[ns] = existingCluster
logger.Infof("resuming cluster %s in namespace %s", c.Metadata.Name, ns)
t.startCluster(existingCluster)
}
t.watchVersion = clusterList.Metadata.ResourceVersion
return nil
}
func (t *clusterTPR) startCluster(c *cluster.Cluster) {
if t.devicesInUse && c.Spec.UseAllDevices {
logger.Warningf("devices in more than one namespace not supported. ignoring devices in namespace %s", c.Spec.Namespace)
c.Spec.UseAllDevices = false
}
if c.Spec.UseAllDevices {
t.devicesInUse = true
}
go func() {
err := c.CreateInstance()
if err != nil {
logger.Errorf("failed to create cluster in namespace %s. %+v", c.Spec.Namespace, err)
return
}
c.Monitor(t.tracker.stopChMap[c.Spec.Namespace])
}()
}
func (t *clusterTPR) isClustersCacheStale(currentClusters []cluster.Cluster) bool {
if len(t.tracker.clusterRVs) != len(currentClusters) {
return true
}
for _, cc := range currentClusters {
rv, ok := t.tracker.clusterRVs[cc.Metadata.Name]
if !ok || rv != cc.Metadata.ResourceVersion {
return true
}
}
return false
}
func (t *clusterTPR) getClusterList() (*cluster.ClusterList, error) {
b, err := getRawList(t.context, t)
if err != nil {
return nil, err
}
clusters := &cluster.ClusterList{}
if err := json.Unmarshal(b, clusters); err != nil {
return nil, err
}
return clusters, nil
}
func (t *clusterTPR) Watch() error {
logger.Infof("start watching %s tpr: %s", t.Name(), t.watchVersion)
defer t.tracker.stop()
eventCh, errCh := t.watch()
go func() {
timer := k8sutil.NewPanicTimer(
time.Minute,
fmt.Sprintf("unexpected long blocking (> 1 Minute) when handling %s event", t.Name()))
for event := range eventCh {
timer.Start()
c := event.Object
switch event.Type {
case kwatch.Added:
ns := c.Spec.Namespace
if ns == "" {
logger.Errorf("missing namespace attribute in rook spec")
continue
}
newCluster := cluster.New(c.Spec, t.context.factory, t.context.clientset)
t.tracker.add(ns, c.Metadata.ResourceVersion)
t.clusters[ns] = newCluster
logger.Infof("starting new cluster %s in namespace %s", c.Metadata.Name, ns)
t.startCluster(newCluster)
case kwatch.Modified:
logger.Infof("modifying a cluster not implemented")
case kwatch.Deleted:
logger.Infof("deleting a cluster not implemented")
}
timer.Stop()
}
}()
return <-errCh
}
// watch creates a go routine, and watches the cluster.rook kind resources from
// the given watch version. It emits events on the resources through the returned
// event chan. Errors will be reported through the returned error chan. The go routine
// exits on any error.
func (t *clusterTPR) watch() (<-chan *clusterEvent, <-chan error) {
eventCh := make(chan *clusterEvent)
// On unexpected error case, the operator should exit
errCh := make(chan error, 1)
go func() {
defer close(eventCh)
for {
cancel, err := t.watchOuterTPR(eventCh)
if err != nil {
logger.Errorf("failed to watch cluster tpr. %+v", err)
}
if cancel {
logger.Warningf("cancelling cluster tpr watch")
return
}
}
}()
return eventCh, errCh
}
func (t *clusterTPR) watchOuterTPR(eventCh chan *clusterEvent) (bool, error) {
resp, err := watchTPR(t.context, t.Name(), t.watchVersion)
if err != nil {
return false, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return false, errors.New("invalid status code: " + resp.Status)
}
decoder := json.NewDecoder(resp.Body)
for {
ev, st, err := pollClusterEvent(decoder)
done, cancel, err := handlePollEventResult(st, err, t.checkStaleCache)
if err != nil {
return false, err
}
if done {
return false, nil
}
if cancel {
return true, nil
}
logger.Debugf("rook cluster event: %+v", ev)
t.watchVersion = ev.Object.Metadata.ResourceVersion
eventCh <- ev
}
}
func (t *clusterTPR) checkStaleCache() (bool, error) {
clusterList, err := t.getClusterList()
if err == nil && !t.isClustersCacheStale(clusterList.Items) {
t.watchVersion = clusterList.Metadata.ResourceVersion
return false, nil
}
return true, err
}
+17
View File
@@ -39,3 +39,20 @@ type Spec struct {
// The path on the host where config and data can be persisted.
DataDirHostPath string `json:"dataDirHostPath"`
}
type PoolSpec struct {
// The namespace where the pool will be created (required). A Rook cluster must be running in this namespace.
Namespace string `json:"namespace"`
// Type of storage pool, 'replicated' or 'erasure-coded' (required) (default "replicated")
Type string `json:"type"`
// Number of copies per object in a replicated storage pool, including the object itself (required for replicated pool type)
ReplicaCount int `json:"replicaCount"`
// Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type)
ECCodingChunks int `json:"ecCodingChunks"`
// Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type)
ECDataChunks int `json:"ecDataChunks"`
}
+42 -11
View File
@@ -29,17 +29,56 @@ import (
kwatch "k8s.io/client-go/pkg/watch"
)
type Event struct {
type clusterEvent struct {
Type kwatch.EventType
Object *cluster.Cluster
}
type poolEvent struct {
Type kwatch.EventType
Object *cluster.PoolSpec
}
type rawEvent struct {
Type kwatch.EventType
Object json.RawMessage
}
func pollEvent(decoder *json.Decoder) (*Event, *unversioned.Status, error) {
func pollClusterEvent(decoder *json.Decoder) (*clusterEvent, *unversioned.Status, error) {
re, status, err := pollEvent(decoder)
if err != nil {
return nil, nil, fmt.Errorf("failed to poll cluster event. %+v", err)
}
ev := &clusterEvent{
Type: re.Type,
Object: &cluster.Cluster{},
}
err = json.Unmarshal(re.Object, ev.Object)
if err != nil {
return nil, nil, fmt.Errorf("fail to unmarshal Cluster object from data (%s): %v", re.Object, err)
}
return ev, status, nil
}
func pollPoolEvent(decoder *json.Decoder) (*poolEvent, *unversioned.Status, error) {
re, status, err := pollEvent(decoder)
if err != nil {
return nil, status, fmt.Errorf("failed to poll cluster event. %+v", err)
}
ev := &poolEvent{
Type: re.Type,
Object: &cluster.PoolSpec{},
}
err = json.Unmarshal(re.Object, ev.Object)
if err != nil {
return nil, nil, fmt.Errorf("fail to unmarshal Cluster object from data (%s): %v", re.Object, err)
}
return ev, nil, nil
}
func pollEvent(decoder *json.Decoder) (*rawEvent, *unversioned.Status, error) {
re := &rawEvent{}
err := decoder.Decode(re)
if err != nil {
@@ -58,13 +97,5 @@ func pollEvent(decoder *json.Decoder) (*Event, *unversioned.Status, error) {
return nil, status, nil
}
ev := &Event{
Type: re.Type,
Object: &cluster.Cluster{},
}
err = json.Unmarshal(re.Object, ev.Object)
if err != nil {
return nil, nil, fmt.Errorf("fail to unmarshal Cluster object from data (%s): %v", re.Object, err)
}
return ev, nil, nil
return re, nil, nil
}
+37 -244
View File
@@ -19,12 +19,9 @@ which also has the apache 2.0 license.
package operator
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"sync"
"time"
"k8s.io/client-go/kubernetes"
@@ -32,13 +29,10 @@ import (
"k8s.io/client-go/rest"
"github.com/rook/rook/pkg/cephmgr/client"
"github.com/rook/rook/pkg/operator/cluster"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/client-go/pkg/api"
"k8s.io/client-go/pkg/api/unversioned"
"k8s.io/client-go/pkg/runtime"
"k8s.io/client-go/pkg/runtime/serializer"
kwatch "k8s.io/client-go/pkg/watch"
)
const (
@@ -49,39 +43,39 @@ var (
ErrVersionOutdated = errors.New("requested version is outdated in apiserver")
)
type context struct {
clientset kubernetes.Interface
namespace string
retryDelay int
maxRetries int
masterHost string
kubeHttpCli *http.Client
factory client.ConnectionFactory
}
type Operator struct {
Namespace string
MasterHost string
devicesInUse bool
retryDelay int
clientset kubernetes.Interface
waitCluster sync.WaitGroup
factory client.ConnectionFactory
stopChMap map[string]chan struct{}
clusters map[string]*cluster.Cluster
kubeHttpCli *http.Client
// Kubernetes resource version of the clusters
clusterRVs map[string]string
context *context
tprs []TPR
}
func New(host, namespace string, factory client.ConnectionFactory, clientset kubernetes.Interface) *Operator {
return &Operator{
Namespace: namespace,
MasterHost: host,
context := &context{
namespace: namespace,
masterHost: host,
factory: factory,
clientset: clientset,
clusters: make(map[string]*cluster.Cluster),
clusterRVs: make(map[string]string),
stopChMap: map[string]chan struct{}{},
retryDelay: 3,
maxRetries: 30,
}
return &Operator{
context: context,
tprs: []TPR{newClusterTPR(context), newPoolTPR(context)},
}
}
func (o *Operator) Run() error {
var watchVersion string
var err error
for {
watchVersion, err = o.initResources()
err := o.initResources()
if err == nil {
break
}
@@ -90,235 +84,34 @@ func (o *Operator) Run() error {
}
// watch for changes to the rook clusters
return o.watchTPR(watchVersion)
for _, tpr := range o.tprs {
if err := tpr.Watch(); err != nil {
return fmt.Errorf("failed to watch tpr %s. %+v", tpr.Name(), err)
}
}
return nil
}
func (o *Operator) initResources() (string, error) {
func (o *Operator) initResources() error {
httpCli, err := newHttpClient()
if err != nil {
return "", fmt.Errorf("failed to get tpr client. %+v", err)
return fmt.Errorf("failed to get tpr client. %+v", err)
}
o.kubeHttpCli = httpCli.Client
o.context.kubeHttpCli = httpCli.Client
err = o.createTPR()
err = createTPRs(o.context, o.tprs)
if err != nil {
return "", fmt.Errorf("failed to create TPR. %+v", err)
}
err = o.waitForTPRInit(o.clientset.CoreV1().RESTClient(), 30, o.Namespace)
if err != nil {
return "", fmt.Errorf("failed to wait for TPR. %+v", err)
return fmt.Errorf("failed to create TPR. %+v", err)
}
// Check if there is an existing cluster to recover
watchVersion, err := o.findAllClusters()
if err != nil {
return "", fmt.Errorf("failed to find clusters. %+v", err)
}
return watchVersion, nil
}
func (o *Operator) watchTPR(watchVersion string) error {
logger.Infof("start watching rook tpr: %s", watchVersion)
defer func() {
for _, stop := range o.stopChMap {
close(stop)
}
o.waitCluster.Wait()
}()
eventCh, errCh := o.watch(watchVersion)
go func() {
pt := k8sutil.NewPanicTimer(time.Minute, "unexpected long blocking (> 1 Minute) when handling cluster event")
for event := range eventCh {
pt.Start()
c := event.Object
switch event.Type {
case kwatch.Added:
ns := c.Spec.Namespace
if ns == "" {
logger.Errorf("missing namespace attribute in rook spec")
continue
}
newCluster := cluster.New(c.Spec, o.factory, o.clientset)
stopCh := make(chan struct{})
o.stopChMap[ns] = stopCh
o.clusters[ns] = newCluster
o.clusterRVs[ns] = c.Metadata.ResourceVersion
logger.Infof("starting new cluster %s in namespace %s", c.Metadata.Name, ns)
o.startCluster(newCluster)
case kwatch.Modified:
logger.Infof("modifying a cluster not implemented")
case kwatch.Deleted:
logger.Infof("deleting a cluster not implemented")
}
pt.Stop()
}
}()
return <-errCh
}
func (o *Operator) startCluster(c *cluster.Cluster) {
if o.devicesInUse && c.Spec.UseAllDevices {
logger.Warningf("devices in more than one namespace not supported. ignoring devices in namespace %s", c.Spec.Namespace)
c.Spec.UseAllDevices = false
}
if c.Spec.UseAllDevices {
o.devicesInUse = true
}
go func() {
err := c.CreateInstance()
if err != nil {
logger.Errorf("failed to create cluster in namespace %s. %+v", c.Spec.Namespace, err)
return
}
c.Monitor(o.stopChMap[c.Spec.Namespace])
}()
}
func (o *Operator) findAllClusters() (string, error) {
logger.Info("finding existing clusters...")
clusterList, err := getClusterList(o.clientset.CoreV1().RESTClient(), o.Namespace)
if err != nil {
return "", err
}
logger.Infof("found %d clusters", len(clusterList.Items))
for i := range clusterList.Items {
c := clusterList.Items[i]
stopCh := make(chan struct{})
ns := c.Spec.Namespace
existingCluster := cluster.New(c.Spec, o.factory, o.clientset)
o.stopChMap[ns] = stopCh
o.clusters[ns] = existingCluster
o.clusterRVs[ns] = c.Metadata.ResourceVersion
logger.Infof("resuming cluster %s in namespace %s", c.Metadata.Name, ns)
o.startCluster(existingCluster)
}
return clusterList.Metadata.ResourceVersion, nil
}
// watch creates a go routine, and watches the cluster.rook kind resources from
// the given watch version. It emits events on the resources through the returned
// event chan. Errors will be reported through the returned error chan. The go routine
// exits on any error.
func (o *Operator) watch(watchVersion string) (<-chan *Event, <-chan error) {
eventCh := make(chan *Event)
// On unexpected error case, the operator should exit
errCh := make(chan error, 1)
go func() {
defer close(eventCh)
for {
resp, err := watchClusters(o.MasterHost, o.Namespace, o.kubeHttpCli, watchVersion)
if err != nil {
errCh <- err
return
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
errCh <- errors.New("invalid status code: " + resp.Status)
return
}
decoder := json.NewDecoder(resp.Body)
for {
ev, st, err := pollEvent(decoder)
if err != nil {
if err == io.EOF { // apiserver will close stream periodically
logger.Debug("apiserver closed stream")
break
}
logger.Errorf("received invalid event from API server: %v", err)
errCh <- err
return
}
if st != nil {
resp.Body.Close()
if st.Code == http.StatusGone {
// event history is outdated.
// if nothing has changed, we can go back to watch again.
clusterList, err := getClusterList(o.clientset.CoreV1().RESTClient(), o.Namespace)
if err == nil && !o.isClustersCacheStale(clusterList.Items) {
watchVersion = clusterList.Metadata.ResourceVersion
break
}
// if anything has changed (or error on relist), we have to rebuild the state.
// go to recovery path
errCh <- ErrVersionOutdated
return
}
logger.Errorf("unexpected status response from API server: %v", st.Message)
break
}
logger.Debugf("rook cluster event: %+v", ev)
watchVersion = ev.Object.Metadata.ResourceVersion
eventCh <- ev
}
resp.Body.Close()
}
}()
return eventCh, errCh
}
func (o *Operator) isClustersCacheStale(currentClusters []cluster.Cluster) bool {
if len(o.clusterRVs) != len(currentClusters) {
return true
}
for _, cc := range currentClusters {
rv, ok := o.clusterRVs[cc.Metadata.Name]
if !ok || rv != cc.Metadata.ResourceVersion {
return true
for _, tpr := range o.tprs {
if err := tpr.Load(); err != nil {
return fmt.Errorf("failed to load tpr %s. %+v", tpr.Name(), err)
}
}
return false
}
func watchClusters(host, ns string, httpClient *http.Client, resourceVersion string) (*http.Response, error) {
return httpClient.Get(fmt.Sprintf("%s/apis/%s/%s/namespaces/%s/clusters?watch=true&resourceVersion=%s",
host, tprGroup, tprVersion, ns, resourceVersion))
}
func getClusterList(restcli rest.Interface, ns string) (*cluster.ClusterList, error) {
b, err := restcli.Get().RequestURI(listClustersURI(ns)).DoRaw()
if err != nil {
return nil, err
}
clusters := &cluster.ClusterList{}
if err := json.Unmarshal(b, clusters); err != nil {
return nil, err
}
return clusters, nil
}
func listClustersURI(ns string) string {
return fmt.Sprintf("/apis/%s/%s/namespaces/%s/clusters", tprGroup, tprVersion, ns)
return nil
}
func newHttpClient() (*rest.RESTClient, error) {
+42
View File
@@ -0,0 +1,42 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Some of the code below came from https://github.com/coreos/etcd-operator
which also has the apache 2.0 license.
*/
package operator
type poolTPR struct {
context *context
}
func newPoolTPR(context *context) *poolTPR {
return &poolTPR{context: context}
}
func (t *poolTPR) Name() string {
return "pool"
}
func (t *poolTPR) Description() string {
return "Managed Rook pools"
}
func (t *poolTPR) Load() error {
return nil
}
func (t *poolTPR) Watch() error {
return nil
}
+103 -24
View File
@@ -20,53 +20,78 @@ package operator
import (
"fmt"
"io"
"net/http"
"time"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/client-go/pkg/api/errors"
"k8s.io/client-go/pkg/api/unversioned"
"k8s.io/client-go/pkg/api/v1"
"k8s.io/client-go/pkg/apis/extensions/v1beta1"
"k8s.io/client-go/rest"
)
const (
tprKind = "cluster"
tprGroup = "rook.io"
tprVersion = "v1beta1"
tprDescription = "Managed rook clusters"
tprGroup = "rook.io"
tprVersion = "v1beta1"
)
func tprName() string {
return fmt.Sprintf("%s.%s", tprKind, tprGroup)
type TPR interface {
Name() string
Description() string
Load() error
Watch() error
}
func (o *Operator) createTPR() error {
logger.Info("creating rook TPR")
tpr := &v1beta1.ThirdPartyResource{
ObjectMeta: v1.ObjectMeta{
Name: tprName(),
},
Versions: []v1beta1.APIVersion{
{Name: tprVersion},
},
Description: tprDescription,
func qualifiedName(tpr TPR) string {
return fmt.Sprintf("%s.%s", tpr.Name(), tprGroup)
}
func createTPRs(context *context, tprs []TPR) error {
for _, tpr := range tprs {
if err := createTPR(context, tpr); err != nil {
return fmt.Errorf("failed to init tpr %s. %+v", tpr.Name(), err)
}
}
_, err := o.clientset.ExtensionsV1beta1().ThirdPartyResources().Create(tpr)
if err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create rook third party resources. %+v", err)
for _, tpr := range tprs {
if err := waitForTPRInit(context, tpr); err != nil {
return fmt.Errorf("failed to complete init %s. %+v", tpr.Name(), err)
}
}
return nil
}
func (o *Operator) waitForTPRInit(restcli rest.Interface, maxRetries int, ns string) error {
uri := fmt.Sprintf("/apis/%s/%s/namespaces/%s/clusters", tprGroup, tprVersion, ns)
return k8sutil.Retry(time.Duration(o.retryDelay)*time.Second, maxRetries, func() (bool, error) {
func createTPR(context *context, tpr TPR) error {
logger.Infof("creating %s TPR", tpr.Name())
r := &v1beta1.ThirdPartyResource{
ObjectMeta: v1.ObjectMeta{
Name: qualifiedName(tpr),
},
Versions: []v1beta1.APIVersion{
{Name: tprVersion},
},
Description: tpr.Description(),
}
_, err := context.clientset.ExtensionsV1beta1().ThirdPartyResources().Create(r)
if err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create %s TPR. %+v", tpr.Name(), err)
}
}
return nil
}
func waitForTPRInit(context *context, tpr TPR) error {
restcli := context.clientset.CoreV1().RESTClient()
uri := tprURI(context, tpr.Name())
return k8sutil.Retry(time.Duration(context.retryDelay)*time.Second, context.maxRetries, func() (bool, error) {
_, err := restcli.Get().RequestURI(uri).DoRaw()
if err != nil {
logger.Infof("failed to query for tpr %s. %+v", tpr.Name(), err)
if errors.IsNotFound(err) {
return false, nil
}
@@ -75,3 +100,57 @@ func (o *Operator) waitForTPRInit(restcli rest.Interface, maxRetries int, ns str
return true, nil
})
}
func watchTPR(context *context, name string, resourceVersion string) (*http.Response, error) {
return context.kubeHttpCli.Get(fmt.Sprintf("%s/%s?watch=true&resourceVersion=%s",
context.masterHost, tprURI(context, name), resourceVersion))
}
func tprURI(context *context, name string) string {
return fmt.Sprintf("/apis/%s/%s/namespaces/%s/%ss", tprGroup, tprVersion, context.namespace, name)
}
func getRawList(context *context, tpr TPR) ([]byte, error) {
restcli := context.clientset.CoreV1().RESTClient()
return restcli.Get().RequestURI(tprURI(context, tpr.Name())).DoRaw()
}
func handlePollEventResult(status *unversioned.Status, errIn error, checkStaleCache func() (bool, error)) (done, cancel bool, err error) {
if errIn != nil {
if errIn == io.EOF { // apiserver will close stream periodically
logger.Debug("apiserver closed stream")
done = true
return
}
logger.Errorf("received invalid event from API server: %v", err)
err = errIn
cancel = true
return
}
if status != nil {
if status.Code == http.StatusGone {
// event history is outdated.
// if nothing has changed, we can go back to watch again.
var stale bool
stale, err = checkStaleCache()
if err == nil && !stale {
done = true
return
}
// if anything has changed (or error on relist), we have to rebuild the state.
// go to recovery path
err = ErrVersionOutdated
cancel = true
return
}
logger.Errorf("unexpected status response from API server: %v", status.Message)
done = true
return
}
return
}
+46
View File
@@ -0,0 +1,46 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Some of the code below came from https://github.com/coreos/etcd-operator
which also has the apache 2.0 license.
*/
package operator
import "sync"
type tprTracker struct {
stopChMap map[string]chan struct{}
clusterRVs map[string]string
waitCluster sync.WaitGroup
}
func newTPRTracker() *tprTracker {
return &tprTracker{
clusterRVs: make(map[string]string),
stopChMap: map[string]chan struct{}{},
}
}
func (t *tprTracker) add(name, version string) {
t.stopChMap[name] = make(chan struct{})
t.clusterRVs[name] = version
}
func (t *tprTracker) stop() {
for _, stop := range t.stopChMap {
close(stop)
}
t.waitCluster.Wait()
}