refactor tprs to support both clusters and pools
This commit is contained in:
1 parent
2f6e4ea2a7
commit
8967dcc4cb
7 files changed
+537
-279
No files matched your search
@@ -0,0 +1,250 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
Some of the code below came from https://github.com/coreos/etcd-operator
|
||||
which also has the apache 2.0 license.
|
||||
*/
|
||||
package operator
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
kwatch "k8s.io/client-go/pkg/watch"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/cluster"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
)
|
||||
|
||||
type clusterTPR struct {
|
||||
context *context
|
||||
watchVersion string
|
||||
devicesInUse bool
|
||||
clusters map[string]*cluster.Cluster
|
||||
tracker *tprTracker
|
||||
}
|
||||
|
||||
func newClusterTPR(context *context) *clusterTPR {
|
||||
return &clusterTPR{
|
||||
context: context,
|
||||
clusters: make(map[string]*cluster.Cluster),
|
||||
tracker: newTPRTracker(),
|
||||
}
|
||||
}
|
||||
|
||||
func (t *clusterTPR) Name() string {
|
||||
return "cluster"
|
||||
}
|
||||
|
||||
func (t *clusterTPR) Description() string {
|
||||
return "Managed Rook clusters"
|
||||
}
|
||||
|
||||
func (t *clusterTPR) Load() error {
|
||||
|
||||
// Check if there is an existing cluster to recover
|
||||
err := t.findAllClusters()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to find clusters. %+v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *clusterTPR) findAllClusters() error {
|
||||
logger.Info("finding existing clusters...")
|
||||
clusterList, err := t.getClusterList()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Infof("found %d clusters", len(clusterList.Items))
|
||||
for i := range clusterList.Items {
|
||||
c := clusterList.Items[i]
|
||||
|
||||
ns := c.Spec.Namespace
|
||||
existingCluster := cluster.New(c.Spec, t.context.factory, t.context.clientset)
|
||||
t.tracker.add(ns, c.Metadata.ResourceVersion)
|
||||
t.clusters[ns] = existingCluster
|
||||
|
||||
logger.Infof("resuming cluster %s in namespace %s", c.Metadata.Name, ns)
|
||||
t.startCluster(existingCluster)
|
||||
}
|
||||
|
||||
t.watchVersion = clusterList.Metadata.ResourceVersion
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *clusterTPR) startCluster(c *cluster.Cluster) {
|
||||
if t.devicesInUse && c.Spec.UseAllDevices {
|
||||
logger.Warningf("devices in more than one namespace not supported. ignoring devices in namespace %s", c.Spec.Namespace)
|
||||
c.Spec.UseAllDevices = false
|
||||
}
|
||||
|
||||
if c.Spec.UseAllDevices {
|
||||
t.devicesInUse = true
|
||||
}
|
||||
|
||||
go func() {
|
||||
err := c.CreateInstance()
|
||||
if err != nil {
|
||||
logger.Errorf("failed to create cluster in namespace %s. %+v", c.Spec.Namespace, err)
|
||||
return
|
||||
}
|
||||
c.Monitor(t.tracker.stopChMap[c.Spec.Namespace])
|
||||
}()
|
||||
}
|
||||
|
||||
func (t *clusterTPR) isClustersCacheStale(currentClusters []cluster.Cluster) bool {
|
||||
if len(t.tracker.clusterRVs) != len(currentClusters) {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, cc := range currentClusters {
|
||||
rv, ok := t.tracker.clusterRVs[cc.Metadata.Name]
|
||||
if !ok || rv != cc.Metadata.ResourceVersion {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *clusterTPR) getClusterList() (*cluster.ClusterList, error) {
|
||||
b, err := getRawList(t.context, t)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clusters := &cluster.ClusterList{}
|
||||
if err := json.Unmarshal(b, clusters); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return clusters, nil
|
||||
}
|
||||
|
||||
func (t *clusterTPR) Watch() error {
|
||||
logger.Infof("start watching %s tpr: %s", t.Name(), t.watchVersion)
|
||||
defer t.tracker.stop()
|
||||
|
||||
eventCh, errCh := t.watch()
|
||||
|
||||
go func() {
|
||||
timer := k8sutil.NewPanicTimer(
|
||||
time.Minute,
|
||||
fmt.Sprintf("unexpected long blocking (> 1 Minute) when handling %s event", t.Name()))
|
||||
|
||||
for event := range eventCh {
|
||||
timer.Start()
|
||||
|
||||
c := event.Object
|
||||
|
||||
switch event.Type {
|
||||
case kwatch.Added:
|
||||
ns := c.Spec.Namespace
|
||||
if ns == "" {
|
||||
logger.Errorf("missing namespace attribute in rook spec")
|
||||
continue
|
||||
}
|
||||
|
||||
newCluster := cluster.New(c.Spec, t.context.factory, t.context.clientset)
|
||||
t.tracker.add(ns, c.Metadata.ResourceVersion)
|
||||
t.clusters[ns] = newCluster
|
||||
|
||||
logger.Infof("starting new cluster %s in namespace %s", c.Metadata.Name, ns)
|
||||
t.startCluster(newCluster)
|
||||
|
||||
case kwatch.Modified:
|
||||
logger.Infof("modifying a cluster not implemented")
|
||||
|
||||
case kwatch.Deleted:
|
||||
logger.Infof("deleting a cluster not implemented")
|
||||
}
|
||||
|
||||
timer.Stop()
|
||||
}
|
||||
}()
|
||||
return <-errCh
|
||||
|
||||
}
|
||||
|
||||
// watch creates a go routine, and watches the cluster.rook kind resources from
|
||||
// the given watch version. It emits events on the resources through the returned
|
||||
// event chan. Errors will be reported through the returned error chan. The go routine
|
||||
// exits on any error.
|
||||
func (t *clusterTPR) watch() (<-chan *clusterEvent, <-chan error) {
|
||||
eventCh := make(chan *clusterEvent)
|
||||
// On unexpected error case, the operator should exit
|
||||
errCh := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
defer close(eventCh)
|
||||
|
||||
for {
|
||||
cancel, err := t.watchOuterTPR(eventCh)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to watch cluster tpr. %+v", err)
|
||||
}
|
||||
if cancel {
|
||||
logger.Warningf("cancelling cluster tpr watch")
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return eventCh, errCh
|
||||
}
|
||||
|
||||
func (t *clusterTPR) watchOuterTPR(eventCh chan *clusterEvent) (bool, error) {
|
||||
resp, err := watchTPR(t.context, t.Name(), t.watchVersion)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return false, errors.New("invalid status code: " + resp.Status)
|
||||
}
|
||||
|
||||
decoder := json.NewDecoder(resp.Body)
|
||||
for {
|
||||
ev, st, err := pollClusterEvent(decoder)
|
||||
done, cancel, err := handlePollEventResult(st, err, t.checkStaleCache)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if done {
|
||||
return false, nil
|
||||
}
|
||||
if cancel {
|
||||
return true, nil
|
||||
}
|
||||
logger.Debugf("rook cluster event: %+v", ev)
|
||||
|
||||
t.watchVersion = ev.Object.Metadata.ResourceVersion
|
||||
eventCh <- ev
|
||||
}
|
||||
}
|
||||
|
||||
func (t *clusterTPR) checkStaleCache() (bool, error) {
|
||||
clusterList, err := t.getClusterList()
|
||||
if err == nil && !t.isClustersCacheStale(clusterList.Items) {
|
||||
t.watchVersion = clusterList.Metadata.ResourceVersion
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return true, err
|
||||
}
|
||||
@@ -39,3 +39,20 @@ type Spec struct {
|
||||
// The path on the host where config and data can be persisted.
|
||||
DataDirHostPath string `json:"dataDirHostPath"`
|
||||
}
|
||||
|
||||
type PoolSpec struct {
|
||||
// The namespace where the pool will be created (required). A Rook cluster must be running in this namespace.
|
||||
Namespace string `json:"namespace"`
|
||||
|
||||
// Type of storage pool, 'replicated' or 'erasure-coded' (required) (default "replicated")
|
||||
Type string `json:"type"`
|
||||
|
||||
// Number of copies per object in a replicated storage pool, including the object itself (required for replicated pool type)
|
||||
ReplicaCount int `json:"replicaCount"`
|
||||
|
||||
// Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type)
|
||||
ECCodingChunks int `json:"ecCodingChunks"`
|
||||
|
||||
// Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type)
|
||||
ECDataChunks int `json:"ecDataChunks"`
|
||||
}
|
||||
+42
-11
@@ -29,17 +29,56 @@ import (
|
||||
kwatch "k8s.io/client-go/pkg/watch"
|
||||
)
|
||||
|
||||
type Event struct {
|
||||
type clusterEvent struct {
|
||||
Type kwatch.EventType
|
||||
Object *cluster.Cluster
|
||||
}
|
||||
|
||||
type poolEvent struct {
|
||||
Type kwatch.EventType
|
||||
Object *cluster.PoolSpec
|
||||
}
|
||||
|
||||
type rawEvent struct {
|
||||
Type kwatch.EventType
|
||||
Object json.RawMessage
|
||||
}
|
||||
|
||||
func pollEvent(decoder *json.Decoder) (*Event, *unversioned.Status, error) {
|
||||
func pollClusterEvent(decoder *json.Decoder) (*clusterEvent, *unversioned.Status, error) {
|
||||
re, status, err := pollEvent(decoder)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to poll cluster event. %+v", err)
|
||||
}
|
||||
|
||||
ev := &clusterEvent{
|
||||
Type: re.Type,
|
||||
Object: &cluster.Cluster{},
|
||||
}
|
||||
err = json.Unmarshal(re.Object, ev.Object)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("fail to unmarshal Cluster object from data (%s): %v", re.Object, err)
|
||||
}
|
||||
return ev, status, nil
|
||||
}
|
||||
|
||||
func pollPoolEvent(decoder *json.Decoder) (*poolEvent, *unversioned.Status, error) {
|
||||
re, status, err := pollEvent(decoder)
|
||||
if err != nil {
|
||||
return nil, status, fmt.Errorf("failed to poll cluster event. %+v", err)
|
||||
}
|
||||
|
||||
ev := &poolEvent{
|
||||
Type: re.Type,
|
||||
Object: &cluster.PoolSpec{},
|
||||
}
|
||||
err = json.Unmarshal(re.Object, ev.Object)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("fail to unmarshal Cluster object from data (%s): %v", re.Object, err)
|
||||
}
|
||||
return ev, nil, nil
|
||||
}
|
||||
|
||||
func pollEvent(decoder *json.Decoder) (*rawEvent, *unversioned.Status, error) {
|
||||
re := &rawEvent{}
|
||||
err := decoder.Decode(re)
|
||||
if err != nil {
|
||||
@@ -58,13 +97,5 @@ func pollEvent(decoder *json.Decoder) (*Event, *unversioned.Status, error) {
|
||||
return nil, status, nil
|
||||
}
|
||||
|
||||
ev := &Event{
|
||||
Type: re.Type,
|
||||
Object: &cluster.Cluster{},
|
||||
}
|
||||
err = json.Unmarshal(re.Object, ev.Object)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("fail to unmarshal Cluster object from data (%s): %v", re.Object, err)
|
||||
}
|
||||
return ev, nil, nil
|
||||
return re, nil, nil
|
||||
}
|
||||
+37
-244
@@ -19,12 +19,9 @@ which also has the apache 2.0 license.
|
||||
package operator
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"k8s.io/client-go/kubernetes"
|
||||
@@ -32,13 +29,10 @@ import (
|
||||
"k8s.io/client-go/rest"
|
||||
|
||||
"github.com/rook/rook/pkg/cephmgr/client"
|
||||
"github.com/rook/rook/pkg/operator/cluster"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/client-go/pkg/api"
|
||||
"k8s.io/client-go/pkg/api/unversioned"
|
||||
"k8s.io/client-go/pkg/runtime"
|
||||
"k8s.io/client-go/pkg/runtime/serializer"
|
||||
kwatch "k8s.io/client-go/pkg/watch"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -49,39 +43,39 @@ var (
|
||||
ErrVersionOutdated = errors.New("requested version is outdated in apiserver")
|
||||
)
|
||||
|
||||
type context struct {
|
||||
clientset kubernetes.Interface
|
||||
namespace string
|
||||
retryDelay int
|
||||
maxRetries int
|
||||
masterHost string
|
||||
kubeHttpCli *http.Client
|
||||
factory client.ConnectionFactory
|
||||
}
|
||||
|
||||
type Operator struct {
|
||||
Namespace string
|
||||
MasterHost string
|
||||
devicesInUse bool
|
||||
retryDelay int
|
||||
clientset kubernetes.Interface
|
||||
waitCluster sync.WaitGroup
|
||||
factory client.ConnectionFactory
|
||||
stopChMap map[string]chan struct{}
|
||||
clusters map[string]*cluster.Cluster
|
||||
kubeHttpCli *http.Client
|
||||
// Kubernetes resource version of the clusters
|
||||
clusterRVs map[string]string
|
||||
context *context
|
||||
tprs []TPR
|
||||
}
|
||||
|
||||
func New(host, namespace string, factory client.ConnectionFactory, clientset kubernetes.Interface) *Operator {
|
||||
return &Operator{
|
||||
Namespace: namespace,
|
||||
MasterHost: host,
|
||||
context := &context{
|
||||
namespace: namespace,
|
||||
masterHost: host,
|
||||
factory: factory,
|
||||
clientset: clientset,
|
||||
clusters: make(map[string]*cluster.Cluster),
|
||||
clusterRVs: make(map[string]string),
|
||||
stopChMap: map[string]chan struct{}{},
|
||||
retryDelay: 3,
|
||||
maxRetries: 30,
|
||||
}
|
||||
return &Operator{
|
||||
context: context,
|
||||
tprs: []TPR{newClusterTPR(context), newPoolTPR(context)},
|
||||
}
|
||||
}
|
||||
|
||||
func (o *Operator) Run() error {
|
||||
var watchVersion string
|
||||
var err error
|
||||
for {
|
||||
watchVersion, err = o.initResources()
|
||||
err := o.initResources()
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
@@ -90,235 +84,34 @@ func (o *Operator) Run() error {
|
||||
}
|
||||
|
||||
// watch for changes to the rook clusters
|
||||
return o.watchTPR(watchVersion)
|
||||
for _, tpr := range o.tprs {
|
||||
if err := tpr.Watch(); err != nil {
|
||||
return fmt.Errorf("failed to watch tpr %s. %+v", tpr.Name(), err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *Operator) initResources() (string, error) {
|
||||
func (o *Operator) initResources() error {
|
||||
httpCli, err := newHttpClient()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get tpr client. %+v", err)
|
||||
return fmt.Errorf("failed to get tpr client. %+v", err)
|
||||
}
|
||||
o.kubeHttpCli = httpCli.Client
|
||||
o.context.kubeHttpCli = httpCli.Client
|
||||
|
||||
err = o.createTPR()
|
||||
err = createTPRs(o.context, o.tprs)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create TPR. %+v", err)
|
||||
}
|
||||
err = o.waitForTPRInit(o.clientset.CoreV1().RESTClient(), 30, o.Namespace)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to wait for TPR. %+v", err)
|
||||
return fmt.Errorf("failed to create TPR. %+v", err)
|
||||
}
|
||||
|
||||
// Check if there is an existing cluster to recover
|
||||
watchVersion, err := o.findAllClusters()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to find clusters. %+v", err)
|
||||
}
|
||||
return watchVersion, nil
|
||||
}
|
||||
|
||||
func (o *Operator) watchTPR(watchVersion string) error {
|
||||
logger.Infof("start watching rook tpr: %s", watchVersion)
|
||||
defer func() {
|
||||
for _, stop := range o.stopChMap {
|
||||
close(stop)
|
||||
}
|
||||
o.waitCluster.Wait()
|
||||
}()
|
||||
|
||||
eventCh, errCh := o.watch(watchVersion)
|
||||
|
||||
go func() {
|
||||
pt := k8sutil.NewPanicTimer(time.Minute, "unexpected long blocking (> 1 Minute) when handling cluster event")
|
||||
|
||||
for event := range eventCh {
|
||||
pt.Start()
|
||||
|
||||
c := event.Object
|
||||
|
||||
switch event.Type {
|
||||
case kwatch.Added:
|
||||
ns := c.Spec.Namespace
|
||||
if ns == "" {
|
||||
logger.Errorf("missing namespace attribute in rook spec")
|
||||
continue
|
||||
}
|
||||
|
||||
newCluster := cluster.New(c.Spec, o.factory, o.clientset)
|
||||
stopCh := make(chan struct{})
|
||||
o.stopChMap[ns] = stopCh
|
||||
o.clusters[ns] = newCluster
|
||||
o.clusterRVs[ns] = c.Metadata.ResourceVersion
|
||||
|
||||
logger.Infof("starting new cluster %s in namespace %s", c.Metadata.Name, ns)
|
||||
o.startCluster(newCluster)
|
||||
|
||||
case kwatch.Modified:
|
||||
logger.Infof("modifying a cluster not implemented")
|
||||
|
||||
case kwatch.Deleted:
|
||||
logger.Infof("deleting a cluster not implemented")
|
||||
}
|
||||
|
||||
pt.Stop()
|
||||
}
|
||||
}()
|
||||
return <-errCh
|
||||
|
||||
}
|
||||
|
||||
func (o *Operator) startCluster(c *cluster.Cluster) {
|
||||
if o.devicesInUse && c.Spec.UseAllDevices {
|
||||
logger.Warningf("devices in more than one namespace not supported. ignoring devices in namespace %s", c.Spec.Namespace)
|
||||
c.Spec.UseAllDevices = false
|
||||
}
|
||||
|
||||
if c.Spec.UseAllDevices {
|
||||
o.devicesInUse = true
|
||||
}
|
||||
|
||||
go func() {
|
||||
err := c.CreateInstance()
|
||||
if err != nil {
|
||||
logger.Errorf("failed to create cluster in namespace %s. %+v", c.Spec.Namespace, err)
|
||||
return
|
||||
}
|
||||
c.Monitor(o.stopChMap[c.Spec.Namespace])
|
||||
}()
|
||||
}
|
||||
|
||||
func (o *Operator) findAllClusters() (string, error) {
|
||||
logger.Info("finding existing clusters...")
|
||||
clusterList, err := getClusterList(o.clientset.CoreV1().RESTClient(), o.Namespace)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
logger.Infof("found %d clusters", len(clusterList.Items))
|
||||
for i := range clusterList.Items {
|
||||
c := clusterList.Items[i]
|
||||
|
||||
stopCh := make(chan struct{})
|
||||
ns := c.Spec.Namespace
|
||||
existingCluster := cluster.New(c.Spec, o.factory, o.clientset)
|
||||
o.stopChMap[ns] = stopCh
|
||||
o.clusters[ns] = existingCluster
|
||||
o.clusterRVs[ns] = c.Metadata.ResourceVersion
|
||||
|
||||
logger.Infof("resuming cluster %s in namespace %s", c.Metadata.Name, ns)
|
||||
o.startCluster(existingCluster)
|
||||
}
|
||||
|
||||
return clusterList.Metadata.ResourceVersion, nil
|
||||
}
|
||||
|
||||
// watch creates a go routine, and watches the cluster.rook kind resources from
|
||||
// the given watch version. It emits events on the resources through the returned
|
||||
// event chan. Errors will be reported through the returned error chan. The go routine
|
||||
// exits on any error.
|
||||
func (o *Operator) watch(watchVersion string) (<-chan *Event, <-chan error) {
|
||||
eventCh := make(chan *Event)
|
||||
// On unexpected error case, the operator should exit
|
||||
errCh := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
defer close(eventCh)
|
||||
|
||||
for {
|
||||
resp, err := watchClusters(o.MasterHost, o.Namespace, o.kubeHttpCli, watchVersion)
|
||||
if err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
errCh <- errors.New("invalid status code: " + resp.Status)
|
||||
return
|
||||
}
|
||||
|
||||
decoder := json.NewDecoder(resp.Body)
|
||||
for {
|
||||
ev, st, err := pollEvent(decoder)
|
||||
if err != nil {
|
||||
if err == io.EOF { // apiserver will close stream periodically
|
||||
logger.Debug("apiserver closed stream")
|
||||
break
|
||||
}
|
||||
|
||||
logger.Errorf("received invalid event from API server: %v", err)
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
|
||||
if st != nil {
|
||||
resp.Body.Close()
|
||||
|
||||
if st.Code == http.StatusGone {
|
||||
// event history is outdated.
|
||||
// if nothing has changed, we can go back to watch again.
|
||||
clusterList, err := getClusterList(o.clientset.CoreV1().RESTClient(), o.Namespace)
|
||||
if err == nil && !o.isClustersCacheStale(clusterList.Items) {
|
||||
watchVersion = clusterList.Metadata.ResourceVersion
|
||||
break
|
||||
}
|
||||
|
||||
// if anything has changed (or error on relist), we have to rebuild the state.
|
||||
// go to recovery path
|
||||
errCh <- ErrVersionOutdated
|
||||
return
|
||||
}
|
||||
|
||||
logger.Errorf("unexpected status response from API server: %v", st.Message)
|
||||
break
|
||||
}
|
||||
|
||||
logger.Debugf("rook cluster event: %+v", ev)
|
||||
|
||||
watchVersion = ev.Object.Metadata.ResourceVersion
|
||||
eventCh <- ev
|
||||
}
|
||||
|
||||
resp.Body.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
return eventCh, errCh
|
||||
}
|
||||
|
||||
func (o *Operator) isClustersCacheStale(currentClusters []cluster.Cluster) bool {
|
||||
if len(o.clusterRVs) != len(currentClusters) {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, cc := range currentClusters {
|
||||
rv, ok := o.clusterRVs[cc.Metadata.Name]
|
||||
if !ok || rv != cc.Metadata.ResourceVersion {
|
||||
return true
|
||||
for _, tpr := range o.tprs {
|
||||
if err := tpr.Load(); err != nil {
|
||||
return fmt.Errorf("failed to load tpr %s. %+v", tpr.Name(), err)
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func watchClusters(host, ns string, httpClient *http.Client, resourceVersion string) (*http.Response, error) {
|
||||
return httpClient.Get(fmt.Sprintf("%s/apis/%s/%s/namespaces/%s/clusters?watch=true&resourceVersion=%s",
|
||||
host, tprGroup, tprVersion, ns, resourceVersion))
|
||||
}
|
||||
|
||||
func getClusterList(restcli rest.Interface, ns string) (*cluster.ClusterList, error) {
|
||||
b, err := restcli.Get().RequestURI(listClustersURI(ns)).DoRaw()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clusters := &cluster.ClusterList{}
|
||||
if err := json.Unmarshal(b, clusters); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return clusters, nil
|
||||
}
|
||||
|
||||
func listClustersURI(ns string) string {
|
||||
return fmt.Sprintf("/apis/%s/%s/namespaces/%s/clusters", tprGroup, tprVersion, ns)
|
||||
return nil
|
||||
}
|
||||
|
||||
func newHttpClient() (*rest.RESTClient, error) {
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
Some of the code below came from https://github.com/coreos/etcd-operator
|
||||
which also has the apache 2.0 license.
|
||||
*/
|
||||
package operator
|
||||
|
||||
type poolTPR struct {
|
||||
context *context
|
||||
}
|
||||
|
||||
func newPoolTPR(context *context) *poolTPR {
|
||||
return &poolTPR{context: context}
|
||||
}
|
||||
|
||||
func (t *poolTPR) Name() string {
|
||||
return "pool"
|
||||
}
|
||||
|
||||
func (t *poolTPR) Description() string {
|
||||
return "Managed Rook pools"
|
||||
}
|
||||
|
||||
func (t *poolTPR) Load() error {
|
||||
return nil
|
||||
}
|
||||
func (t *poolTPR) Watch() error {
|
||||
return nil
|
||||
}
|
||||
+103
-24
@@ -20,53 +20,78 @@ package operator
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
|
||||
"k8s.io/client-go/pkg/api/errors"
|
||||
"k8s.io/client-go/pkg/api/unversioned"
|
||||
"k8s.io/client-go/pkg/api/v1"
|
||||
"k8s.io/client-go/pkg/apis/extensions/v1beta1"
|
||||
"k8s.io/client-go/rest"
|
||||
)
|
||||
|
||||
const (
|
||||
tprKind = "cluster"
|
||||
tprGroup = "rook.io"
|
||||
tprVersion = "v1beta1"
|
||||
tprDescription = "Managed rook clusters"
|
||||
tprGroup = "rook.io"
|
||||
tprVersion = "v1beta1"
|
||||
)
|
||||
|
||||
func tprName() string {
|
||||
return fmt.Sprintf("%s.%s", tprKind, tprGroup)
|
||||
type TPR interface {
|
||||
Name() string
|
||||
Description() string
|
||||
Load() error
|
||||
Watch() error
|
||||
}
|
||||
|
||||
func (o *Operator) createTPR() error {
|
||||
logger.Info("creating rook TPR")
|
||||
tpr := &v1beta1.ThirdPartyResource{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: tprName(),
|
||||
},
|
||||
Versions: []v1beta1.APIVersion{
|
||||
{Name: tprVersion},
|
||||
},
|
||||
Description: tprDescription,
|
||||
func qualifiedName(tpr TPR) string {
|
||||
return fmt.Sprintf("%s.%s", tpr.Name(), tprGroup)
|
||||
}
|
||||
|
||||
func createTPRs(context *context, tprs []TPR) error {
|
||||
for _, tpr := range tprs {
|
||||
if err := createTPR(context, tpr); err != nil {
|
||||
return fmt.Errorf("failed to init tpr %s. %+v", tpr.Name(), err)
|
||||
}
|
||||
}
|
||||
_, err := o.clientset.ExtensionsV1beta1().ThirdPartyResources().Create(tpr)
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create rook third party resources. %+v", err)
|
||||
|
||||
for _, tpr := range tprs {
|
||||
if err := waitForTPRInit(context, tpr); err != nil {
|
||||
return fmt.Errorf("failed to complete init %s. %+v", tpr.Name(), err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *Operator) waitForTPRInit(restcli rest.Interface, maxRetries int, ns string) error {
|
||||
uri := fmt.Sprintf("/apis/%s/%s/namespaces/%s/clusters", tprGroup, tprVersion, ns)
|
||||
return k8sutil.Retry(time.Duration(o.retryDelay)*time.Second, maxRetries, func() (bool, error) {
|
||||
func createTPR(context *context, tpr TPR) error {
|
||||
logger.Infof("creating %s TPR", tpr.Name())
|
||||
r := &v1beta1.ThirdPartyResource{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: qualifiedName(tpr),
|
||||
},
|
||||
Versions: []v1beta1.APIVersion{
|
||||
{Name: tprVersion},
|
||||
},
|
||||
Description: tpr.Description(),
|
||||
}
|
||||
_, err := context.clientset.ExtensionsV1beta1().ThirdPartyResources().Create(r)
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create %s TPR. %+v", tpr.Name(), err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func waitForTPRInit(context *context, tpr TPR) error {
|
||||
restcli := context.clientset.CoreV1().RESTClient()
|
||||
uri := tprURI(context, tpr.Name())
|
||||
return k8sutil.Retry(time.Duration(context.retryDelay)*time.Second, context.maxRetries, func() (bool, error) {
|
||||
_, err := restcli.Get().RequestURI(uri).DoRaw()
|
||||
if err != nil {
|
||||
logger.Infof("failed to query for tpr %s. %+v", tpr.Name(), err)
|
||||
if errors.IsNotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
@@ -75,3 +100,57 @@ func (o *Operator) waitForTPRInit(restcli rest.Interface, maxRetries int, ns str
|
||||
return true, nil
|
||||
})
|
||||
}
|
||||
|
||||
func watchTPR(context *context, name string, resourceVersion string) (*http.Response, error) {
|
||||
return context.kubeHttpCli.Get(fmt.Sprintf("%s/%s?watch=true&resourceVersion=%s",
|
||||
context.masterHost, tprURI(context, name), resourceVersion))
|
||||
}
|
||||
|
||||
func tprURI(context *context, name string) string {
|
||||
return fmt.Sprintf("/apis/%s/%s/namespaces/%s/%ss", tprGroup, tprVersion, context.namespace, name)
|
||||
}
|
||||
|
||||
func getRawList(context *context, tpr TPR) ([]byte, error) {
|
||||
restcli := context.clientset.CoreV1().RESTClient()
|
||||
return restcli.Get().RequestURI(tprURI(context, tpr.Name())).DoRaw()
|
||||
}
|
||||
|
||||
func handlePollEventResult(status *unversioned.Status, errIn error, checkStaleCache func() (bool, error)) (done, cancel bool, err error) {
|
||||
if errIn != nil {
|
||||
if errIn == io.EOF { // apiserver will close stream periodically
|
||||
logger.Debug("apiserver closed stream")
|
||||
done = true
|
||||
return
|
||||
}
|
||||
|
||||
logger.Errorf("received invalid event from API server: %v", err)
|
||||
err = errIn
|
||||
cancel = true
|
||||
return
|
||||
}
|
||||
|
||||
if status != nil {
|
||||
|
||||
if status.Code == http.StatusGone {
|
||||
// event history is outdated.
|
||||
// if nothing has changed, we can go back to watch again.
|
||||
var stale bool
|
||||
stale, err = checkStaleCache()
|
||||
if err == nil && !stale {
|
||||
done = true
|
||||
return
|
||||
}
|
||||
|
||||
// if anything has changed (or error on relist), we have to rebuild the state.
|
||||
// go to recovery path
|
||||
err = ErrVersionOutdated
|
||||
cancel = true
|
||||
return
|
||||
}
|
||||
|
||||
logger.Errorf("unexpected status response from API server: %v", status.Message)
|
||||
done = true
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
Some of the code below came from https://github.com/coreos/etcd-operator
|
||||
which also has the apache 2.0 license.
|
||||
*/
|
||||
package operator
|
||||
|
||||
import "sync"
|
||||
|
||||
type tprTracker struct {
|
||||
stopChMap map[string]chan struct{}
|
||||
clusterRVs map[string]string
|
||||
waitCluster sync.WaitGroup
|
||||
}
|
||||
|
||||
func newTPRTracker() *tprTracker {
|
||||
return &tprTracker{
|
||||
clusterRVs: make(map[string]string),
|
||||
stopChMap: map[string]chan struct{}{},
|
||||
}
|
||||
}
|
||||
|
||||
func (t *tprTracker) add(name, version string) {
|
||||
t.stopChMap[name] = make(chan struct{})
|
||||
t.clusterRVs[name] = version
|
||||
}
|
||||
|
||||
func (t *tprTracker) stop() {
|
||||
for _, stop := range t.stopChMap {
|
||||
close(stop)
|
||||
}
|
||||
t.waitCluster.Wait()
|
||||
}
|
||||
Reference in new issue
Block a user