diff --git a/pkg/operator/cluster.go b/pkg/operator/cluster.go index e3eb83fa1..ca9f159c9 100644 --- a/pkg/operator/cluster.go +++ b/pkg/operator/cluster.go @@ -36,7 +36,6 @@ import ( type clusterManager struct { context *clusterd.Context - name string watchVersion string devicesInUse bool clusters map[string]*cluster.Cluster @@ -45,7 +44,7 @@ type clusterManager struct { // The initiators that create TPRs specific to specific Rook clusters. // For example, pools, object services, and file services, only make sense in the context of a Rook cluster inclusterInitiators []inclusterInitiator - inclusterMgrs []tprManager + inclusterMgrs []k8sutil.TPRManager } func newClusterManager(context *clusterd.Context, inclusterInitiators []inclusterInitiator) *clusterManager { @@ -57,16 +56,6 @@ func newClusterManager(context *clusterd.Context, inclusterInitiators []incluste } } -// Gets the name of the TPR -func (m *clusterManager) Name() string { - return "cluster" -} - -// Gets the description of the TPR -func (m *clusterManager) Description() string { - return "Managed Rook clusters" -} - func (m *clusterManager) Manage() { for { logger.Infof("Managing clusters") @@ -165,11 +154,11 @@ func (m *clusterManager) startCluster(c *cluster.Cluster) { } // Start all the TPRs for this cluster - for _, tpr := range m.inclusterInitiators { + for _, initiator := range m.inclusterInitiators { k8sutil.Retry(m.context, func() (bool, error) { - tprMgr, err := tpr.Create(m, c.Name, c.Namespace) + tprMgr, err := initiator.Create(m, c.Namespace) if err != nil { - logger.Warningf("cannot create in-cluster tpr %s. %+v. retrying...", m.Name(), err) + logger.Warningf("cannot create in-cluster tpr %s. %+v. retrying...", initiator.Scheme().Name, err) return false, nil } @@ -203,7 +192,7 @@ func (m *clusterManager) isClustersCacheStale(currentClusters []cluster.Cluster) } func (m *clusterManager) getClusterList() (*cluster.ClusterList, error) { - b, err := getRawList(m.context.Clientset, m.Name()) + b, err := k8sutil.GetRawList(m.context.Clientset, cluster.ClusterScheme) if err != nil { return nil, err } @@ -274,7 +263,7 @@ func (m *clusterManager) watch() (<-chan *clusterEvent, <-chan error) { } func (m *clusterManager) watchOuterTPR(eventCh chan *clusterEvent, errCh chan error) error { - resp, err := watchTPR(m.context, m.Name(), m.watchVersion) + resp, err := k8sutil.WatchTPR(m.context, cluster.ClusterScheme, m.watchVersion) if err != nil { errCh <- err return err @@ -323,3 +312,13 @@ func (m *clusterManager) getRookClient(namespace string) (rookclient.RookRestCli return nil, fmt.Errorf("namespace %s not found", namespace) } + +func (m *clusterManager) getCluster(namespace string) (*cluster.Cluster, error) { + m.Lock() + defer m.Unlock() + if c, ok := m.clusters[namespace]; ok { + return c, nil + } + + return nil, fmt.Errorf("cluster namespace %s not found", namespace) +} diff --git a/pkg/operator/cluster/cluster.go b/pkg/operator/cluster/cluster.go index 3406a1fdd..7b7bcc962 100644 --- a/pkg/operator/cluster/cluster.go +++ b/pkg/operator/cluster/cluster.go @@ -49,6 +49,11 @@ var ( healthCheckInterval = 10 * time.Second clientTimeout = 15 * time.Second logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-cluster") + ClusterScheme = k8sutil.TPRScheme{ + Name: "cluster", + Version: k8sutil.V1Alpha1, + Description: "Managed Rook clusters", + } ) type Cluster struct { diff --git a/pkg/operator/cluster/pool.go b/pkg/operator/cluster/pool.go index 59d6832ff..35aa6ecb5 100644 --- a/pkg/operator/cluster/pool.go +++ b/pkg/operator/cluster/pool.go @@ -21,6 +21,7 @@ package cluster import ( "fmt" + "github.com/rook/rook/pkg/operator/k8sutil" rookclient "github.com/rook/rook/pkg/rook/client" "k8s.io/client-go/pkg/api/v1" @@ -32,6 +33,12 @@ const ( erasureCodeType = "erasure-coded" ) +var PoolScheme = k8sutil.TPRScheme{ + Name: "pool", + Version: k8sutil.V1Alpha1, + Description: "Managed Rook pools", +} + type Pool struct { v1.ObjectMeta `json:"metadata,omitempty"` PoolSpec `json:"spec"` diff --git a/pkg/operator/k8sutil/k8sutil.go b/pkg/operator/k8sutil/k8sutil.go index 15e156c62..e4fb57f68 100644 --- a/pkg/operator/k8sutil/k8sutil.go +++ b/pkg/operator/k8sutil/k8sutil.go @@ -24,6 +24,11 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/rook/rook/pkg/clusterd" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/client-go/pkg/api" + "k8s.io/client-go/rest" ) var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-k8sutil") @@ -62,3 +67,23 @@ func Retry(context *clusterd.Context, f ConditionFunc) error { } return fmt.Errorf("failed after max retries %d.", context.MaxRetries) } + +func NewHttpClient() (*rest.RESTClient, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + + config.GroupVersion = &schema.GroupVersion{ + Group: tprGroup, + } + config.APIPath = "/apis" + config.ContentType = runtime.ContentTypeJSON + config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: api.Codecs} + + restcli, err := rest.RESTClientFor(config) + if err != nil { + return nil, err + } + return restcli, nil +} diff --git a/pkg/operator/k8sutil/tpr.go b/pkg/operator/k8sutil/tpr.go new file mode 100644 index 000000000..5afdd1efe --- /dev/null +++ b/pkg/operator/k8sutil/tpr.go @@ -0,0 +1,138 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Some of the code was modified from https://github.com/coreos/etcd-operator +which also has the apache 2.0 license. +*/ +package k8sutil + +import ( + "fmt" + "net/http" + + "github.com/rook/rook/pkg/clusterd" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/pkg/apis/extensions/v1beta1" +) + +const ( + tprGroup = "rook.io" + V1Alpha1 = "v1alpha1" + V1Beta1 = "v1beta1" + V1 = "v1" +) + +type TPRScheme struct { + Name string + Version string + Description string +} + +type TPRManager interface { + Load() error + Watch() error + Manage() +} + +func CreateTPRs(context *clusterd.Context, tprs []TPRScheme) error { + for _, tpr := range tprs { + if err := CreateTPR(context, tpr); err != nil { + return fmt.Errorf("failed to init tpr %s. %+v", tpr.Name, err) + } + } + + for _, tpr := range tprs { + if err := waitForTPRInit(context, tpr); err != nil { + return fmt.Errorf("failed to complete init %s. %+v", tpr.Name, err) + } + } + + return nil +} + +func CreateTPR(context *clusterd.Context, tpr TPRScheme) error { + logger.Infof("creating %s TPR", tpr.Name) + r := &v1beta1.ThirdPartyResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s.%s", tpr.Name, tprGroup), + }, + Versions: []v1beta1.APIVersion{ + {Name: tpr.Version}, + }, + Description: tpr.Description, + } + _, err := context.Clientset.ExtensionsV1beta1().ThirdPartyResources().Create(r) + if err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create %s TPR. %+v", tpr.Name, err) + } + } + + return nil +} + +func waitForTPRInit(context *clusterd.Context, scheme TPRScheme) error { + restcli := context.Clientset.CoreV1().RESTClient() + uri := TPRURI(scheme) + return Retry(context, func() (bool, error) { + _, err := restcli.Get().RequestURI(uri).DoRaw() + if err != nil { + logger.Infof("did not yet find tpr %s at %s. %+v", scheme.Name, uri, err) + if errors.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil + }) +} + +func WatchTPRNamespaced(context *clusterd.Context, scheme TPRScheme, namespace, resourceVersion string) (*http.Response, error) { + uri := fmt.Sprintf("%s/%s?watch=true&resourceVersion=%s", context.MasterHost, TPRURINamespaced(scheme, namespace), resourceVersion) + logger.Debugf("watching tpr: %s", uri) + return context.KubeHttpCli.Get(uri) +} + +func WatchTPR(context *clusterd.Context, scheme TPRScheme, resourceVersion string) (*http.Response, error) { + uri := fmt.Sprintf("%s/%s?watch=true&resourceVersion=%s", context.MasterHost, TPRURI(scheme), resourceVersion) + logger.Debugf("watching tpr: %s", uri) + return context.KubeHttpCli.Get(uri) +} + +func TPRURINamespaced(scheme TPRScheme, namespace string) string { + // /apis/rook.io/v1alpha1/namespaces/rook/pools + return fmt.Sprintf("apis/%s/%s/namespaces/%s/%ss", tprGroup, scheme.Version, namespace, scheme.Name) +} + +func TPRURI(scheme TPRScheme) string { + // creates a uri that is for retrieving or watching a tpr in all namespaces. For example: + // /apis/rook.io/v1alpha1/clusters + return fmt.Sprintf("apis/%s/%s/%ss", tprGroup, scheme.Version, scheme.Name) +} + +func GetRawListNamespaced(clientset kubernetes.Interface, scheme TPRScheme, namespace string) ([]byte, error) { + restcli := clientset.CoreV1().RESTClient() + uri := TPRURINamespaced(scheme, namespace) + logger.Debugf("getting tpr: %s", uri) + return restcli.Get().RequestURI(uri).DoRaw() +} + +func GetRawList(clientset kubernetes.Interface, scheme TPRScheme) ([]byte, error) { + restcli := clientset.CoreV1().RESTClient() + return restcli.Get().RequestURI(TPRURI(scheme)).DoRaw() +} diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 59c03b048..58ce3130d 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -25,12 +25,9 @@ import ( "github.com/kubernetes-incubator/external-storage/lib/controller" "github.com/rook/rook/pkg/clusterd" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" + "github.com/rook/rook/pkg/operator/cluster" + "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/pkg/api" - "k8s.io/client-go/rest" ) const ( @@ -55,7 +52,7 @@ var ( type Operator struct { context *clusterd.Context - tprSchemes []tprScheme + tprSchemes []k8sutil.TPRScheme // The TPR that is global to the kubernetes cluster. // The cluster TPR is global because you create multiple clusers in k8s clusterMgr *clusterManager @@ -68,7 +65,7 @@ func New(context *clusterd.Context) *Operator { clusterMgr := newClusterManager(context, []inclusterInitiator{poolInitiator}) volumeProvisioner := newRookVolumeProvisioner(clusterMgr) - schemes := []tprScheme{clusterMgr, poolInitiator} + schemes := []k8sutil.TPRScheme{cluster.ClusterScheme, cluster.PoolScheme} return &Operator{ context: context, clusterMgr: clusterMgr, @@ -115,37 +112,16 @@ func (o *Operator) Run() error { } func (o *Operator) initResources() error { - httpCli, err := newHttpClient() + httpCli, err := k8sutil.NewHttpClient() if err != nil { return fmt.Errorf("failed to get tpr client. %+v", err) } o.context.KubeHttpCli = httpCli.Client - err = createTPRs(o.context, o.tprSchemes) + err = k8sutil.CreateTPRs(o.context, o.tprSchemes) if err != nil { return fmt.Errorf("failed to create TPR. %+v", err) } return nil } - -func newHttpClient() (*rest.RESTClient, error) { - config, err := rest.InClusterConfig() - if err != nil { - return nil, err - } - - config.GroupVersion = &schema.GroupVersion{ - Group: tprGroup, - Version: tprVersion, - } - config.APIPath = "/apis" - config.ContentType = runtime.ContentTypeJSON - config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: api.Codecs} - - restcli, err := rest.RESTClientFor(config) - if err != nil { - return nil, err - } - return restcli, nil -} diff --git a/pkg/operator/operator_test.go b/pkg/operator/operator_test.go index cc0009cd1..4b1e2205b 100644 --- a/pkg/operator/operator_test.go +++ b/pkg/operator/operator_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -35,30 +36,14 @@ func TestCreateCluster(t *testing.T) { assert.NotNil(t, err) // create the tpr - test := &testTPR{} - err = createTPR(o.context, test) + scheme := k8sutil.TPRScheme{Name: "test", Version: "alpha", Description: "test description"} + err = k8sutil.CreateTPR(o.context, scheme) assert.Nil(t, err) tpr, err := clientset.ExtensionsV1beta1().ThirdPartyResources().List(v1.ListOptions{}) assert.Nil(t, err) assert.Equal(t, 1, len(tpr.Items)) assert.Equal(t, "test.rook.io", tpr.Items[0].Name) - assert.Equal(t, test.Description(), tpr.Items[0].Description) + assert.Equal(t, scheme.Description, tpr.Items[0].Description) // TODO: Watch for a new Rook cluster and create it. Need a mocked http client to be working } - -type testTPR struct { -} - -func (t *testTPR) Name() string { - return "test" -} -func (t *testTPR) Description() string { - return "test description" -} -func (t *testTPR) Load() error { - return nil -} -func (t *testTPR) Watch() error { - return nil -} diff --git a/pkg/operator/pool.go b/pkg/operator/pool.go index e86283618..8a264a83b 100644 --- a/pkg/operator/pool.go +++ b/pkg/operator/pool.go @@ -32,10 +32,6 @@ import ( kwatch "k8s.io/apimachinery/pkg/watch" ) -const ( - poolTprName = "pool" -) - type poolInitiator struct { context *clusterd.Context } @@ -51,20 +47,16 @@ func newPoolInitiator(context *clusterd.Context) *poolInitiator { return &poolInitiator{context: context} } -func (p *poolInitiator) Create(clusterMgr *clusterManager, clusterName, namespace string) (tprManager, error) { +func (p *poolInitiator) Create(clusterMgr *clusterManager, namespace string) (k8sutil.TPRManager, error) { rclient, err := clusterMgr.getRookClient(namespace) if err != nil { - return nil, fmt.Errorf("failed to get api client for pool tpr for cluster %s in namespace %s. %+v", clusterName, namespace, err) + return nil, fmt.Errorf("failed to get api client for pool tpr for cluster in namespace %s. %+v", namespace, err) } return &poolManager{context: p.context, namespace: namespace, rclient: rclient}, nil } -func (p *poolInitiator) Name() string { - return poolTprName -} - -func (p *poolInitiator) Description() string { - return "Managed Rook pools" +func (p *poolInitiator) Scheme() k8sutil.TPRScheme { + return cluster.PoolScheme } // Run the tpr manager until the caller signals with an EndWatch() @@ -108,7 +100,7 @@ func (p *poolManager) Load() error { } func (p *poolManager) getPoolList() (*cluster.PoolList, error) { - b, err := getRawListNamespaced(p.context.Clientset, poolTprName, p.namespace) + b, err := k8sutil.GetRawListNamespaced(p.context.Clientset, cluster.PoolScheme, p.namespace) if err != nil { return nil, err } @@ -189,7 +181,7 @@ func (p *poolManager) watch() (<-chan *poolEvent, <-chan error) { } func (p *poolManager) watchOuterTPR(eventCh chan *poolEvent, errCh chan error) error { - resp, err := watchTPRNamespaced(p.context, poolTprName, p.namespace, p.watchVersion) + resp, err := k8sutil.WatchTPRNamespaced(p.context, cluster.PoolScheme, p.namespace, p.watchVersion) if err != nil { errCh <- err return err diff --git a/pkg/operator/tpr.go b/pkg/operator/tpr.go index 09f3b55d2..0fbea612e 100644 --- a/pkg/operator/tpr.go +++ b/pkg/operator/tpr.go @@ -23,125 +23,14 @@ import ( "io" "net/http" - "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/pkg/apis/extensions/v1beta1" ) -const ( - tprGroup = "rook.io" - tprVersion = "v1alpha1" -) - -type tprScheme interface { - Name() string - Description() string -} - type inclusterInitiator interface { - Create(clusterMgr *clusterManager, name, namespace string) (tprManager, error) -} - -type tprManager interface { - Load() error - Watch() error - Manage() -} - -func qualifiedName(tpr tprScheme) string { - return fmt.Sprintf("%s.%s", tpr.Name(), tprGroup) -} - -func createTPRs(context *clusterd.Context, tprs []tprScheme) error { - for _, tpr := range tprs { - if err := createTPR(context, tpr); err != nil { - return fmt.Errorf("failed to init tpr %s. %+v", tpr.Name(), err) - } - } - - for _, tpr := range tprs { - if err := waitForTPRInit(context, tpr); err != nil { - return fmt.Errorf("failed to complete init %s. %+v", tpr.Name(), err) - } - } - - return nil -} - -func createTPR(context *clusterd.Context, tpr tprScheme) error { - logger.Infof("creating %s TPR", tpr.Name()) - r := &v1beta1.ThirdPartyResource{ - ObjectMeta: metav1.ObjectMeta{ - Name: qualifiedName(tpr), - }, - Versions: []v1beta1.APIVersion{ - {Name: tprVersion}, - }, - Description: tpr.Description(), - } - _, err := context.Clientset.ExtensionsV1beta1().ThirdPartyResources().Create(r) - if err != nil { - if !errors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create %s TPR. %+v", tpr.Name(), err) - } - } - - return nil -} - -func waitForTPRInit(context *clusterd.Context, tpr tprScheme) error { - restcli := context.Clientset.CoreV1().RESTClient() - uri := tprURI(tpr.Name()) - return k8sutil.Retry(context, func() (bool, error) { - _, err := restcli.Get().RequestURI(uri).DoRaw() - if err != nil { - logger.Infof("did not yet find tpr %s. %+v", tpr.Name(), err) - if errors.IsNotFound(err) { - return false, nil - } - return false, err - } - return true, nil - }) -} - -func watchTPRNamespaced(context *clusterd.Context, name, namespace, resourceVersion string) (*http.Response, error) { - uri := fmt.Sprintf("%s/%s?watch=true&resourceVersion=%s", context.MasterHost, tprURINamespaced(name, namespace), resourceVersion) - logger.Debugf("watching tpr: %s", uri) - return context.KubeHttpCli.Get(uri) -} - -func watchTPR(context *clusterd.Context, name, resourceVersion string) (*http.Response, error) { - uri := fmt.Sprintf("%s/%s?watch=true&resourceVersion=%s", context.MasterHost, tprURI(name), resourceVersion) - logger.Debugf("watching tpr: %s", uri) - return context.KubeHttpCli.Get(uri) -} - -func tprURINamespaced(name, namespace string) string { - // /apis/rook.io/v1alpha1/namespaces/rook/pools - return fmt.Sprintf("apis/%s/%s/namespaces/%s/%ss", tprGroup, tprVersion, namespace, name) -} - -func tprURI(name string) string { - // creates a uri that is for retrieving or watching a tpr in all namespaces. For example: - // /apis/rook.io/v1alpha1/clusters - return fmt.Sprintf("apis/%s/%s/%ss", tprGroup, tprVersion, name) -} - -func getRawListNamespaced(clientset kubernetes.Interface, name, namespace string) ([]byte, error) { - restcli := clientset.CoreV1().RESTClient() - uri := tprURINamespaced(name, namespace) - logger.Debugf("getting tpr: %s", uri) - return restcli.Get().RequestURI(uri).DoRaw() -} - -func getRawList(clientset kubernetes.Interface, name string) ([]byte, error) { - restcli := clientset.CoreV1().RESTClient() - return restcli.Get().RequestURI(tprURI(name)).DoRaw() + Create(clusterMgr *clusterManager, namespace string) (k8sutil.TPRManager, error) + Scheme() k8sutil.TPRScheme } func handlePollEventResult(status *metav1.Status, errIn error, checkStaleCache func() (bool, error), errCh chan error) (done bool, err error) {