operator: refactor to work with multiple TPRs

This commit is contained in:
Travis Nielsen committed 2017-06-28 08:43:27 -07:00
1 parent 67383efe7f
commit a5da34fe4c
9 files changed
+209 -193

No files matched your search

+16 -17
View File
@@ -36,7 +36,6 @@ import (
type clusterManager struct {
context *clusterd.Context
name string
watchVersion string
devicesInUse bool
clusters map[string]*cluster.Cluster
@@ -45,7 +44,7 @@ type clusterManager struct {
// The initiators that create TPRs specific to specific Rook clusters.
// For example, pools, object services, and file services, only make sense in the context of a Rook cluster
inclusterInitiators []inclusterInitiator
inclusterMgrs []tprManager
inclusterMgrs []k8sutil.TPRManager
}
func newClusterManager(context *clusterd.Context, inclusterInitiators []inclusterInitiator) *clusterManager {
@@ -57,16 +56,6 @@ func newClusterManager(context *clusterd.Context, inclusterInitiators []incluste
}
}
// Gets the name of the TPR
func (m *clusterManager) Name() string {
return "cluster"
}
// Gets the description of the TPR
func (m *clusterManager) Description() string {
return "Managed Rook clusters"
}
func (m *clusterManager) Manage() {
for {
logger.Infof("Managing clusters")
@@ -165,11 +154,11 @@ func (m *clusterManager) startCluster(c *cluster.Cluster) {
}
// Start all the TPRs for this cluster
for _, tpr := range m.inclusterInitiators {
for _, initiator := range m.inclusterInitiators {
k8sutil.Retry(m.context, func() (bool, error) {
tprMgr, err := tpr.Create(m, c.Name, c.Namespace)
tprMgr, err := initiator.Create(m, c.Namespace)
if err != nil {
logger.Warningf("cannot create in-cluster tpr %s. %+v. retrying...", m.Name(), err)
logger.Warningf("cannot create in-cluster tpr %s. %+v. retrying...", initiator.Scheme().Name, err)
return false, nil
}
@@ -203,7 +192,7 @@ func (m *clusterManager) isClustersCacheStale(currentClusters []cluster.Cluster)
}
func (m *clusterManager) getClusterList() (*cluster.ClusterList, error) {
b, err := getRawList(m.context.Clientset, m.Name())
b, err := k8sutil.GetRawList(m.context.Clientset, cluster.ClusterScheme)
if err != nil {
return nil, err
}
@@ -274,7 +263,7 @@ func (m *clusterManager) watch() (<-chan *clusterEvent, <-chan error) {
}
func (m *clusterManager) watchOuterTPR(eventCh chan *clusterEvent, errCh chan error) error {
resp, err := watchTPR(m.context, m.Name(), m.watchVersion)
resp, err := k8sutil.WatchTPR(m.context, cluster.ClusterScheme, m.watchVersion)
if err != nil {
errCh <- err
return err
@@ -323,3 +312,13 @@ func (m *clusterManager) getRookClient(namespace string) (rookclient.RookRestCli
return nil, fmt.Errorf("namespace %s not found", namespace)
}
func (m *clusterManager) getCluster(namespace string) (*cluster.Cluster, error) {
m.Lock()
defer m.Unlock()
if c, ok := m.clusters[namespace]; ok {
return c, nil
}
return nil, fmt.Errorf("cluster namespace %s not found", namespace)
}
+5
View File
@@ -49,6 +49,11 @@ var (
healthCheckInterval = 10 * time.Second
clientTimeout = 15 * time.Second
logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-cluster")
ClusterScheme = k8sutil.TPRScheme{
Name: "cluster",
Version: k8sutil.V1Alpha1,
Description: "Managed Rook clusters",
}
)
type Cluster struct {
+7
View File
@@ -21,6 +21,7 @@ package cluster
import (
"fmt"
"github.com/rook/rook/pkg/operator/k8sutil"
rookclient "github.com/rook/rook/pkg/rook/client"
"k8s.io/client-go/pkg/api/v1"
@@ -32,6 +33,12 @@ const (
erasureCodeType = "erasure-coded"
)
var PoolScheme = k8sutil.TPRScheme{
Name: "pool",
Version: k8sutil.V1Alpha1,
Description: "Managed Rook pools",
}
type Pool struct {
v1.ObjectMeta `json:"metadata,omitempty"`
PoolSpec `json:"spec"`
+25
View File
@@ -24,6 +24,11 @@ import (
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/pkg/clusterd"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/client-go/pkg/api"
"k8s.io/client-go/rest"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-k8sutil")
@@ -62,3 +67,23 @@ func Retry(context *clusterd.Context, f ConditionFunc) error {
}
return fmt.Errorf("failed after max retries %d.", context.MaxRetries)
}
func NewHttpClient() (*rest.RESTClient, error) {
config, err := rest.InClusterConfig()
if err != nil {
return nil, err
}
config.GroupVersion = &schema.GroupVersion{
Group: tprGroup,
}
config.APIPath = "/apis"
config.ContentType = runtime.ContentTypeJSON
config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: api.Codecs}
restcli, err := rest.RESTClientFor(config)
if err != nil {
return nil, err
}
return restcli, nil
}
+138
View File
@@ -0,0 +1,138 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Some of the code was modified from https://github.com/coreos/etcd-operator
which also has the apache 2.0 license.
*/
package k8sutil
import (
"fmt"
"net/http"
"github.com/rook/rook/pkg/clusterd"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/pkg/apis/extensions/v1beta1"
)
const (
tprGroup = "rook.io"
V1Alpha1 = "v1alpha1"
V1Beta1 = "v1beta1"
V1 = "v1"
)
type TPRScheme struct {
Name string
Version string
Description string
}
type TPRManager interface {
Load() error
Watch() error
Manage()
}
func CreateTPRs(context *clusterd.Context, tprs []TPRScheme) error {
for _, tpr := range tprs {
if err := CreateTPR(context, tpr); err != nil {
return fmt.Errorf("failed to init tpr %s. %+v", tpr.Name, err)
}
}
for _, tpr := range tprs {
if err := waitForTPRInit(context, tpr); err != nil {
return fmt.Errorf("failed to complete init %s. %+v", tpr.Name, err)
}
}
return nil
}
func CreateTPR(context *clusterd.Context, tpr TPRScheme) error {
logger.Infof("creating %s TPR", tpr.Name)
r := &v1beta1.ThirdPartyResource{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s.%s", tpr.Name, tprGroup),
},
Versions: []v1beta1.APIVersion{
{Name: tpr.Version},
},
Description: tpr.Description,
}
_, err := context.Clientset.ExtensionsV1beta1().ThirdPartyResources().Create(r)
if err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create %s TPR. %+v", tpr.Name, err)
}
}
return nil
}
func waitForTPRInit(context *clusterd.Context, scheme TPRScheme) error {
restcli := context.Clientset.CoreV1().RESTClient()
uri := TPRURI(scheme)
return Retry(context, func() (bool, error) {
_, err := restcli.Get().RequestURI(uri).DoRaw()
if err != nil {
logger.Infof("did not yet find tpr %s at %s. %+v", scheme.Name, uri, err)
if errors.IsNotFound(err) {
return false, nil
}
return false, err
}
return true, nil
})
}
func WatchTPRNamespaced(context *clusterd.Context, scheme TPRScheme, namespace, resourceVersion string) (*http.Response, error) {
uri := fmt.Sprintf("%s/%s?watch=true&resourceVersion=%s", context.MasterHost, TPRURINamespaced(scheme, namespace), resourceVersion)
logger.Debugf("watching tpr: %s", uri)
return context.KubeHttpCli.Get(uri)
}
func WatchTPR(context *clusterd.Context, scheme TPRScheme, resourceVersion string) (*http.Response, error) {
uri := fmt.Sprintf("%s/%s?watch=true&resourceVersion=%s", context.MasterHost, TPRURI(scheme), resourceVersion)
logger.Debugf("watching tpr: %s", uri)
return context.KubeHttpCli.Get(uri)
}
func TPRURINamespaced(scheme TPRScheme, namespace string) string {
// /apis/rook.io/v1alpha1/namespaces/rook/pools
return fmt.Sprintf("apis/%s/%s/namespaces/%s/%ss", tprGroup, scheme.Version, namespace, scheme.Name)
}
func TPRURI(scheme TPRScheme) string {
// creates a uri that is for retrieving or watching a tpr in all namespaces. For example:
// /apis/rook.io/v1alpha1/clusters
return fmt.Sprintf("apis/%s/%s/%ss", tprGroup, scheme.Version, scheme.Name)
}
func GetRawListNamespaced(clientset kubernetes.Interface, scheme TPRScheme, namespace string) ([]byte, error) {
restcli := clientset.CoreV1().RESTClient()
uri := TPRURINamespaced(scheme, namespace)
logger.Debugf("getting tpr: %s", uri)
return restcli.Get().RequestURI(uri).DoRaw()
}
func GetRawList(clientset kubernetes.Interface, scheme TPRScheme) ([]byte, error) {
restcli := clientset.CoreV1().RESTClient()
return restcli.Get().RequestURI(TPRURI(scheme)).DoRaw()
}
+6 -30
View File
@@ -25,12 +25,9 @@ import (
"github.com/kubernetes-incubator/external-storage/lib/controller"
"github.com/rook/rook/pkg/clusterd"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"github.com/rook/rook/pkg/operator/cluster"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/pkg/api"
"k8s.io/client-go/rest"
)
const (
@@ -55,7 +52,7 @@ var (
type Operator struct {
context *clusterd.Context
tprSchemes []tprScheme
tprSchemes []k8sutil.TPRScheme
// The TPR that is global to the kubernetes cluster.
// The cluster TPR is global because you create multiple clusers in k8s
clusterMgr *clusterManager
@@ -68,7 +65,7 @@ func New(context *clusterd.Context) *Operator {
clusterMgr := newClusterManager(context, []inclusterInitiator{poolInitiator})
volumeProvisioner := newRookVolumeProvisioner(clusterMgr)
schemes := []tprScheme{clusterMgr, poolInitiator}
schemes := []k8sutil.TPRScheme{cluster.ClusterScheme, cluster.PoolScheme}
return &Operator{
context: context,
clusterMgr: clusterMgr,
@@ -115,37 +112,16 @@ func (o *Operator) Run() error {
}
func (o *Operator) initResources() error {
httpCli, err := newHttpClient()
httpCli, err := k8sutil.NewHttpClient()
if err != nil {
return fmt.Errorf("failed to get tpr client. %+v", err)
}
o.context.KubeHttpCli = httpCli.Client
err = createTPRs(o.context, o.tprSchemes)
err = k8sutil.CreateTPRs(o.context, o.tprSchemes)
if err != nil {
return fmt.Errorf("failed to create TPR. %+v", err)
}
return nil
}
func newHttpClient() (*rest.RESTClient, error) {
config, err := rest.InClusterConfig()
if err != nil {
return nil, err
}
config.GroupVersion = &schema.GroupVersion{
Group: tprGroup,
Version: tprVersion,
}
config.APIPath = "/apis"
config.ContentType = runtime.ContentTypeJSON
config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: api.Codecs}
restcli, err := rest.RESTClientFor(config)
if err != nil {
return nil, err
}
return restcli, nil
}
+4 -19
View File
@@ -19,6 +19,7 @@ import (
"testing"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/operator/test"
"github.com/stretchr/testify/assert"
"k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,30 +36,14 @@ func TestCreateCluster(t *testing.T) {
assert.NotNil(t, err)
// create the tpr
test := &testTPR{}
err = createTPR(o.context, test)
scheme := k8sutil.TPRScheme{Name: "test", Version: "alpha", Description: "test description"}
err = k8sutil.CreateTPR(o.context, scheme)
assert.Nil(t, err)
tpr, err := clientset.ExtensionsV1beta1().ThirdPartyResources().List(v1.ListOptions{})
assert.Nil(t, err)
assert.Equal(t, 1, len(tpr.Items))
assert.Equal(t, "test.rook.io", tpr.Items[0].Name)
assert.Equal(t, test.Description(), tpr.Items[0].Description)
assert.Equal(t, scheme.Description, tpr.Items[0].Description)
// TODO: Watch for a new Rook cluster and create it. Need a mocked http client to be working
}
type testTPR struct {
}
func (t *testTPR) Name() string {
return "test"
}
func (t *testTPR) Description() string {
return "test description"
}
func (t *testTPR) Load() error {
return nil
}
func (t *testTPR) Watch() error {
return nil
}
+6 -14
View File
@@ -32,10 +32,6 @@ import (
kwatch "k8s.io/apimachinery/pkg/watch"
)
const (
poolTprName = "pool"
)
type poolInitiator struct {
context *clusterd.Context
}
@@ -51,20 +47,16 @@ func newPoolInitiator(context *clusterd.Context) *poolInitiator {
return &poolInitiator{context: context}
}
func (p *poolInitiator) Create(clusterMgr *clusterManager, clusterName, namespace string) (tprManager, error) {
func (p *poolInitiator) Create(clusterMgr *clusterManager, namespace string) (k8sutil.TPRManager, error) {
rclient, err := clusterMgr.getRookClient(namespace)
if err != nil {
return nil, fmt.Errorf("failed to get api client for pool tpr for cluster %s in namespace %s. %+v", clusterName, namespace, err)
return nil, fmt.Errorf("failed to get api client for pool tpr for cluster in namespace %s. %+v", namespace, err)
}
return &poolManager{context: p.context, namespace: namespace, rclient: rclient}, nil
}
func (p *poolInitiator) Name() string {
return poolTprName
}
func (p *poolInitiator) Description() string {
return "Managed Rook pools"
func (p *poolInitiator) Scheme() k8sutil.TPRScheme {
return cluster.PoolScheme
}
// Run the tpr manager until the caller signals with an EndWatch()
@@ -108,7 +100,7 @@ func (p *poolManager) Load() error {
}
func (p *poolManager) getPoolList() (*cluster.PoolList, error) {
b, err := getRawListNamespaced(p.context.Clientset, poolTprName, p.namespace)
b, err := k8sutil.GetRawListNamespaced(p.context.Clientset, cluster.PoolScheme, p.namespace)
if err != nil {
return nil, err
}
@@ -189,7 +181,7 @@ func (p *poolManager) watch() (<-chan *poolEvent, <-chan error) {
}
func (p *poolManager) watchOuterTPR(eventCh chan *poolEvent, errCh chan error) error {
resp, err := watchTPRNamespaced(p.context, poolTprName, p.namespace, p.watchVersion)
resp, err := k8sutil.WatchTPRNamespaced(p.context, cluster.PoolScheme, p.namespace, p.watchVersion)
if err != nil {
errCh <- err
return err
+2 -113
View File
@@ -23,125 +23,14 @@ import (
"io"
"net/http"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/pkg/apis/extensions/v1beta1"
)
const (
tprGroup = "rook.io"
tprVersion = "v1alpha1"
)
type tprScheme interface {
Name() string
Description() string
}
type inclusterInitiator interface {
Create(clusterMgr *clusterManager, name, namespace string) (tprManager, error)
}
type tprManager interface {
Load() error
Watch() error
Manage()
}
func qualifiedName(tpr tprScheme) string {
return fmt.Sprintf("%s.%s", tpr.Name(), tprGroup)
}
func createTPRs(context *clusterd.Context, tprs []tprScheme) error {
for _, tpr := range tprs {
if err := createTPR(context, tpr); err != nil {
return fmt.Errorf("failed to init tpr %s. %+v", tpr.Name(), err)
}
}
for _, tpr := range tprs {
if err := waitForTPRInit(context, tpr); err != nil {
return fmt.Errorf("failed to complete init %s. %+v", tpr.Name(), err)
}
}
return nil
}
func createTPR(context *clusterd.Context, tpr tprScheme) error {
logger.Infof("creating %s TPR", tpr.Name())
r := &v1beta1.ThirdPartyResource{
ObjectMeta: metav1.ObjectMeta{
Name: qualifiedName(tpr),
},
Versions: []v1beta1.APIVersion{
{Name: tprVersion},
},
Description: tpr.Description(),
}
_, err := context.Clientset.ExtensionsV1beta1().ThirdPartyResources().Create(r)
if err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create %s TPR. %+v", tpr.Name(), err)
}
}
return nil
}
func waitForTPRInit(context *clusterd.Context, tpr tprScheme) error {
restcli := context.Clientset.CoreV1().RESTClient()
uri := tprURI(tpr.Name())
return k8sutil.Retry(context, func() (bool, error) {
_, err := restcli.Get().RequestURI(uri).DoRaw()
if err != nil {
logger.Infof("did not yet find tpr %s. %+v", tpr.Name(), err)
if errors.IsNotFound(err) {
return false, nil
}
return false, err
}
return true, nil
})
}
func watchTPRNamespaced(context *clusterd.Context, name, namespace, resourceVersion string) (*http.Response, error) {
uri := fmt.Sprintf("%s/%s?watch=true&resourceVersion=%s", context.MasterHost, tprURINamespaced(name, namespace), resourceVersion)
logger.Debugf("watching tpr: %s", uri)
return context.KubeHttpCli.Get(uri)
}
func watchTPR(context *clusterd.Context, name, resourceVersion string) (*http.Response, error) {
uri := fmt.Sprintf("%s/%s?watch=true&resourceVersion=%s", context.MasterHost, tprURI(name), resourceVersion)
logger.Debugf("watching tpr: %s", uri)
return context.KubeHttpCli.Get(uri)
}
func tprURINamespaced(name, namespace string) string {
// /apis/rook.io/v1alpha1/namespaces/rook/pools
return fmt.Sprintf("apis/%s/%s/namespaces/%s/%ss", tprGroup, tprVersion, namespace, name)
}
func tprURI(name string) string {
// creates a uri that is for retrieving or watching a tpr in all namespaces. For example:
// /apis/rook.io/v1alpha1/clusters
return fmt.Sprintf("apis/%s/%s/%ss", tprGroup, tprVersion, name)
}
func getRawListNamespaced(clientset kubernetes.Interface, name, namespace string) ([]byte, error) {
restcli := clientset.CoreV1().RESTClient()
uri := tprURINamespaced(name, namespace)
logger.Debugf("getting tpr: %s", uri)
return restcli.Get().RequestURI(uri).DoRaw()
}
func getRawList(clientset kubernetes.Interface, name string) ([]byte, error) {
restcli := clientset.CoreV1().RESTClient()
return restcli.Get().RequestURI(tprURI(name)).DoRaw()
Create(clusterMgr *clusterManager, namespace string) (k8sutil.TPRManager, error)
Scheme() k8sutil.TPRScheme
}
func handlePollEventResult(status *metav1.Status, errIn error, checkStaleCache func() (bool, error), errCh chan error) (done bool, err error) {