use third party resources to trigger cluster creation

This commit is contained in:
Travis Nielsen committed 2017-02-24 15:57:37 -08:00
1 parent 1573bdb1dd
commit 34b6ad32d7
13 files changed
+757 -116

No files matched your search

+10 -10
View File
@@ -23,7 +23,6 @@ import (
"github.com/rook/rook/pkg/cephmgr/cephd"
"github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/operator"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/flags"
"github.com/spf13/cobra"
"k8s.io/client-go/kubernetes"
@@ -46,8 +45,8 @@ type config struct {
dataDir string
cephConfigOverride string
clusterInfo mon.ClusterInfo
namespace string
monEndpoints string
useAllDevices bool
}
var logLevelRaw string
@@ -75,7 +74,7 @@ func init() {
rootCmd.PersistentFlags().StringVar(&cfg.dataDir, "data-dir", "/var/lib/rook", "directory for storing configuration")
rootCmd.PersistentFlags().StringVar(&logLevelRaw, "log-level", "INFO", "logging level for logging/tracing output (valid values: CRITICAL,ERROR,WARNING,NOTICE,INFO,DEBUG,TRACE)")
rootCmd.Flags().BoolVar(&cfg.useAllDevices, "use-all-devices", false, "true to use all storage devices, false to require local device selection")
rootCmd.Flags().StringVar(&cfg.namespace, "namespace", "", "the namespace in which the operator is running (required)")
// load the environment variables
flags.SetFlagsFromEnv(rootCmd.Flags(), "ROOK_OPERATOR")
@@ -86,20 +85,20 @@ func init() {
func startOperator(cmd *cobra.Command, args []string) error {
// verify required flags
if err := flags.VerifyRequiredFlags(cmd, []string{}); err != nil {
if err := flags.VerifyRequiredFlags(cmd, []string{"namespace"}); err != nil {
return err
}
setLogLevel()
clientset, err := getClientset()
host, clientset, err := getClientset()
if err != nil {
fmt.Printf("failed to get k8s client. %+v", err)
os.Exit(1)
}
logger.Infof("starting operator. containerVersion=%s", cfg.containerVersion)
op := operator.New(k8sutil.Namespace, cephd.New(), clientset, cfg.containerVersion, cfg.useAllDevices)
logger.Infof("starting operator in namespace %s", cfg.namespace)
op := operator.New(host, cfg.namespace, cephd.New(), clientset)
err = op.Run()
if err != nil {
fmt.Printf("failed to run operator. %+v\n", err)
@@ -119,12 +118,13 @@ func setLogLevel() {
capnslog.SetGlobalLogLevel(cfg.logLevel)
}
func getClientset() (*kubernetes.Clientset, error) {
func getClientset() (string, *kubernetes.Clientset, error) {
// create the k8s client
config, err := rest.InClusterConfig()
if err != nil {
return nil, fmt.Errorf("failed to get k8s config. %+v", err)
return "", nil, fmt.Errorf("failed to get k8s config. %+v", err)
}
return kubernetes.NewForConfig(config)
c, err := kubernetes.NewForConfig(config)
return config.Host, c, err
}
+1 -1
View File
@@ -55,7 +55,7 @@ func startAPI(cmd *cobra.Command, args []string) error {
setLogLevel()
clientset, err := getClientset()
_, clientset, err := getClientset()
if err != nil {
fmt.Printf("failed to init k8s client. %+v\n", err)
os.Exit(1)
+2 -2
View File
@@ -14,12 +14,12 @@ Note that we are striving for even more smooth integration with Kubernetes in th
### Deploy Rook
With your Kubernetes cluster running, Rook can be setup and deployed by simply deploying the [rook-operator](rook-operator.yaml) deployment manifest.
With your Kubernetes cluster running, Rook can be setup and deployed by simply creating the [rook-operator](rook-operator.yaml) deployment and creating a [rook cluster](rook-cluster.yaml).
```
cd demo/kubernetes
kubectl create namespace rook
kubectl create -f rook-operator.yaml
kubectl create -f rook-cluster.yaml
```
Use `kubectl` to list pods in the rook namespace. You should be able to see the following:
+8
View File
@@ -0,0 +1,8 @@
apiVersion: rook.io/v1beta1
kind: Cluster
metadata:
name: my-rook
spec:
namespace: rook
version: dev-2017-02-24-k8s
useAllDevices: false
+6 -3
View File
@@ -2,7 +2,6 @@ apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: rook-operator
namespace: rook
spec:
replicas: 1
template:
@@ -12,5 +11,9 @@ spec:
spec:
containers:
- name: rook-operator
image: quay.io/rook/rook-operator:dev-2017-02-17-k8s
args: ["--container-version=dev-2017-02-17-k8s"]
image: quay.io/rook/rook-operator:dev-2017-02-24-k8s
env:
- name: ROOK_OPERATOR_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
+169
View File
@@ -0,0 +1,169 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Some of the code below came from https://github.com/coreos/etcd-operator
which also has the apache 2.0 license.
*/
package operator
import (
"fmt"
"github.com/rook/rook/pkg/cephmgr/client"
cephmon "github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/api"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/operator/mon"
"github.com/rook/rook/pkg/operator/osd"
"github.com/rook/rook/pkg/operator/rgw"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/pkg/api/v1"
)
type Cluster struct {
Namespace string
factory client.ConnectionFactory
clientset *kubernetes.Clientset
Metadata v1.ObjectMeta `json:"metadata,omitempty"`
Spec ClusterSpec `json:"spec"`
}
type ClusterSpec struct {
// The namespace where the the rook resources will all be created.
Namespace string `json:"namespace"`
// Version is the expected version of the rook container to run in the cluster.
// The rook-operator will eventually make the rook cluster version
// equal to the expected version.
Version string `json:"version"`
// Paused is to pause the control of the operator for the rook cluster.
Paused bool `json:"paused,omitempty"`
// Whether to consume all the storage devices found on a machine
UseAllDevices bool `json:"useAllDevices"`
}
func newCluster(namespace, version string, useAllDevices bool, factory client.ConnectionFactory, clientset *kubernetes.Clientset) *Cluster {
c := &Cluster{
Namespace: namespace,
factory: factory,
clientset: clientset,
}
c.Spec.Version = version
c.Spec.UseAllDevices = useAllDevices
return c
}
func (c *Cluster) CreateInstance() error {
// Create the namespace if not already created
ns := &v1.Namespace{ObjectMeta: v1.ObjectMeta{Name: c.Namespace}}
_, err := c.clientset.Namespaces().Create(ns)
if err != nil {
if !k8sutil.IsKubernetesResourceAlreadyExistError(err) {
return fmt.Errorf("failed to create namespace %s. %+v", c.Namespace, err)
}
}
// Start the mon pods
m := mon.New(c.Namespace, c.factory, c.Spec.Version)
cluster, err := m.Start(c.clientset)
if err != nil {
return fmt.Errorf("failed to start the mons. %+v", err)
}
a := api.New(c.Namespace, c.Spec.Version)
err = a.Start(c.clientset, cluster)
if err != nil {
return fmt.Errorf("failed to start the REST api. %+v", err)
}
// Start the OSDs
osds := osd.New(c.Namespace, c.Spec.Version, c.Spec.UseAllDevices)
err = osds.Start(c.clientset, cluster)
if err != nil {
return fmt.Errorf("failed to start the osds. %+v", err)
}
// Start the object store
r := rgw.New(c.Namespace, c.Spec.Version, c.factory)
err = r.Start(c.clientset, cluster)
if err != nil {
return fmt.Errorf("failed to start rgw. %+v", err)
}
err = c.createClientAccess(cluster)
if err != nil {
return fmt.Errorf("failed to create client access. %+v", err)
}
logger.Infof("Done creating rook instance in namespace %s", c.Namespace)
return nil
}
func (c *Cluster) createClientAccess(clusterInfo *cephmon.ClusterInfo) error {
context := &clusterd.Context{}
conn, err := cephmon.ConnectToClusterAsAdmin(context, c.factory, clusterInfo)
if err != nil {
return fmt.Errorf("failed to connect to cluster: %+v", err)
}
defer conn.Shutdown()
// create the default rook pool
pool := client.CephStoragePoolDetails{Name: defaultPool}
_, err = client.CreatePool(conn, pool)
if err != nil {
return fmt.Errorf("failed to create default rook pool: %+v", err)
}
// create a user for rbd clients
username := "client.rook-rbd-user"
access := []string{"osd", "allow rwx", "mon", "allow r"}
// get-or-create-key for the user account
rbdKey, err := client.AuthGetOrCreateKey(conn, username, access)
if err != nil {
return fmt.Errorf("failed to get or create auth key for %s. %+v", username, err)
}
// store the secret for the rbd user in the default namespace
secrets := map[string]string{
"key": rbdKey,
}
secret := &v1.Secret{
ObjectMeta: v1.ObjectMeta{Name: "rook-rbd-user"},
StringData: secrets,
Type: k8sutil.RbdType,
}
_, err = c.clientset.Secrets(k8sutil.DefaultNamespace).Create(secret)
if err != nil {
if !k8sutil.IsKubernetesResourceAlreadyExistError(err) {
return fmt.Errorf("failed to save rook-rbd-user secret. %+v", err)
}
// update the secret in case we have a new cluster
_, err = c.clientset.Secrets(k8sutil.DefaultNamespace).Update(secret)
if err != nil {
return fmt.Errorf("failed to update rook-rbd-user secret. %+v", err)
}
logger.Infof("updated existing rook-rbd-user secret")
} else {
logger.Infof("saved rook-rbd-user secret")
}
return nil
}
+66
View File
@@ -0,0 +1,66 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Some of the code below came from https://github.com/coreos/etcd-operator
which also has the apache 2.0 license.
*/
package operator
import (
"encoding/json"
"k8s.io/client-go/pkg/api/unversioned"
)
// ClusterList is a list of rook clusters.
type ClusterList struct {
unversioned.TypeMeta `json:",inline"`
// Standard list metadata
// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
Metadata unversioned.ListMeta `json:"metadata,omitempty"`
// Items is a list of third party objects
Items []Cluster `json:"items"`
}
// There is known issue with TPR in client-go:
// https://github.com/kubernetes/client-go/issues/8
// Workarounds:
// - We include `Metadata` field in object explicitly.
// - we have the code below to work around a known problem with third-party resources and ugorji.
type ClusterListCopy ClusterList
type ClusterCopy Cluster
func (c *Cluster) UnmarshalJSON(data []byte) error {
tmp := ClusterCopy{}
err := json.Unmarshal(data, &tmp)
if err != nil {
return err
}
tmp2 := Cluster(tmp)
*c = tmp2
return nil
}
func (cl *ClusterList) UnmarshalJSON(data []byte) error {
tmp := ClusterListCopy{}
err := json.Unmarshal(data, &tmp)
if err != nil {
return err
}
tmp2 := ClusterList(tmp)
*cl = tmp2
return nil
}
+68
View File
@@ -0,0 +1,68 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Some of the code below came from https://github.com/coreos/etcd-operator
which also has the apache 2.0 license.
*/
package operator
import (
"encoding/json"
"fmt"
"io"
"k8s.io/client-go/pkg/api/unversioned"
kwatch "k8s.io/client-go/pkg/watch"
)
type Event struct {
Type kwatch.EventType
Object *Cluster
}
type rawEvent struct {
Type kwatch.EventType
Object json.RawMessage
}
func pollEvent(decoder *json.Decoder) (*Event, *unversioned.Status, error) {
re := &rawEvent{}
err := decoder.Decode(re)
if err != nil {
if err == io.EOF {
return nil, nil, err
}
return nil, nil, fmt.Errorf("fail to decode raw event from apiserver (%v)", err)
}
if re.Type == kwatch.Error {
status := &unversioned.Status{}
err = json.Unmarshal(re.Object, status)
if err != nil {
return nil, nil, fmt.Errorf("fail to decode (%s) into unversioned.Status (%v)", re.Object, err)
}
return nil, status, nil
}
ev := &Event{
Type: re.Type,
Object: &Cluster{},
}
err = json.Unmarshal(re.Object, ev.Object)
if err != nil {
return nil, nil, fmt.Errorf("fail to unmarshal Cluster object from data (%s): %v", re.Object, err)
}
return ev, nil, nil
}
+27
View File
@@ -19,7 +19,9 @@ which also has the apache 2.0 license.
package k8sutil
import (
"fmt"
"net/http"
"time"
apierrors "k8s.io/client-go/pkg/api/errors"
"k8s.io/client-go/pkg/api/unversioned"
@@ -34,6 +36,8 @@ const (
RbdType = "kubernetes.io/rbd"
)
type ConditionFunc func() (bool, error)
func IsKubernetesResourceAlreadyExistError(err error) bool {
se, ok := err.(*apierrors.StatusError)
if !ok {
@@ -55,3 +59,26 @@ func IsKubernetesResourceNotFoundError(err error) bool {
}
return false
}
// Retry retries f every interval until after maxRetries.
// The interval won't be affected by how long f takes.
// For example, if interval is 3s, f takes 1s, another f will be called 2s later.
// However, if f takes longer than interval, it will be delayed.
func Retry(interval time.Duration, maxRetries int, f ConditionFunc) error {
tick := time.NewTicker(interval)
defer tick.Stop()
for i := 0; i < maxRetries; i++ {
ok, err := f()
if err != nil {
return fmt.Errorf("failed on retry %d. %+v", i, err)
}
if ok {
return nil
}
if i < maxRetries-1 {
<-tick.C
}
}
return fmt.Errorf("failed after max retries %d.", maxRetries)
}
+49
View File
@@ -0,0 +1,49 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Some of the code below came from https://github.com/coreos/etcd-operator
which also has the apache 2.0 license.
*/
package k8sutil
import "time"
// panicTimer panics when it reaches the given duration.
type PanicTimer struct {
d time.Duration
msg string
t *time.Timer
}
func NewPanicTimer(d time.Duration, msg string) *PanicTimer {
return &PanicTimer{
d: d,
msg: msg,
}
}
func (pt *PanicTimer) Start() {
pt.t = time.AfterFunc(pt.d, func() {
panic(pt.msg)
})
}
// stop stops the timer and resets the elapsed duration.
func (pt *PanicTimer) Stop() {
if pt.t != nil {
pt.t.Stop()
pt.t = nil
}
}
+275 -99
View File
@@ -12,25 +12,40 @@ distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Some of the code below came from https://github.com/coreos/etcd-operator
which also has the apache 2.0 license.
*/
package operator
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"sync"
"time"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/pkg/api/v1"
"k8s.io/client-go/rest"
"github.com/rook/rook/pkg/cephmgr/client"
cephmon "github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/api"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/operator/mon"
"github.com/rook/rook/pkg/operator/osd"
"github.com/rook/rook/pkg/operator/rgw"
"k8s.io/client-go/pkg/api"
"k8s.io/client-go/pkg/api/unversioned"
"k8s.io/client-go/pkg/runtime"
"k8s.io/client-go/pkg/runtime/serializer"
kwatch "k8s.io/client-go/pkg/watch"
)
const (
initRetryDelay = 10 * time.Second
)
var (
ErrVersionOutdated = errors.New("requested version is outdated in apiserver")
)
const (
@@ -38,114 +53,275 @@ const (
)
type Operator struct {
Namespace string
MasterHost string
containerVersion string
clientset *kubernetes.Clientset
waitCluster sync.WaitGroup
factory client.ConnectionFactory
useAllDevices bool
Namespace string
MasterHost string
clientset *kubernetes.Clientset
waitCluster sync.WaitGroup
factory client.ConnectionFactory
stopChMap map[string]chan struct{}
clusters map[string]*Cluster
kubeHttpCli *http.Client
// Kubernetes resource version of the clusters
clusterRVs map[string]string
}
func New(namespace string, factory client.ConnectionFactory, clientset *kubernetes.Clientset, containerVersion string, useAllDevices bool) *Operator {
func New(host, namespace string, factory client.ConnectionFactory, clientset *kubernetes.Clientset) *Operator {
return &Operator{
Namespace: namespace,
factory: factory,
clientset: clientset,
containerVersion: containerVersion,
useAllDevices: useAllDevices,
Namespace: namespace,
MasterHost: host,
factory: factory,
clientset: clientset,
clusters: make(map[string]*Cluster),
clusterRVs: make(map[string]string),
stopChMap: map[string]chan struct{}{},
}
}
func (o *Operator) Run() error {
// Start the mon pods
m := mon.New(o.Namespace, o.factory, o.containerVersion)
cluster, err := m.Start(o.clientset)
if err != nil {
return fmt.Errorf("failed to start the mons. %+v", err)
var watchVersion string
var err error
for {
watchVersion, err = o.initResources()
if err == nil {
break
}
logger.Errorf("failed to create tpr. %+v. retrying...", err)
<-time.After(initRetryDelay)
}
a := api.New(o.Namespace, o.containerVersion)
err = a.Start(o.clientset, cluster)
if err != nil {
return fmt.Errorf("failed to start the REST api. %+v", err)
}
// Start the OSDs
osds := osd.New(o.Namespace, o.containerVersion, o.useAllDevices)
err = osds.Start(o.clientset, cluster)
if err != nil {
return fmt.Errorf("failed to start the osds. %+v", err)
}
// Start the object store
r := rgw.New(o.Namespace, o.containerVersion, o.factory)
err = r.Start(o.clientset, cluster)
if err != nil {
return fmt.Errorf("failed to start rgw. %+v", err)
}
err = o.createClientAccess(cluster)
if err != nil {
return fmt.Errorf("failed to create client access. %+v", err)
}
logger.Infof("DONE!")
<-time.After(1000000 * time.Second)
return nil
// watch for changes to the rook clusters
return o.watchTPR(watchVersion)
}
func (o *Operator) createClientAccess(clusterInfo *cephmon.ClusterInfo) error {
context := &clusterd.Context{}
conn, err := cephmon.ConnectToClusterAsAdmin(context, o.factory, clusterInfo)
func (o *Operator) initResources() (string, error) {
httpCli, err := newHttpClient()
if err != nil {
return fmt.Errorf("failed to connect to cluster: %+v", err)
return "", fmt.Errorf("failed to get tpr client. %+v", err)
}
defer conn.Shutdown()
o.kubeHttpCli = httpCli.Client
// create the default rook pool
pool := client.CephStoragePoolDetails{Name: defaultPool}
_, err = client.CreatePool(conn, pool)
err = o.createTPR()
if err != nil {
return fmt.Errorf("failed to create default rook pool: %+v", err)
}
// create a user for rbd clients
username := "client.rook-rbd-user"
access := []string{"osd", "allow rwx", "mon", "allow r"}
// get-or-create-key for the user account
rbdKey, err := client.AuthGetOrCreateKey(conn, username, access)
if err != nil {
return fmt.Errorf("failed to get or create auth key for %s. %+v", username, err)
}
// store the secret for the rbd user in the default namespace
secrets := map[string]string{
"key": rbdKey,
}
secret := &v1.Secret{
ObjectMeta: v1.ObjectMeta{Name: "rook-rbd-user"},
StringData: secrets,
Type: k8sutil.RbdType,
}
_, err = o.clientset.Secrets(k8sutil.DefaultNamespace).Create(secret)
if err != nil {
if !k8sutil.IsKubernetesResourceAlreadyExistError(err) {
return fmt.Errorf("failed to save rook-rbd-user secret. %+v", err)
if k8sutil.IsKubernetesResourceAlreadyExistError(err) {
// TPR has been initialized before. We need to recover existing cluster.
watchVersion, err := o.findAllClusters()
if err != nil {
return "", fmt.Errorf("failed to find clusters. %+v", err)
}
return watchVersion, nil
}
// update the secret in case we have a new cluster
_, err = o.clientset.Secrets(k8sutil.DefaultNamespace).Update(secret)
if err != nil {
return fmt.Errorf("failed to update rook-rbd-user secret. %+v", err)
}
logger.Infof("updated existing rook-rbd-user secret")
} else {
logger.Infof("saved rook-rbd-user secret")
}
return nil
return "0", nil
}
func (o *Operator) watchTPR(watchVersion string) error {
logger.Infof("start watching rook tpr: %s", watchVersion)
defer func() {
for _, stop := range o.stopChMap {
close(stop)
}
o.waitCluster.Wait()
}()
eventCh, errCh := o.watch(watchVersion)
go func() {
pt := k8sutil.NewPanicTimer(time.Minute, "unexpected long blocking (> 1 Minute) when handling cluster event")
for event := range eventCh {
pt.Start()
c := event.Object
switch event.Type {
case kwatch.Added:
ns := c.Spec.Namespace
if ns == "" {
logger.Errorf("missing namespace attribute in rook spec")
continue
}
if len(o.stopChMap) > 0 {
logger.Warningf("only a single rook cluster is currently supported. Ignoring cluster for namespace %s", ns)
} else {
newCluster := newCluster(ns, c.Spec.Version, c.Spec.UseAllDevices, o.factory, o.clientset)
stopCh := make(chan struct{})
logger.Infof("starting cluster. %+v", newCluster)
o.stopChMap[ns] = stopCh
o.clusters[ns] = newCluster
o.clusterRVs[ns] = c.Metadata.ResourceVersion
err := newCluster.CreateInstance()
if err != nil {
logger.Errorf("failed to create cluster %s. %+v", ns, err)
}
}
case kwatch.Modified:
logger.Infof("modifying a cluster not implemented")
case kwatch.Deleted:
logger.Infof("deleting a cluster not implemented")
}
pt.Stop()
}
}()
return <-errCh
}
func (o *Operator) findAllClusters() (string, error) {
logger.Info("finding existing clusters...")
clusterList, err := getClusterList(o.clientset.CoreV1().RESTClient(), o.Namespace)
if err != nil {
return "", err
}
for i := range clusterList.Items {
c := clusterList.Items[i]
stopCh := make(chan struct{})
ns := c.Spec.Namespace
newCluster := newCluster(ns, c.Spec.Version, c.Spec.UseAllDevices, o.factory, o.clientset)
o.stopChMap[ns] = stopCh
o.clusters[ns] = newCluster
o.clusterRVs[ns] = c.Metadata.ResourceVersion
}
return clusterList.Metadata.ResourceVersion, nil
}
// watch creates a go routine, and watches the cluster.rook kind resources from
// the given watch version. It emits events on the resources through the returned
// event chan. Errors will be reported through the returned error chan. The go routine
// exits on any error.
func (o *Operator) watch(watchVersion string) (<-chan *Event, <-chan error) {
eventCh := make(chan *Event)
// On unexpected error case, the operator should exit
errCh := make(chan error, 1)
go func() {
defer close(eventCh)
for {
resp, err := watchClusters(o.MasterHost, o.Namespace, o.kubeHttpCli, watchVersion)
if err != nil {
errCh <- err
return
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
errCh <- errors.New("invalid status code: " + resp.Status)
return
}
logger.Infof("start watching at %v", watchVersion)
decoder := json.NewDecoder(resp.Body)
for {
ev, st, err := pollEvent(decoder)
if err != nil {
if err == io.EOF { // apiserver will close stream periodically
logger.Debug("apiserver closed stream")
break
}
logger.Errorf("received invalid event from API server: %v", err)
errCh <- err
return
}
if st != nil {
resp.Body.Close()
if st.Code == http.StatusGone {
// event history is outdated.
// if nothing has changed, we can go back to watch again.
clusterList, err := getClusterList(o.clientset.CoreV1().RESTClient(), o.Namespace)
if err == nil && !o.isClustersCacheStale(clusterList.Items) {
watchVersion = clusterList.Metadata.ResourceVersion
break
}
// if anything has changed (or error on relist), we have to rebuild the state.
// go to recovery path
errCh <- ErrVersionOutdated
return
}
logger.Errorf("unexpected status response from API server: %v", st.Message)
break
}
logger.Debugf("rook cluster event: %+v", ev)
watchVersion = ev.Object.Metadata.ResourceVersion
eventCh <- ev
}
resp.Body.Close()
}
}()
return eventCh, errCh
}
func (o *Operator) isClustersCacheStale(currentClusters []Cluster) bool {
if len(o.clusterRVs) != len(currentClusters) {
return true
}
for _, cc := range currentClusters {
rv, ok := o.clusterRVs[cc.Metadata.Name]
if !ok || rv != cc.Metadata.ResourceVersion {
return true
}
}
return false
}
func watchClusters(host, ns string, httpClient *http.Client, resourceVersion string) (*http.Response, error) {
return httpClient.Get(fmt.Sprintf("%s/apis/%s/%s/namespaces/%s/clusters?watch=true&resourceVersion=%s",
host, tprGroup, tprVersion, ns, resourceVersion))
}
func getClusterList(restcli rest.Interface, ns string) (*ClusterList, error) {
b, err := restcli.Get().RequestURI(listClustersURI(ns)).DoRaw()
if err != nil {
return nil, err
}
clusters := &ClusterList{}
if err := json.Unmarshal(b, clusters); err != nil {
return nil, err
}
return clusters, nil
}
func listClustersURI(ns string) string {
return fmt.Sprintf("/apis/%s/%s/namespaces/%s/clusters", tprGroup, tprVersion, ns)
}
func newHttpClient() (*rest.RESTClient, error) {
config, err := rest.InClusterConfig()
if err != nil {
return nil, err
}
config.GroupVersion = &unversioned.GroupVersion{
Group: tprGroup,
Version: tprVersion,
}
config.APIPath = "/apis"
config.ContentType = runtime.ContentTypeJSON
config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: api.Codecs}
restcli, err := rest.RESTClientFor(config)
if err != nil {
return nil, err
}
return restcli, nil
}
+1 -1
View File
@@ -18,5 +18,5 @@ package operator
import "testing"
func TestOperator(t *testing.T) {
New("foo", nil, nil, "version", true)
New("foo", "rook", nil, nil)
}
+75
View File
@@ -0,0 +1,75 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Some of the code was modified from https://github.com/coreos/etcd-operator
which also has the apache 2.0 license.
*/
package operator
import (
"fmt"
"time"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/client-go/pkg/api/errors"
"k8s.io/client-go/pkg/api/v1"
"k8s.io/client-go/pkg/apis/extensions/v1beta1"
"k8s.io/client-go/rest"
)
const (
tprKind = "cluster"
tprGroup = "rook.io"
tprVersion = "v1beta1"
tprDescription = "Managed rook clusters"
)
func tprName() string {
return fmt.Sprintf("%s.%s", tprKind, tprGroup)
}
func (o *Operator) createTPR() error {
logger.Info("creating rook TPR")
tpr := &v1beta1.ThirdPartyResource{
ObjectMeta: v1.ObjectMeta{
Name: tprName(),
},
Versions: []v1beta1.APIVersion{
{Name: tprVersion},
},
Description: tprDescription,
}
_, err := o.clientset.ExtensionsV1beta1().ThirdPartyResources().Create(tpr)
if err != nil {
return fmt.Errorf("failed to create rook third party resources. %+v", err)
}
return o.waitForTPRInit(o.clientset.CoreV1().RESTClient(), 3*time.Second, 90*time.Second, k8sutil.Namespace)
}
func (o *Operator) waitForTPRInit(restcli rest.Interface, interval, timeout time.Duration, ns string) error {
uri := fmt.Sprintf("/apis/%s/%s/namespaces/%s/clusters", tprGroup, tprVersion, ns)
return k8sutil.Retry(interval, int(timeout/interval), func() (bool, error) {
_, err := restcli.Get().RequestURI(uri).DoRaw()
if err != nil {
if errors.IsNotFound(err) {
return false, nil
}
return false, err
}
return true, nil
})
}