diff --git a/cmd/rook-operator/main.go b/cmd/rook-operator/main.go index 4c2f57933..14818c168 100644 --- a/cmd/rook-operator/main.go +++ b/cmd/rook-operator/main.go @@ -23,7 +23,6 @@ import ( "github.com/rook/rook/pkg/cephmgr/cephd" "github.com/rook/rook/pkg/cephmgr/mon" "github.com/rook/rook/pkg/operator" - "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/flags" "github.com/spf13/cobra" "k8s.io/client-go/kubernetes" @@ -46,8 +45,8 @@ type config struct { dataDir string cephConfigOverride string clusterInfo mon.ClusterInfo + namespace string monEndpoints string - useAllDevices bool } var logLevelRaw string @@ -75,7 +74,7 @@ func init() { rootCmd.PersistentFlags().StringVar(&cfg.dataDir, "data-dir", "/var/lib/rook", "directory for storing configuration") rootCmd.PersistentFlags().StringVar(&logLevelRaw, "log-level", "INFO", "logging level for logging/tracing output (valid values: CRITICAL,ERROR,WARNING,NOTICE,INFO,DEBUG,TRACE)") - rootCmd.Flags().BoolVar(&cfg.useAllDevices, "use-all-devices", false, "true to use all storage devices, false to require local device selection") + rootCmd.Flags().StringVar(&cfg.namespace, "namespace", "", "the namespace in which the operator is running (required)") // load the environment variables flags.SetFlagsFromEnv(rootCmd.Flags(), "ROOK_OPERATOR") @@ -86,20 +85,20 @@ func init() { func startOperator(cmd *cobra.Command, args []string) error { // verify required flags - if err := flags.VerifyRequiredFlags(cmd, []string{}); err != nil { + if err := flags.VerifyRequiredFlags(cmd, []string{"namespace"}); err != nil { return err } setLogLevel() - clientset, err := getClientset() + host, clientset, err := getClientset() if err != nil { fmt.Printf("failed to get k8s client. %+v", err) os.Exit(1) } - logger.Infof("starting operator. containerVersion=%s", cfg.containerVersion) - op := operator.New(k8sutil.Namespace, cephd.New(), clientset, cfg.containerVersion, cfg.useAllDevices) + logger.Infof("starting operator in namespace %s", cfg.namespace) + op := operator.New(host, cfg.namespace, cephd.New(), clientset) err = op.Run() if err != nil { fmt.Printf("failed to run operator. %+v\n", err) @@ -119,12 +118,13 @@ func setLogLevel() { capnslog.SetGlobalLogLevel(cfg.logLevel) } -func getClientset() (*kubernetes.Clientset, error) { +func getClientset() (string, *kubernetes.Clientset, error) { // create the k8s client config, err := rest.InClusterConfig() if err != nil { - return nil, fmt.Errorf("failed to get k8s config. %+v", err) + return "", nil, fmt.Errorf("failed to get k8s config. %+v", err) } - return kubernetes.NewForConfig(config) + c, err := kubernetes.NewForConfig(config) + return config.Host, c, err } diff --git a/cmd/rook-operator/restapi.go b/cmd/rook-operator/restapi.go index 86b4e7b1f..8e19e8f57 100644 --- a/cmd/rook-operator/restapi.go +++ b/cmd/rook-operator/restapi.go @@ -55,7 +55,7 @@ func startAPI(cmd *cobra.Command, args []string) error { setLogLevel() - clientset, err := getClientset() + _, clientset, err := getClientset() if err != nil { fmt.Printf("failed to init k8s client. %+v\n", err) os.Exit(1) diff --git a/demo/kubernetes/README.md b/demo/kubernetes/README.md index 7cd43e87e..8d9aa2a13 100644 --- a/demo/kubernetes/README.md +++ b/demo/kubernetes/README.md @@ -14,12 +14,12 @@ Note that we are striving for even more smooth integration with Kubernetes in th ### Deploy Rook -With your Kubernetes cluster running, Rook can be setup and deployed by simply deploying the [rook-operator](rook-operator.yaml) deployment manifest. +With your Kubernetes cluster running, Rook can be setup and deployed by simply creating the [rook-operator](rook-operator.yaml) deployment and creating a [rook cluster](rook-cluster.yaml). ``` cd demo/kubernetes -kubectl create namespace rook kubectl create -f rook-operator.yaml +kubectl create -f rook-cluster.yaml ``` Use `kubectl` to list pods in the rook namespace. You should be able to see the following: diff --git a/demo/kubernetes/rook-cluster.yaml b/demo/kubernetes/rook-cluster.yaml new file mode 100644 index 000000000..7780a467b --- /dev/null +++ b/demo/kubernetes/rook-cluster.yaml @@ -0,0 +1,8 @@ +apiVersion: rook.io/v1beta1 +kind: Cluster +metadata: + name: my-rook +spec: + namespace: rook + version: dev-2017-02-24-k8s + useAllDevices: false \ No newline at end of file diff --git a/demo/kubernetes/rook-operator.yaml b/demo/kubernetes/rook-operator.yaml index 7bdd4e262..9ed7c46fc 100644 --- a/demo/kubernetes/rook-operator.yaml +++ b/demo/kubernetes/rook-operator.yaml @@ -2,7 +2,6 @@ apiVersion: extensions/v1beta1 kind: Deployment metadata: name: rook-operator - namespace: rook spec: replicas: 1 template: @@ -12,5 +11,9 @@ spec: spec: containers: - name: rook-operator - image: quay.io/rook/rook-operator:dev-2017-02-17-k8s - args: ["--container-version=dev-2017-02-17-k8s"] + image: quay.io/rook/rook-operator:dev-2017-02-24-k8s + env: + - name: ROOK_OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace diff --git a/pkg/operator/cluster.go b/pkg/operator/cluster.go new file mode 100644 index 000000000..4c17d4f86 --- /dev/null +++ b/pkg/operator/cluster.go @@ -0,0 +1,169 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Some of the code below came from https://github.com/coreos/etcd-operator +which also has the apache 2.0 license. +*/ +package operator + +import ( + "fmt" + + "github.com/rook/rook/pkg/cephmgr/client" + cephmon "github.com/rook/rook/pkg/cephmgr/mon" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/api" + "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/operator/mon" + "github.com/rook/rook/pkg/operator/osd" + "github.com/rook/rook/pkg/operator/rgw" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/pkg/api/v1" +) + +type Cluster struct { + Namespace string + factory client.ConnectionFactory + clientset *kubernetes.Clientset + Metadata v1.ObjectMeta `json:"metadata,omitempty"` + Spec ClusterSpec `json:"spec"` +} + +type ClusterSpec struct { + // The namespace where the the rook resources will all be created. + Namespace string `json:"namespace"` + + // Version is the expected version of the rook container to run in the cluster. + // The rook-operator will eventually make the rook cluster version + // equal to the expected version. + Version string `json:"version"` + + // Paused is to pause the control of the operator for the rook cluster. + Paused bool `json:"paused,omitempty"` + + // Whether to consume all the storage devices found on a machine + UseAllDevices bool `json:"useAllDevices"` +} + +func newCluster(namespace, version string, useAllDevices bool, factory client.ConnectionFactory, clientset *kubernetes.Clientset) *Cluster { + c := &Cluster{ + Namespace: namespace, + factory: factory, + clientset: clientset, + } + c.Spec.Version = version + c.Spec.UseAllDevices = useAllDevices + return c +} + +func (c *Cluster) CreateInstance() error { + + // Create the namespace if not already created + ns := &v1.Namespace{ObjectMeta: v1.ObjectMeta{Name: c.Namespace}} + _, err := c.clientset.Namespaces().Create(ns) + if err != nil { + if !k8sutil.IsKubernetesResourceAlreadyExistError(err) { + return fmt.Errorf("failed to create namespace %s. %+v", c.Namespace, err) + } + } + + // Start the mon pods + m := mon.New(c.Namespace, c.factory, c.Spec.Version) + cluster, err := m.Start(c.clientset) + if err != nil { + return fmt.Errorf("failed to start the mons. %+v", err) + } + + a := api.New(c.Namespace, c.Spec.Version) + err = a.Start(c.clientset, cluster) + if err != nil { + return fmt.Errorf("failed to start the REST api. %+v", err) + } + + // Start the OSDs + osds := osd.New(c.Namespace, c.Spec.Version, c.Spec.UseAllDevices) + err = osds.Start(c.clientset, cluster) + if err != nil { + return fmt.Errorf("failed to start the osds. %+v", err) + } + + // Start the object store + r := rgw.New(c.Namespace, c.Spec.Version, c.factory) + err = r.Start(c.clientset, cluster) + if err != nil { + return fmt.Errorf("failed to start rgw. %+v", err) + } + + err = c.createClientAccess(cluster) + if err != nil { + return fmt.Errorf("failed to create client access. %+v", err) + } + + logger.Infof("Done creating rook instance in namespace %s", c.Namespace) + return nil +} + +func (c *Cluster) createClientAccess(clusterInfo *cephmon.ClusterInfo) error { + context := &clusterd.Context{} + conn, err := cephmon.ConnectToClusterAsAdmin(context, c.factory, clusterInfo) + if err != nil { + return fmt.Errorf("failed to connect to cluster: %+v", err) + } + defer conn.Shutdown() + + // create the default rook pool + pool := client.CephStoragePoolDetails{Name: defaultPool} + _, err = client.CreatePool(conn, pool) + if err != nil { + return fmt.Errorf("failed to create default rook pool: %+v", err) + } + + // create a user for rbd clients + username := "client.rook-rbd-user" + access := []string{"osd", "allow rwx", "mon", "allow r"} + + // get-or-create-key for the user account + rbdKey, err := client.AuthGetOrCreateKey(conn, username, access) + if err != nil { + return fmt.Errorf("failed to get or create auth key for %s. %+v", username, err) + } + + // store the secret for the rbd user in the default namespace + secrets := map[string]string{ + "key": rbdKey, + } + secret := &v1.Secret{ + ObjectMeta: v1.ObjectMeta{Name: "rook-rbd-user"}, + StringData: secrets, + Type: k8sutil.RbdType, + } + _, err = c.clientset.Secrets(k8sutil.DefaultNamespace).Create(secret) + if err != nil { + if !k8sutil.IsKubernetesResourceAlreadyExistError(err) { + return fmt.Errorf("failed to save rook-rbd-user secret. %+v", err) + } + + // update the secret in case we have a new cluster + _, err = c.clientset.Secrets(k8sutil.DefaultNamespace).Update(secret) + if err != nil { + return fmt.Errorf("failed to update rook-rbd-user secret. %+v", err) + } + logger.Infof("updated existing rook-rbd-user secret") + } else { + logger.Infof("saved rook-rbd-user secret") + } + + return nil +} diff --git a/pkg/operator/cluster_list.go b/pkg/operator/cluster_list.go new file mode 100644 index 000000000..4ee1cbe9a --- /dev/null +++ b/pkg/operator/cluster_list.go @@ -0,0 +1,66 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Some of the code below came from https://github.com/coreos/etcd-operator +which also has the apache 2.0 license. +*/ +package operator + +import ( + "encoding/json" + + "k8s.io/client-go/pkg/api/unversioned" +) + +// ClusterList is a list of rook clusters. +type ClusterList struct { + unversioned.TypeMeta `json:",inline"` + // Standard list metadata + // More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata + Metadata unversioned.ListMeta `json:"metadata,omitempty"` + // Items is a list of third party objects + Items []Cluster `json:"items"` +} + +// There is known issue with TPR in client-go: +// https://github.com/kubernetes/client-go/issues/8 +// Workarounds: +// - We include `Metadata` field in object explicitly. +// - we have the code below to work around a known problem with third-party resources and ugorji. + +type ClusterListCopy ClusterList +type ClusterCopy Cluster + +func (c *Cluster) UnmarshalJSON(data []byte) error { + tmp := ClusterCopy{} + err := json.Unmarshal(data, &tmp) + if err != nil { + return err + } + tmp2 := Cluster(tmp) + *c = tmp2 + return nil +} + +func (cl *ClusterList) UnmarshalJSON(data []byte) error { + tmp := ClusterListCopy{} + err := json.Unmarshal(data, &tmp) + if err != nil { + return err + } + tmp2 := ClusterList(tmp) + *cl = tmp2 + return nil +} diff --git a/pkg/operator/event.go b/pkg/operator/event.go new file mode 100644 index 000000000..4db5a44ba --- /dev/null +++ b/pkg/operator/event.go @@ -0,0 +1,68 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Some of the code below came from https://github.com/coreos/etcd-operator +which also has the apache 2.0 license. +*/ +package operator + +import ( + "encoding/json" + "fmt" + "io" + + "k8s.io/client-go/pkg/api/unversioned" + kwatch "k8s.io/client-go/pkg/watch" +) + +type Event struct { + Type kwatch.EventType + Object *Cluster +} + +type rawEvent struct { + Type kwatch.EventType + Object json.RawMessage +} + +func pollEvent(decoder *json.Decoder) (*Event, *unversioned.Status, error) { + re := &rawEvent{} + err := decoder.Decode(re) + if err != nil { + if err == io.EOF { + return nil, nil, err + } + return nil, nil, fmt.Errorf("fail to decode raw event from apiserver (%v)", err) + } + + if re.Type == kwatch.Error { + status := &unversioned.Status{} + err = json.Unmarshal(re.Object, status) + if err != nil { + return nil, nil, fmt.Errorf("fail to decode (%s) into unversioned.Status (%v)", re.Object, err) + } + return nil, status, nil + } + + ev := &Event{ + Type: re.Type, + Object: &Cluster{}, + } + err = json.Unmarshal(re.Object, ev.Object) + if err != nil { + return nil, nil, fmt.Errorf("fail to unmarshal Cluster object from data (%s): %v", re.Object, err) + } + return ev, nil, nil +} diff --git a/pkg/operator/k8sutil/k8sutil.go b/pkg/operator/k8sutil/k8sutil.go index 7dc2c5b4f..a02918938 100644 --- a/pkg/operator/k8sutil/k8sutil.go +++ b/pkg/operator/k8sutil/k8sutil.go @@ -19,7 +19,9 @@ which also has the apache 2.0 license. package k8sutil import ( + "fmt" "net/http" + "time" apierrors "k8s.io/client-go/pkg/api/errors" "k8s.io/client-go/pkg/api/unversioned" @@ -34,6 +36,8 @@ const ( RbdType = "kubernetes.io/rbd" ) +type ConditionFunc func() (bool, error) + func IsKubernetesResourceAlreadyExistError(err error) bool { se, ok := err.(*apierrors.StatusError) if !ok { @@ -55,3 +59,26 @@ func IsKubernetesResourceNotFoundError(err error) bool { } return false } + +// Retry retries f every interval until after maxRetries. +// The interval won't be affected by how long f takes. +// For example, if interval is 3s, f takes 1s, another f will be called 2s later. +// However, if f takes longer than interval, it will be delayed. +func Retry(interval time.Duration, maxRetries int, f ConditionFunc) error { + tick := time.NewTicker(interval) + defer tick.Stop() + + for i := 0; i < maxRetries; i++ { + ok, err := f() + if err != nil { + return fmt.Errorf("failed on retry %d. %+v", i, err) + } + if ok { + return nil + } + if i < maxRetries-1 { + <-tick.C + } + } + return fmt.Errorf("failed after max retries %d.", maxRetries) +} diff --git a/pkg/operator/k8sutil/timer.go b/pkg/operator/k8sutil/timer.go new file mode 100644 index 000000000..fefd7244d --- /dev/null +++ b/pkg/operator/k8sutil/timer.go @@ -0,0 +1,49 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Some of the code below came from https://github.com/coreos/etcd-operator +which also has the apache 2.0 license. +*/ +package k8sutil + +import "time" + +// panicTimer panics when it reaches the given duration. +type PanicTimer struct { + d time.Duration + msg string + t *time.Timer +} + +func NewPanicTimer(d time.Duration, msg string) *PanicTimer { + return &PanicTimer{ + d: d, + msg: msg, + } +} + +func (pt *PanicTimer) Start() { + pt.t = time.AfterFunc(pt.d, func() { + panic(pt.msg) + }) +} + +// stop stops the timer and resets the elapsed duration. +func (pt *PanicTimer) Stop() { + if pt.t != nil { + pt.t.Stop() + pt.t = nil + } +} diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index c92813a1f..b2bf876a5 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -12,25 +12,40 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +Some of the code below came from https://github.com/coreos/etcd-operator +which also has the apache 2.0 license. */ package operator import ( + "encoding/json" + "errors" "fmt" + "io" + "net/http" "sync" "time" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/pkg/api/v1" + + "k8s.io/client-go/rest" "github.com/rook/rook/pkg/cephmgr/client" - cephmon "github.com/rook/rook/pkg/cephmgr/mon" - "github.com/rook/rook/pkg/clusterd" - "github.com/rook/rook/pkg/operator/api" "github.com/rook/rook/pkg/operator/k8sutil" - "github.com/rook/rook/pkg/operator/mon" - "github.com/rook/rook/pkg/operator/osd" - "github.com/rook/rook/pkg/operator/rgw" + "k8s.io/client-go/pkg/api" + "k8s.io/client-go/pkg/api/unversioned" + "k8s.io/client-go/pkg/runtime" + "k8s.io/client-go/pkg/runtime/serializer" + kwatch "k8s.io/client-go/pkg/watch" +) + +const ( + initRetryDelay = 10 * time.Second +) + +var ( + ErrVersionOutdated = errors.New("requested version is outdated in apiserver") ) const ( @@ -38,114 +53,275 @@ const ( ) type Operator struct { - Namespace string - MasterHost string - containerVersion string - clientset *kubernetes.Clientset - waitCluster sync.WaitGroup - factory client.ConnectionFactory - useAllDevices bool + Namespace string + MasterHost string + clientset *kubernetes.Clientset + waitCluster sync.WaitGroup + factory client.ConnectionFactory + stopChMap map[string]chan struct{} + clusters map[string]*Cluster + kubeHttpCli *http.Client + // Kubernetes resource version of the clusters + clusterRVs map[string]string } -func New(namespace string, factory client.ConnectionFactory, clientset *kubernetes.Clientset, containerVersion string, useAllDevices bool) *Operator { +func New(host, namespace string, factory client.ConnectionFactory, clientset *kubernetes.Clientset) *Operator { return &Operator{ - Namespace: namespace, - factory: factory, - clientset: clientset, - containerVersion: containerVersion, - useAllDevices: useAllDevices, + Namespace: namespace, + MasterHost: host, + factory: factory, + clientset: clientset, + clusters: make(map[string]*Cluster), + clusterRVs: make(map[string]string), + stopChMap: map[string]chan struct{}{}, } } func (o *Operator) Run() error { - - // Start the mon pods - m := mon.New(o.Namespace, o.factory, o.containerVersion) - cluster, err := m.Start(o.clientset) - if err != nil { - return fmt.Errorf("failed to start the mons. %+v", err) + var watchVersion string + var err error + for { + watchVersion, err = o.initResources() + if err == nil { + break + } + logger.Errorf("failed to create tpr. %+v. retrying...", err) + <-time.After(initRetryDelay) } - a := api.New(o.Namespace, o.containerVersion) - err = a.Start(o.clientset, cluster) - if err != nil { - return fmt.Errorf("failed to start the REST api. %+v", err) - } - - // Start the OSDs - osds := osd.New(o.Namespace, o.containerVersion, o.useAllDevices) - err = osds.Start(o.clientset, cluster) - if err != nil { - return fmt.Errorf("failed to start the osds. %+v", err) - } - - // Start the object store - r := rgw.New(o.Namespace, o.containerVersion, o.factory) - err = r.Start(o.clientset, cluster) - if err != nil { - return fmt.Errorf("failed to start rgw. %+v", err) - } - - err = o.createClientAccess(cluster) - if err != nil { - return fmt.Errorf("failed to create client access. %+v", err) - } - - logger.Infof("DONE!") - <-time.After(1000000 * time.Second) - - return nil + // watch for changes to the rook clusters + return o.watchTPR(watchVersion) } -func (o *Operator) createClientAccess(clusterInfo *cephmon.ClusterInfo) error { - context := &clusterd.Context{} - conn, err := cephmon.ConnectToClusterAsAdmin(context, o.factory, clusterInfo) +func (o *Operator) initResources() (string, error) { + httpCli, err := newHttpClient() if err != nil { - return fmt.Errorf("failed to connect to cluster: %+v", err) + return "", fmt.Errorf("failed to get tpr client. %+v", err) } - defer conn.Shutdown() + o.kubeHttpCli = httpCli.Client - // create the default rook pool - pool := client.CephStoragePoolDetails{Name: defaultPool} - _, err = client.CreatePool(conn, pool) + err = o.createTPR() if err != nil { - return fmt.Errorf("failed to create default rook pool: %+v", err) - } - - // create a user for rbd clients - username := "client.rook-rbd-user" - access := []string{"osd", "allow rwx", "mon", "allow r"} - - // get-or-create-key for the user account - rbdKey, err := client.AuthGetOrCreateKey(conn, username, access) - if err != nil { - return fmt.Errorf("failed to get or create auth key for %s. %+v", username, err) - } - - // store the secret for the rbd user in the default namespace - secrets := map[string]string{ - "key": rbdKey, - } - secret := &v1.Secret{ - ObjectMeta: v1.ObjectMeta{Name: "rook-rbd-user"}, - StringData: secrets, - Type: k8sutil.RbdType, - } - _, err = o.clientset.Secrets(k8sutil.DefaultNamespace).Create(secret) - if err != nil { - if !k8sutil.IsKubernetesResourceAlreadyExistError(err) { - return fmt.Errorf("failed to save rook-rbd-user secret. %+v", err) + if k8sutil.IsKubernetesResourceAlreadyExistError(err) { + // TPR has been initialized before. We need to recover existing cluster. + watchVersion, err := o.findAllClusters() + if err != nil { + return "", fmt.Errorf("failed to find clusters. %+v", err) + } + return watchVersion, nil } - - // update the secret in case we have a new cluster - _, err = o.clientset.Secrets(k8sutil.DefaultNamespace).Update(secret) - if err != nil { - return fmt.Errorf("failed to update rook-rbd-user secret. %+v", err) - } - logger.Infof("updated existing rook-rbd-user secret") - } else { - logger.Infof("saved rook-rbd-user secret") } - return nil + return "0", nil +} + +func (o *Operator) watchTPR(watchVersion string) error { + logger.Infof("start watching rook tpr: %s", watchVersion) + defer func() { + for _, stop := range o.stopChMap { + close(stop) + } + o.waitCluster.Wait() + }() + + eventCh, errCh := o.watch(watchVersion) + + go func() { + pt := k8sutil.NewPanicTimer(time.Minute, "unexpected long blocking (> 1 Minute) when handling cluster event") + + for event := range eventCh { + pt.Start() + + c := event.Object + + switch event.Type { + case kwatch.Added: + ns := c.Spec.Namespace + if ns == "" { + logger.Errorf("missing namespace attribute in rook spec") + continue + } + + if len(o.stopChMap) > 0 { + logger.Warningf("only a single rook cluster is currently supported. Ignoring cluster for namespace %s", ns) + } else { + newCluster := newCluster(ns, c.Spec.Version, c.Spec.UseAllDevices, o.factory, o.clientset) + stopCh := make(chan struct{}) + logger.Infof("starting cluster. %+v", newCluster) + + o.stopChMap[ns] = stopCh + o.clusters[ns] = newCluster + o.clusterRVs[ns] = c.Metadata.ResourceVersion + err := newCluster.CreateInstance() + if err != nil { + logger.Errorf("failed to create cluster %s. %+v", ns, err) + } + } + + case kwatch.Modified: + logger.Infof("modifying a cluster not implemented") + + case kwatch.Deleted: + logger.Infof("deleting a cluster not implemented") + } + + pt.Stop() + } + }() + return <-errCh + +} + +func (o *Operator) findAllClusters() (string, error) { + logger.Info("finding existing clusters...") + clusterList, err := getClusterList(o.clientset.CoreV1().RESTClient(), o.Namespace) + if err != nil { + return "", err + } + + for i := range clusterList.Items { + c := clusterList.Items[i] + + stopCh := make(chan struct{}) + ns := c.Spec.Namespace + newCluster := newCluster(ns, c.Spec.Version, c.Spec.UseAllDevices, o.factory, o.clientset) + o.stopChMap[ns] = stopCh + o.clusters[ns] = newCluster + o.clusterRVs[ns] = c.Metadata.ResourceVersion + } + + return clusterList.Metadata.ResourceVersion, nil +} + +// watch creates a go routine, and watches the cluster.rook kind resources from +// the given watch version. It emits events on the resources through the returned +// event chan. Errors will be reported through the returned error chan. The go routine +// exits on any error. +func (o *Operator) watch(watchVersion string) (<-chan *Event, <-chan error) { + eventCh := make(chan *Event) + // On unexpected error case, the operator should exit + errCh := make(chan error, 1) + + go func() { + defer close(eventCh) + + for { + resp, err := watchClusters(o.MasterHost, o.Namespace, o.kubeHttpCli, watchVersion) + if err != nil { + errCh <- err + return + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + errCh <- errors.New("invalid status code: " + resp.Status) + return + } + + logger.Infof("start watching at %v", watchVersion) + + decoder := json.NewDecoder(resp.Body) + for { + ev, st, err := pollEvent(decoder) + if err != nil { + if err == io.EOF { // apiserver will close stream periodically + logger.Debug("apiserver closed stream") + break + } + + logger.Errorf("received invalid event from API server: %v", err) + errCh <- err + return + } + + if st != nil { + resp.Body.Close() + + if st.Code == http.StatusGone { + // event history is outdated. + // if nothing has changed, we can go back to watch again. + clusterList, err := getClusterList(o.clientset.CoreV1().RESTClient(), o.Namespace) + if err == nil && !o.isClustersCacheStale(clusterList.Items) { + watchVersion = clusterList.Metadata.ResourceVersion + break + } + + // if anything has changed (or error on relist), we have to rebuild the state. + // go to recovery path + errCh <- ErrVersionOutdated + return + } + + logger.Errorf("unexpected status response from API server: %v", st.Message) + break + } + + logger.Debugf("rook cluster event: %+v", ev) + + watchVersion = ev.Object.Metadata.ResourceVersion + eventCh <- ev + } + + resp.Body.Close() + } + }() + + return eventCh, errCh +} + +func (o *Operator) isClustersCacheStale(currentClusters []Cluster) bool { + if len(o.clusterRVs) != len(currentClusters) { + return true + } + + for _, cc := range currentClusters { + rv, ok := o.clusterRVs[cc.Metadata.Name] + if !ok || rv != cc.Metadata.ResourceVersion { + return true + } + } + + return false +} + +func watchClusters(host, ns string, httpClient *http.Client, resourceVersion string) (*http.Response, error) { + return httpClient.Get(fmt.Sprintf("%s/apis/%s/%s/namespaces/%s/clusters?watch=true&resourceVersion=%s", + host, tprGroup, tprVersion, ns, resourceVersion)) +} + +func getClusterList(restcli rest.Interface, ns string) (*ClusterList, error) { + b, err := restcli.Get().RequestURI(listClustersURI(ns)).DoRaw() + if err != nil { + return nil, err + } + + clusters := &ClusterList{} + if err := json.Unmarshal(b, clusters); err != nil { + return nil, err + } + return clusters, nil +} + +func listClustersURI(ns string) string { + return fmt.Sprintf("/apis/%s/%s/namespaces/%s/clusters", tprGroup, tprVersion, ns) +} + +func newHttpClient() (*rest.RESTClient, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + + config.GroupVersion = &unversioned.GroupVersion{ + Group: tprGroup, + Version: tprVersion, + } + config.APIPath = "/apis" + config.ContentType = runtime.ContentTypeJSON + config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: api.Codecs} + + restcli, err := rest.RESTClientFor(config) + if err != nil { + return nil, err + } + return restcli, nil } diff --git a/pkg/operator/operator_test.go b/pkg/operator/operator_test.go index 4773f9685..146ed0984 100644 --- a/pkg/operator/operator_test.go +++ b/pkg/operator/operator_test.go @@ -18,5 +18,5 @@ package operator import "testing" func TestOperator(t *testing.T) { - New("foo", nil, nil, "version", true) + New("foo", "rook", nil, nil) } diff --git a/pkg/operator/tpr.go b/pkg/operator/tpr.go new file mode 100644 index 000000000..a77255a70 --- /dev/null +++ b/pkg/operator/tpr.go @@ -0,0 +1,75 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Some of the code was modified from https://github.com/coreos/etcd-operator +which also has the apache 2.0 license. +*/ +package operator + +import ( + "fmt" + "time" + + "github.com/rook/rook/pkg/operator/k8sutil" + + "k8s.io/client-go/pkg/api/errors" + "k8s.io/client-go/pkg/api/v1" + "k8s.io/client-go/pkg/apis/extensions/v1beta1" + "k8s.io/client-go/rest" +) + +const ( + tprKind = "cluster" + tprGroup = "rook.io" + tprVersion = "v1beta1" + tprDescription = "Managed rook clusters" +) + +func tprName() string { + return fmt.Sprintf("%s.%s", tprKind, tprGroup) +} + +func (o *Operator) createTPR() error { + logger.Info("creating rook TPR") + tpr := &v1beta1.ThirdPartyResource{ + ObjectMeta: v1.ObjectMeta{ + Name: tprName(), + }, + Versions: []v1beta1.APIVersion{ + {Name: tprVersion}, + }, + Description: tprDescription, + } + _, err := o.clientset.ExtensionsV1beta1().ThirdPartyResources().Create(tpr) + if err != nil { + return fmt.Errorf("failed to create rook third party resources. %+v", err) + } + + return o.waitForTPRInit(o.clientset.CoreV1().RESTClient(), 3*time.Second, 90*time.Second, k8sutil.Namespace) +} + +func (o *Operator) waitForTPRInit(restcli rest.Interface, interval, timeout time.Duration, ns string) error { + uri := fmt.Sprintf("/apis/%s/%s/namespaces/%s/clusters", tprGroup, tprVersion, ns) + return k8sutil.Retry(interval, int(timeout/interval), func() (bool, error) { + _, err := restcli.Get().RequestURI(uri).DoRaw() + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil + }) +}