use third party resources to trigger cluster creation
This commit is contained in:
1 parent
1573bdb1dd
commit
34b6ad32d7
13 files changed
+757
-116
No files matched your search
+10
-10
@@ -23,7 +23,6 @@ import (
|
||||
"github.com/rook/rook/pkg/cephmgr/cephd"
|
||||
"github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/operator"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/flags"
|
||||
"github.com/spf13/cobra"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
@@ -46,8 +45,8 @@ type config struct {
|
||||
dataDir string
|
||||
cephConfigOverride string
|
||||
clusterInfo mon.ClusterInfo
|
||||
namespace string
|
||||
monEndpoints string
|
||||
useAllDevices bool
|
||||
}
|
||||
|
||||
var logLevelRaw string
|
||||
@@ -75,7 +74,7 @@ func init() {
|
||||
rootCmd.PersistentFlags().StringVar(&cfg.dataDir, "data-dir", "/var/lib/rook", "directory for storing configuration")
|
||||
rootCmd.PersistentFlags().StringVar(&logLevelRaw, "log-level", "INFO", "logging level for logging/tracing output (valid values: CRITICAL,ERROR,WARNING,NOTICE,INFO,DEBUG,TRACE)")
|
||||
|
||||
rootCmd.Flags().BoolVar(&cfg.useAllDevices, "use-all-devices", false, "true to use all storage devices, false to require local device selection")
|
||||
rootCmd.Flags().StringVar(&cfg.namespace, "namespace", "", "the namespace in which the operator is running (required)")
|
||||
|
||||
// load the environment variables
|
||||
flags.SetFlagsFromEnv(rootCmd.Flags(), "ROOK_OPERATOR")
|
||||
@@ -86,20 +85,20 @@ func init() {
|
||||
|
||||
func startOperator(cmd *cobra.Command, args []string) error {
|
||||
// verify required flags
|
||||
if err := flags.VerifyRequiredFlags(cmd, []string{}); err != nil {
|
||||
if err := flags.VerifyRequiredFlags(cmd, []string{"namespace"}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
setLogLevel()
|
||||
|
||||
clientset, err := getClientset()
|
||||
host, clientset, err := getClientset()
|
||||
if err != nil {
|
||||
fmt.Printf("failed to get k8s client. %+v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
logger.Infof("starting operator. containerVersion=%s", cfg.containerVersion)
|
||||
op := operator.New(k8sutil.Namespace, cephd.New(), clientset, cfg.containerVersion, cfg.useAllDevices)
|
||||
logger.Infof("starting operator in namespace %s", cfg.namespace)
|
||||
op := operator.New(host, cfg.namespace, cephd.New(), clientset)
|
||||
err = op.Run()
|
||||
if err != nil {
|
||||
fmt.Printf("failed to run operator. %+v\n", err)
|
||||
@@ -119,12 +118,13 @@ func setLogLevel() {
|
||||
capnslog.SetGlobalLogLevel(cfg.logLevel)
|
||||
}
|
||||
|
||||
func getClientset() (*kubernetes.Clientset, error) {
|
||||
func getClientset() (string, *kubernetes.Clientset, error) {
|
||||
// create the k8s client
|
||||
config, err := rest.InClusterConfig()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get k8s config. %+v", err)
|
||||
return "", nil, fmt.Errorf("failed to get k8s config. %+v", err)
|
||||
}
|
||||
|
||||
return kubernetes.NewForConfig(config)
|
||||
c, err := kubernetes.NewForConfig(config)
|
||||
return config.Host, c, err
|
||||
}
|
||||
@@ -55,7 +55,7 @@ func startAPI(cmd *cobra.Command, args []string) error {
|
||||
|
||||
setLogLevel()
|
||||
|
||||
clientset, err := getClientset()
|
||||
_, clientset, err := getClientset()
|
||||
if err != nil {
|
||||
fmt.Printf("failed to init k8s client. %+v\n", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -14,12 +14,12 @@ Note that we are striving for even more smooth integration with Kubernetes in th
|
||||
|
||||
### Deploy Rook
|
||||
|
||||
With your Kubernetes cluster running, Rook can be setup and deployed by simply deploying the [rook-operator](rook-operator.yaml) deployment manifest.
|
||||
With your Kubernetes cluster running, Rook can be setup and deployed by simply creating the [rook-operator](rook-operator.yaml) deployment and creating a [rook cluster](rook-cluster.yaml).
|
||||
|
||||
```
|
||||
cd demo/kubernetes
|
||||
kubectl create namespace rook
|
||||
kubectl create -f rook-operator.yaml
|
||||
kubectl create -f rook-cluster.yaml
|
||||
```
|
||||
|
||||
Use `kubectl` to list pods in the rook namespace. You should be able to see the following:
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
apiVersion: rook.io/v1beta1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: my-rook
|
||||
spec:
|
||||
namespace: rook
|
||||
version: dev-2017-02-24-k8s
|
||||
useAllDevices: false
|
||||
@@ -2,7 +2,6 @@ apiVersion: extensions/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: rook-operator
|
||||
namespace: rook
|
||||
spec:
|
||||
replicas: 1
|
||||
template:
|
||||
@@ -12,5 +11,9 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: rook-operator
|
||||
image: quay.io/rook/rook-operator:dev-2017-02-17-k8s
|
||||
args: ["--container-version=dev-2017-02-17-k8s"]
|
||||
image: quay.io/rook/rook-operator:dev-2017-02-24-k8s
|
||||
env:
|
||||
- name: ROOK_OPERATOR_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
@@ -0,0 +1,169 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
Some of the code below came from https://github.com/coreos/etcd-operator
|
||||
which also has the apache 2.0 license.
|
||||
*/
|
||||
package operator
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/rook/rook/pkg/cephmgr/client"
|
||||
cephmon "github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/api"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/operator/mon"
|
||||
"github.com/rook/rook/pkg/operator/osd"
|
||||
"github.com/rook/rook/pkg/operator/rgw"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/pkg/api/v1"
|
||||
)
|
||||
|
||||
type Cluster struct {
|
||||
Namespace string
|
||||
factory client.ConnectionFactory
|
||||
clientset *kubernetes.Clientset
|
||||
Metadata v1.ObjectMeta `json:"metadata,omitempty"`
|
||||
Spec ClusterSpec `json:"spec"`
|
||||
}
|
||||
|
||||
type ClusterSpec struct {
|
||||
// The namespace where the the rook resources will all be created.
|
||||
Namespace string `json:"namespace"`
|
||||
|
||||
// Version is the expected version of the rook container to run in the cluster.
|
||||
// The rook-operator will eventually make the rook cluster version
|
||||
// equal to the expected version.
|
||||
Version string `json:"version"`
|
||||
|
||||
// Paused is to pause the control of the operator for the rook cluster.
|
||||
Paused bool `json:"paused,omitempty"`
|
||||
|
||||
// Whether to consume all the storage devices found on a machine
|
||||
UseAllDevices bool `json:"useAllDevices"`
|
||||
}
|
||||
|
||||
func newCluster(namespace, version string, useAllDevices bool, factory client.ConnectionFactory, clientset *kubernetes.Clientset) *Cluster {
|
||||
c := &Cluster{
|
||||
Namespace: namespace,
|
||||
factory: factory,
|
||||
clientset: clientset,
|
||||
}
|
||||
c.Spec.Version = version
|
||||
c.Spec.UseAllDevices = useAllDevices
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *Cluster) CreateInstance() error {
|
||||
|
||||
// Create the namespace if not already created
|
||||
ns := &v1.Namespace{ObjectMeta: v1.ObjectMeta{Name: c.Namespace}}
|
||||
_, err := c.clientset.Namespaces().Create(ns)
|
||||
if err != nil {
|
||||
if !k8sutil.IsKubernetesResourceAlreadyExistError(err) {
|
||||
return fmt.Errorf("failed to create namespace %s. %+v", c.Namespace, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Start the mon pods
|
||||
m := mon.New(c.Namespace, c.factory, c.Spec.Version)
|
||||
cluster, err := m.Start(c.clientset)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the mons. %+v", err)
|
||||
}
|
||||
|
||||
a := api.New(c.Namespace, c.Spec.Version)
|
||||
err = a.Start(c.clientset, cluster)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the REST api. %+v", err)
|
||||
}
|
||||
|
||||
// Start the OSDs
|
||||
osds := osd.New(c.Namespace, c.Spec.Version, c.Spec.UseAllDevices)
|
||||
err = osds.Start(c.clientset, cluster)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the osds. %+v", err)
|
||||
}
|
||||
|
||||
// Start the object store
|
||||
r := rgw.New(c.Namespace, c.Spec.Version, c.factory)
|
||||
err = r.Start(c.clientset, cluster)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start rgw. %+v", err)
|
||||
}
|
||||
|
||||
err = c.createClientAccess(cluster)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create client access. %+v", err)
|
||||
}
|
||||
|
||||
logger.Infof("Done creating rook instance in namespace %s", c.Namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) createClientAccess(clusterInfo *cephmon.ClusterInfo) error {
|
||||
context := &clusterd.Context{}
|
||||
conn, err := cephmon.ConnectToClusterAsAdmin(context, c.factory, clusterInfo)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to cluster: %+v", err)
|
||||
}
|
||||
defer conn.Shutdown()
|
||||
|
||||
// create the default rook pool
|
||||
pool := client.CephStoragePoolDetails{Name: defaultPool}
|
||||
_, err = client.CreatePool(conn, pool)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create default rook pool: %+v", err)
|
||||
}
|
||||
|
||||
// create a user for rbd clients
|
||||
username := "client.rook-rbd-user"
|
||||
access := []string{"osd", "allow rwx", "mon", "allow r"}
|
||||
|
||||
// get-or-create-key for the user account
|
||||
rbdKey, err := client.AuthGetOrCreateKey(conn, username, access)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get or create auth key for %s. %+v", username, err)
|
||||
}
|
||||
|
||||
// store the secret for the rbd user in the default namespace
|
||||
secrets := map[string]string{
|
||||
"key": rbdKey,
|
||||
}
|
||||
secret := &v1.Secret{
|
||||
ObjectMeta: v1.ObjectMeta{Name: "rook-rbd-user"},
|
||||
StringData: secrets,
|
||||
Type: k8sutil.RbdType,
|
||||
}
|
||||
_, err = c.clientset.Secrets(k8sutil.DefaultNamespace).Create(secret)
|
||||
if err != nil {
|
||||
if !k8sutil.IsKubernetesResourceAlreadyExistError(err) {
|
||||
return fmt.Errorf("failed to save rook-rbd-user secret. %+v", err)
|
||||
}
|
||||
|
||||
// update the secret in case we have a new cluster
|
||||
_, err = c.clientset.Secrets(k8sutil.DefaultNamespace).Update(secret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update rook-rbd-user secret. %+v", err)
|
||||
}
|
||||
logger.Infof("updated existing rook-rbd-user secret")
|
||||
} else {
|
||||
logger.Infof("saved rook-rbd-user secret")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
Some of the code below came from https://github.com/coreos/etcd-operator
|
||||
which also has the apache 2.0 license.
|
||||
*/
|
||||
package operator
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"k8s.io/client-go/pkg/api/unversioned"
|
||||
)
|
||||
|
||||
// ClusterList is a list of rook clusters.
|
||||
type ClusterList struct {
|
||||
unversioned.TypeMeta `json:",inline"`
|
||||
// Standard list metadata
|
||||
// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
|
||||
Metadata unversioned.ListMeta `json:"metadata,omitempty"`
|
||||
// Items is a list of third party objects
|
||||
Items []Cluster `json:"items"`
|
||||
}
|
||||
|
||||
// There is known issue with TPR in client-go:
|
||||
// https://github.com/kubernetes/client-go/issues/8
|
||||
// Workarounds:
|
||||
// - We include `Metadata` field in object explicitly.
|
||||
// - we have the code below to work around a known problem with third-party resources and ugorji.
|
||||
|
||||
type ClusterListCopy ClusterList
|
||||
type ClusterCopy Cluster
|
||||
|
||||
func (c *Cluster) UnmarshalJSON(data []byte) error {
|
||||
tmp := ClusterCopy{}
|
||||
err := json.Unmarshal(data, &tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp2 := Cluster(tmp)
|
||||
*c = tmp2
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cl *ClusterList) UnmarshalJSON(data []byte) error {
|
||||
tmp := ClusterListCopy{}
|
||||
err := json.Unmarshal(data, &tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp2 := ClusterList(tmp)
|
||||
*cl = tmp2
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
Some of the code below came from https://github.com/coreos/etcd-operator
|
||||
which also has the apache 2.0 license.
|
||||
*/
|
||||
package operator
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"k8s.io/client-go/pkg/api/unversioned"
|
||||
kwatch "k8s.io/client-go/pkg/watch"
|
||||
)
|
||||
|
||||
type Event struct {
|
||||
Type kwatch.EventType
|
||||
Object *Cluster
|
||||
}
|
||||
|
||||
type rawEvent struct {
|
||||
Type kwatch.EventType
|
||||
Object json.RawMessage
|
||||
}
|
||||
|
||||
func pollEvent(decoder *json.Decoder) (*Event, *unversioned.Status, error) {
|
||||
re := &rawEvent{}
|
||||
err := decoder.Decode(re)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return nil, nil, err
|
||||
}
|
||||
return nil, nil, fmt.Errorf("fail to decode raw event from apiserver (%v)", err)
|
||||
}
|
||||
|
||||
if re.Type == kwatch.Error {
|
||||
status := &unversioned.Status{}
|
||||
err = json.Unmarshal(re.Object, status)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("fail to decode (%s) into unversioned.Status (%v)", re.Object, err)
|
||||
}
|
||||
return nil, status, nil
|
||||
}
|
||||
|
||||
ev := &Event{
|
||||
Type: re.Type,
|
||||
Object: &Cluster{},
|
||||
}
|
||||
err = json.Unmarshal(re.Object, ev.Object)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("fail to unmarshal Cluster object from data (%s): %v", re.Object, err)
|
||||
}
|
||||
return ev, nil, nil
|
||||
}
|
||||
@@ -19,7 +19,9 @@ which also has the apache 2.0 license.
|
||||
package k8sutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
apierrors "k8s.io/client-go/pkg/api/errors"
|
||||
"k8s.io/client-go/pkg/api/unversioned"
|
||||
@@ -34,6 +36,8 @@ const (
|
||||
RbdType = "kubernetes.io/rbd"
|
||||
)
|
||||
|
||||
type ConditionFunc func() (bool, error)
|
||||
|
||||
func IsKubernetesResourceAlreadyExistError(err error) bool {
|
||||
se, ok := err.(*apierrors.StatusError)
|
||||
if !ok {
|
||||
@@ -55,3 +59,26 @@ func IsKubernetesResourceNotFoundError(err error) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Retry retries f every interval until after maxRetries.
|
||||
// The interval won't be affected by how long f takes.
|
||||
// For example, if interval is 3s, f takes 1s, another f will be called 2s later.
|
||||
// However, if f takes longer than interval, it will be delayed.
|
||||
func Retry(interval time.Duration, maxRetries int, f ConditionFunc) error {
|
||||
tick := time.NewTicker(interval)
|
||||
defer tick.Stop()
|
||||
|
||||
for i := 0; i < maxRetries; i++ {
|
||||
ok, err := f()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed on retry %d. %+v", i, err)
|
||||
}
|
||||
if ok {
|
||||
return nil
|
||||
}
|
||||
if i < maxRetries-1 {
|
||||
<-tick.C
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("failed after max retries %d.", maxRetries)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
Some of the code below came from https://github.com/coreos/etcd-operator
|
||||
which also has the apache 2.0 license.
|
||||
*/
|
||||
package k8sutil
|
||||
|
||||
import "time"
|
||||
|
||||
// panicTimer panics when it reaches the given duration.
|
||||
type PanicTimer struct {
|
||||
d time.Duration
|
||||
msg string
|
||||
t *time.Timer
|
||||
}
|
||||
|
||||
func NewPanicTimer(d time.Duration, msg string) *PanicTimer {
|
||||
return &PanicTimer{
|
||||
d: d,
|
||||
msg: msg,
|
||||
}
|
||||
}
|
||||
|
||||
func (pt *PanicTimer) Start() {
|
||||
pt.t = time.AfterFunc(pt.d, func() {
|
||||
panic(pt.msg)
|
||||
})
|
||||
}
|
||||
|
||||
// stop stops the timer and resets the elapsed duration.
|
||||
func (pt *PanicTimer) Stop() {
|
||||
if pt.t != nil {
|
||||
pt.t.Stop()
|
||||
pt.t = nil
|
||||
}
|
||||
}
|
||||
+275
-99
@@ -12,25 +12,40 @@ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
Some of the code below came from https://github.com/coreos/etcd-operator
|
||||
which also has the apache 2.0 license.
|
||||
*/
|
||||
package operator
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/pkg/api/v1"
|
||||
|
||||
"k8s.io/client-go/rest"
|
||||
|
||||
"github.com/rook/rook/pkg/cephmgr/client"
|
||||
cephmon "github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/api"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/operator/mon"
|
||||
"github.com/rook/rook/pkg/operator/osd"
|
||||
"github.com/rook/rook/pkg/operator/rgw"
|
||||
"k8s.io/client-go/pkg/api"
|
||||
"k8s.io/client-go/pkg/api/unversioned"
|
||||
"k8s.io/client-go/pkg/runtime"
|
||||
"k8s.io/client-go/pkg/runtime/serializer"
|
||||
kwatch "k8s.io/client-go/pkg/watch"
|
||||
)
|
||||
|
||||
const (
|
||||
initRetryDelay = 10 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
ErrVersionOutdated = errors.New("requested version is outdated in apiserver")
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -38,114 +53,275 @@ const (
|
||||
)
|
||||
|
||||
type Operator struct {
|
||||
Namespace string
|
||||
MasterHost string
|
||||
containerVersion string
|
||||
clientset *kubernetes.Clientset
|
||||
waitCluster sync.WaitGroup
|
||||
factory client.ConnectionFactory
|
||||
useAllDevices bool
|
||||
Namespace string
|
||||
MasterHost string
|
||||
clientset *kubernetes.Clientset
|
||||
waitCluster sync.WaitGroup
|
||||
factory client.ConnectionFactory
|
||||
stopChMap map[string]chan struct{}
|
||||
clusters map[string]*Cluster
|
||||
kubeHttpCli *http.Client
|
||||
// Kubernetes resource version of the clusters
|
||||
clusterRVs map[string]string
|
||||
}
|
||||
|
||||
func New(namespace string, factory client.ConnectionFactory, clientset *kubernetes.Clientset, containerVersion string, useAllDevices bool) *Operator {
|
||||
func New(host, namespace string, factory client.ConnectionFactory, clientset *kubernetes.Clientset) *Operator {
|
||||
return &Operator{
|
||||
Namespace: namespace,
|
||||
factory: factory,
|
||||
clientset: clientset,
|
||||
containerVersion: containerVersion,
|
||||
useAllDevices: useAllDevices,
|
||||
Namespace: namespace,
|
||||
MasterHost: host,
|
||||
factory: factory,
|
||||
clientset: clientset,
|
||||
clusters: make(map[string]*Cluster),
|
||||
clusterRVs: make(map[string]string),
|
||||
stopChMap: map[string]chan struct{}{},
|
||||
}
|
||||
}
|
||||
|
||||
func (o *Operator) Run() error {
|
||||
|
||||
// Start the mon pods
|
||||
m := mon.New(o.Namespace, o.factory, o.containerVersion)
|
||||
cluster, err := m.Start(o.clientset)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the mons. %+v", err)
|
||||
var watchVersion string
|
||||
var err error
|
||||
for {
|
||||
watchVersion, err = o.initResources()
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
logger.Errorf("failed to create tpr. %+v. retrying...", err)
|
||||
<-time.After(initRetryDelay)
|
||||
}
|
||||
|
||||
a := api.New(o.Namespace, o.containerVersion)
|
||||
err = a.Start(o.clientset, cluster)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the REST api. %+v", err)
|
||||
}
|
||||
|
||||
// Start the OSDs
|
||||
osds := osd.New(o.Namespace, o.containerVersion, o.useAllDevices)
|
||||
err = osds.Start(o.clientset, cluster)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the osds. %+v", err)
|
||||
}
|
||||
|
||||
// Start the object store
|
||||
r := rgw.New(o.Namespace, o.containerVersion, o.factory)
|
||||
err = r.Start(o.clientset, cluster)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start rgw. %+v", err)
|
||||
}
|
||||
|
||||
err = o.createClientAccess(cluster)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create client access. %+v", err)
|
||||
}
|
||||
|
||||
logger.Infof("DONE!")
|
||||
<-time.After(1000000 * time.Second)
|
||||
|
||||
return nil
|
||||
// watch for changes to the rook clusters
|
||||
return o.watchTPR(watchVersion)
|
||||
}
|
||||
|
||||
func (o *Operator) createClientAccess(clusterInfo *cephmon.ClusterInfo) error {
|
||||
context := &clusterd.Context{}
|
||||
conn, err := cephmon.ConnectToClusterAsAdmin(context, o.factory, clusterInfo)
|
||||
func (o *Operator) initResources() (string, error) {
|
||||
httpCli, err := newHttpClient()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to cluster: %+v", err)
|
||||
return "", fmt.Errorf("failed to get tpr client. %+v", err)
|
||||
}
|
||||
defer conn.Shutdown()
|
||||
o.kubeHttpCli = httpCli.Client
|
||||
|
||||
// create the default rook pool
|
||||
pool := client.CephStoragePoolDetails{Name: defaultPool}
|
||||
_, err = client.CreatePool(conn, pool)
|
||||
err = o.createTPR()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create default rook pool: %+v", err)
|
||||
}
|
||||
|
||||
// create a user for rbd clients
|
||||
username := "client.rook-rbd-user"
|
||||
access := []string{"osd", "allow rwx", "mon", "allow r"}
|
||||
|
||||
// get-or-create-key for the user account
|
||||
rbdKey, err := client.AuthGetOrCreateKey(conn, username, access)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get or create auth key for %s. %+v", username, err)
|
||||
}
|
||||
|
||||
// store the secret for the rbd user in the default namespace
|
||||
secrets := map[string]string{
|
||||
"key": rbdKey,
|
||||
}
|
||||
secret := &v1.Secret{
|
||||
ObjectMeta: v1.ObjectMeta{Name: "rook-rbd-user"},
|
||||
StringData: secrets,
|
||||
Type: k8sutil.RbdType,
|
||||
}
|
||||
_, err = o.clientset.Secrets(k8sutil.DefaultNamespace).Create(secret)
|
||||
if err != nil {
|
||||
if !k8sutil.IsKubernetesResourceAlreadyExistError(err) {
|
||||
return fmt.Errorf("failed to save rook-rbd-user secret. %+v", err)
|
||||
if k8sutil.IsKubernetesResourceAlreadyExistError(err) {
|
||||
// TPR has been initialized before. We need to recover existing cluster.
|
||||
watchVersion, err := o.findAllClusters()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to find clusters. %+v", err)
|
||||
}
|
||||
return watchVersion, nil
|
||||
}
|
||||
|
||||
// update the secret in case we have a new cluster
|
||||
_, err = o.clientset.Secrets(k8sutil.DefaultNamespace).Update(secret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update rook-rbd-user secret. %+v", err)
|
||||
}
|
||||
logger.Infof("updated existing rook-rbd-user secret")
|
||||
} else {
|
||||
logger.Infof("saved rook-rbd-user secret")
|
||||
}
|
||||
|
||||
return nil
|
||||
return "0", nil
|
||||
}
|
||||
|
||||
func (o *Operator) watchTPR(watchVersion string) error {
|
||||
logger.Infof("start watching rook tpr: %s", watchVersion)
|
||||
defer func() {
|
||||
for _, stop := range o.stopChMap {
|
||||
close(stop)
|
||||
}
|
||||
o.waitCluster.Wait()
|
||||
}()
|
||||
|
||||
eventCh, errCh := o.watch(watchVersion)
|
||||
|
||||
go func() {
|
||||
pt := k8sutil.NewPanicTimer(time.Minute, "unexpected long blocking (> 1 Minute) when handling cluster event")
|
||||
|
||||
for event := range eventCh {
|
||||
pt.Start()
|
||||
|
||||
c := event.Object
|
||||
|
||||
switch event.Type {
|
||||
case kwatch.Added:
|
||||
ns := c.Spec.Namespace
|
||||
if ns == "" {
|
||||
logger.Errorf("missing namespace attribute in rook spec")
|
||||
continue
|
||||
}
|
||||
|
||||
if len(o.stopChMap) > 0 {
|
||||
logger.Warningf("only a single rook cluster is currently supported. Ignoring cluster for namespace %s", ns)
|
||||
} else {
|
||||
newCluster := newCluster(ns, c.Spec.Version, c.Spec.UseAllDevices, o.factory, o.clientset)
|
||||
stopCh := make(chan struct{})
|
||||
logger.Infof("starting cluster. %+v", newCluster)
|
||||
|
||||
o.stopChMap[ns] = stopCh
|
||||
o.clusters[ns] = newCluster
|
||||
o.clusterRVs[ns] = c.Metadata.ResourceVersion
|
||||
err := newCluster.CreateInstance()
|
||||
if err != nil {
|
||||
logger.Errorf("failed to create cluster %s. %+v", ns, err)
|
||||
}
|
||||
}
|
||||
|
||||
case kwatch.Modified:
|
||||
logger.Infof("modifying a cluster not implemented")
|
||||
|
||||
case kwatch.Deleted:
|
||||
logger.Infof("deleting a cluster not implemented")
|
||||
}
|
||||
|
||||
pt.Stop()
|
||||
}
|
||||
}()
|
||||
return <-errCh
|
||||
|
||||
}
|
||||
|
||||
func (o *Operator) findAllClusters() (string, error) {
|
||||
logger.Info("finding existing clusters...")
|
||||
clusterList, err := getClusterList(o.clientset.CoreV1().RESTClient(), o.Namespace)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for i := range clusterList.Items {
|
||||
c := clusterList.Items[i]
|
||||
|
||||
stopCh := make(chan struct{})
|
||||
ns := c.Spec.Namespace
|
||||
newCluster := newCluster(ns, c.Spec.Version, c.Spec.UseAllDevices, o.factory, o.clientset)
|
||||
o.stopChMap[ns] = stopCh
|
||||
o.clusters[ns] = newCluster
|
||||
o.clusterRVs[ns] = c.Metadata.ResourceVersion
|
||||
}
|
||||
|
||||
return clusterList.Metadata.ResourceVersion, nil
|
||||
}
|
||||
|
||||
// watch creates a go routine, and watches the cluster.rook kind resources from
|
||||
// the given watch version. It emits events on the resources through the returned
|
||||
// event chan. Errors will be reported through the returned error chan. The go routine
|
||||
// exits on any error.
|
||||
func (o *Operator) watch(watchVersion string) (<-chan *Event, <-chan error) {
|
||||
eventCh := make(chan *Event)
|
||||
// On unexpected error case, the operator should exit
|
||||
errCh := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
defer close(eventCh)
|
||||
|
||||
for {
|
||||
resp, err := watchClusters(o.MasterHost, o.Namespace, o.kubeHttpCli, watchVersion)
|
||||
if err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
errCh <- errors.New("invalid status code: " + resp.Status)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof("start watching at %v", watchVersion)
|
||||
|
||||
decoder := json.NewDecoder(resp.Body)
|
||||
for {
|
||||
ev, st, err := pollEvent(decoder)
|
||||
if err != nil {
|
||||
if err == io.EOF { // apiserver will close stream periodically
|
||||
logger.Debug("apiserver closed stream")
|
||||
break
|
||||
}
|
||||
|
||||
logger.Errorf("received invalid event from API server: %v", err)
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
|
||||
if st != nil {
|
||||
resp.Body.Close()
|
||||
|
||||
if st.Code == http.StatusGone {
|
||||
// event history is outdated.
|
||||
// if nothing has changed, we can go back to watch again.
|
||||
clusterList, err := getClusterList(o.clientset.CoreV1().RESTClient(), o.Namespace)
|
||||
if err == nil && !o.isClustersCacheStale(clusterList.Items) {
|
||||
watchVersion = clusterList.Metadata.ResourceVersion
|
||||
break
|
||||
}
|
||||
|
||||
// if anything has changed (or error on relist), we have to rebuild the state.
|
||||
// go to recovery path
|
||||
errCh <- ErrVersionOutdated
|
||||
return
|
||||
}
|
||||
|
||||
logger.Errorf("unexpected status response from API server: %v", st.Message)
|
||||
break
|
||||
}
|
||||
|
||||
logger.Debugf("rook cluster event: %+v", ev)
|
||||
|
||||
watchVersion = ev.Object.Metadata.ResourceVersion
|
||||
eventCh <- ev
|
||||
}
|
||||
|
||||
resp.Body.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
return eventCh, errCh
|
||||
}
|
||||
|
||||
func (o *Operator) isClustersCacheStale(currentClusters []Cluster) bool {
|
||||
if len(o.clusterRVs) != len(currentClusters) {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, cc := range currentClusters {
|
||||
rv, ok := o.clusterRVs[cc.Metadata.Name]
|
||||
if !ok || rv != cc.Metadata.ResourceVersion {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func watchClusters(host, ns string, httpClient *http.Client, resourceVersion string) (*http.Response, error) {
|
||||
return httpClient.Get(fmt.Sprintf("%s/apis/%s/%s/namespaces/%s/clusters?watch=true&resourceVersion=%s",
|
||||
host, tprGroup, tprVersion, ns, resourceVersion))
|
||||
}
|
||||
|
||||
func getClusterList(restcli rest.Interface, ns string) (*ClusterList, error) {
|
||||
b, err := restcli.Get().RequestURI(listClustersURI(ns)).DoRaw()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clusters := &ClusterList{}
|
||||
if err := json.Unmarshal(b, clusters); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return clusters, nil
|
||||
}
|
||||
|
||||
func listClustersURI(ns string) string {
|
||||
return fmt.Sprintf("/apis/%s/%s/namespaces/%s/clusters", tprGroup, tprVersion, ns)
|
||||
}
|
||||
|
||||
func newHttpClient() (*rest.RESTClient, error) {
|
||||
config, err := rest.InClusterConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
config.GroupVersion = &unversioned.GroupVersion{
|
||||
Group: tprGroup,
|
||||
Version: tprVersion,
|
||||
}
|
||||
config.APIPath = "/apis"
|
||||
config.ContentType = runtime.ContentTypeJSON
|
||||
config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: api.Codecs}
|
||||
|
||||
restcli, err := rest.RESTClientFor(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return restcli, nil
|
||||
}
|
||||
@@ -18,5 +18,5 @@ package operator
|
||||
import "testing"
|
||||
|
||||
func TestOperator(t *testing.T) {
|
||||
New("foo", nil, nil, "version", true)
|
||||
New("foo", "rook", nil, nil)
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
Some of the code was modified from https://github.com/coreos/etcd-operator
|
||||
which also has the apache 2.0 license.
|
||||
*/
|
||||
package operator
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
|
||||
"k8s.io/client-go/pkg/api/errors"
|
||||
"k8s.io/client-go/pkg/api/v1"
|
||||
"k8s.io/client-go/pkg/apis/extensions/v1beta1"
|
||||
"k8s.io/client-go/rest"
|
||||
)
|
||||
|
||||
const (
|
||||
tprKind = "cluster"
|
||||
tprGroup = "rook.io"
|
||||
tprVersion = "v1beta1"
|
||||
tprDescription = "Managed rook clusters"
|
||||
)
|
||||
|
||||
func tprName() string {
|
||||
return fmt.Sprintf("%s.%s", tprKind, tprGroup)
|
||||
}
|
||||
|
||||
func (o *Operator) createTPR() error {
|
||||
logger.Info("creating rook TPR")
|
||||
tpr := &v1beta1.ThirdPartyResource{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: tprName(),
|
||||
},
|
||||
Versions: []v1beta1.APIVersion{
|
||||
{Name: tprVersion},
|
||||
},
|
||||
Description: tprDescription,
|
||||
}
|
||||
_, err := o.clientset.ExtensionsV1beta1().ThirdPartyResources().Create(tpr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create rook third party resources. %+v", err)
|
||||
}
|
||||
|
||||
return o.waitForTPRInit(o.clientset.CoreV1().RESTClient(), 3*time.Second, 90*time.Second, k8sutil.Namespace)
|
||||
}
|
||||
|
||||
func (o *Operator) waitForTPRInit(restcli rest.Interface, interval, timeout time.Duration, ns string) error {
|
||||
uri := fmt.Sprintf("/apis/%s/%s/namespaces/%s/clusters", tprGroup, tprVersion, ns)
|
||||
return k8sutil.Retry(interval, int(timeout/interval), func() (bool, error) {
|
||||
_, err := restcli.Get().RequestURI(uri).DoRaw()
|
||||
if err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user