detect if mons are healthy

This commit is contained in:
Travis Nielsen committed 2017-03-23 11:00:26 -07:00
1 parent 175003ed6a
commit 2d1c150539
10 files changed
+206 -90

No files matched your search

+1
View File
@@ -251,6 +251,7 @@ func ConnectToCluster(context *clusterd.Context, factory client.ConnectionFactor
return nil, fmt.Errorf("failed to connect to cluster %s: %+v", cluster.Name, err)
}
logger.Infof("successfully connected to cluster %s", cluster.Name)
return conn, nil
}
+3 -2
View File
@@ -42,7 +42,7 @@ type Leader struct {
}
func NewLeader() *Leader {
return &Leader{waitForQuorum: waitForQuorum}
return &Leader{waitForQuorum: WaitForQuorum}
}
// Apply the desired state to the cluster. The context provides all the information needed to make changes to the service.
@@ -356,7 +356,8 @@ func calculateMonitorCount(nodeCount int) int {
}
}
func waitForQuorum(factory client.ConnectionFactory, context *clusterd.Context, cluster *ClusterInfo) error {
func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context, cluster *ClusterInfo) error {
logger.Infof("waiting for mon quorum...")
// open an admin connection to the cluster
adminConn, err := ConnectToClusterAsAdmin(context, factory, cluster)
@@ -16,13 +16,15 @@ limitations under the License.
Some of the code below came from https://github.com/coreos/etcd-operator
which also has the apache 2.0 license.
*/
package operator
package cluster
import (
"fmt"
"time"
"k8s.io/client-go/pkg/api/errors"
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/pkg/cephmgr/client"
cephmon "github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/clusterd"
@@ -30,41 +32,33 @@ import (
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/operator/mon"
"github.com/rook/rook/pkg/operator/osd"
"github.com/rook/rook/pkg/operator/rgw"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/pkg/api/v1"
)
const (
defaultPool = "rook"
)
var (
healthCheckInterval = 8 * time.Second
logger = capnslog.NewPackageLogger("github.com/rook/rook-operator", "op-cluster")
)
type Cluster struct {
factory client.ConnectionFactory
clientset kubernetes.Interface
Metadata v1.ObjectMeta `json:"metadata,omitempty"`
Spec ClusterSpec `json:"spec"`
Spec `json:"spec"`
dataDir string
mons *mon.Cluster
osds *osd.Cluster
apis *api.Cluster
rgws *rgw.Cluster
}
type ClusterSpec struct {
// The namespace where the the rook resources will all be created.
Namespace string `json:"namespace"`
// Version is the expected version of the rook container to run in the cluster.
// The rook-operator will eventually make the rook cluster version
// equal to the expected version.
Version string `json:"version"`
// Paused is to pause the control of the operator for the rook cluster.
Paused bool `json:"paused,omitempty"`
// Whether to consume all the storage devices found on a machine
UseAllDevices bool `json:"useAllDevices"`
// A regular expression to allow more fine-grained selection of devices on nodes across the cluster
DeviceFilter string `json:"deviceFilter"`
// The path on the host where config and data can be persisted.
DataDirHostPath string `json:"dataDirHostPath"`
}
func newCluster(spec ClusterSpec, factory client.ConnectionFactory, clientset kubernetes.Interface) *Cluster {
func New(spec Spec, factory client.ConnectionFactory, clientset kubernetes.Interface) *Cluster {
return &Cluster{
Spec: spec,
factory: factory,
@@ -85,26 +79,26 @@ func (c *Cluster) CreateInstance() error {
}
// Start the mon pods
m := mon.New(c.clientset, c.factory, c.Spec.Namespace, c.Spec.DataDirHostPath, c.Spec.Version)
cluster, err := m.Start()
c.mons = mon.New(c.clientset, c.factory, c.Spec.Namespace, c.Spec.DataDirHostPath, c.Spec.Version)
clusterInfo, err := c.mons.Start()
if err != nil {
return fmt.Errorf("failed to start the mons. %+v", err)
}
a := api.New(c.clientset, c.Spec.Namespace, c.Spec.Version)
err = a.Start()
c.apis = api.New(c.clientset, c.Spec.Namespace, c.Spec.Version)
err = c.apis.Start()
if err != nil {
return fmt.Errorf("failed to start the REST api. %+v", err)
}
// Start the OSDs
osds := osd.New(c.clientset, c.Spec.Namespace, c.Spec.Version, c.Spec.DeviceFilter, c.Spec.DataDirHostPath, c.Spec.UseAllDevices)
err = osds.Start()
c.osds = osd.New(c.clientset, c.Spec.Namespace, c.Spec.Version, c.Spec.DeviceFilter, c.Spec.DataDirHostPath, c.Spec.UseAllDevices)
err = c.osds.Start()
if err != nil {
return fmt.Errorf("failed to start the osds. %+v", err)
}
err = c.createClientAccess(cluster)
err = c.createClientAccess(clusterInfo)
if err != nil {
return fmt.Errorf("failed to create client access. %+v", err)
}
@@ -113,6 +107,23 @@ func (c *Cluster) CreateInstance() error {
return nil
}
func (c *Cluster) Monitor(stopCh <-chan struct{}) {
for {
select {
case <-stopCh:
logger.Infof("Stopping monitoring of cluster %s", c.Spec.Namespace)
return
case <-time.After(healthCheckInterval):
logger.Infof("checking health of mons")
err := c.mons.CheckHealth()
if err != nil {
logger.Infof("failed to check mon health. %+v", err)
}
}
}
}
func (c *Cluster) createClientAccess(clusterInfo *cephmon.ClusterInfo) error {
context := &clusterd.Context{ConfigDir: c.dataDir}
conn, err := cephmon.ConnectToClusterAsAdmin(context, c.factory, clusterInfo)
@@ -16,7 +16,7 @@ limitations under the License.
Some of the code below came from https://github.com/coreos/etcd-operator
which also has the apache 2.0 license.
*/
package operator
package cluster
import (
"encoding/json"
@@ -16,7 +16,7 @@ limitations under the License.
Some of the code below came from https://github.com/coreos/etcd-operator
which also has the apache 2.0 license.
*/
package operator
package cluster
import (
"fmt"
@@ -39,8 +39,8 @@ func TestCreateSecrets(t *testing.T) {
response := "{\"key\":\"mysecurekey\"}"
return []byte(response), "", nil
}
spec := ClusterSpec{Namespace: "ns", Version: "myversion"}
c := newCluster(spec, factory, clientset)
spec := Spec{Namespace: "ns", Version: "myversion"}
c := New(spec, factory, clientset)
c.dataDir = "/tmp/testdir"
defer os.RemoveAll(c.dataDir)
+41
View File
@@ -0,0 +1,41 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Some of the code below came from https://github.com/coreos/etcd-operator
which also has the apache 2.0 license.
*/
package cluster
type Spec struct {
// The namespace where the the rook resources will all be created.
Namespace string `json:"namespace"`
// Version is the expected version of the rook container to run in the cluster.
// The rook-operator will eventually make the rook cluster version
// equal to the expected version.
Version string `json:"version"`
// Paused is to pause the control of the operator for the rook cluster.
Paused bool `json:"paused,omitempty"`
// Whether to consume all the storage devices found on a machine
UseAllDevices bool `json:"useAllDevices"`
// A regular expression to allow more fine-grained selection of devices on nodes across the cluster
DeviceFilter string `json:"deviceFilter"`
// The path on the host where config and data can be persisted.
DataDirHostPath string `json:"dataDirHostPath"`
}
+4 -2
View File
@@ -23,13 +23,15 @@ import (
"fmt"
"io"
"github.com/rook/rook/pkg/operator/cluster"
"k8s.io/client-go/pkg/api/unversioned"
kwatch "k8s.io/client-go/pkg/watch"
)
type Event struct {
Type kwatch.EventType
Object *Cluster
Object *cluster.Cluster
}
type rawEvent struct {
@@ -58,7 +60,7 @@ func pollEvent(decoder *json.Decoder) (*Event, *unversioned.Status, error) {
ev := &Event{
Type: re.Type,
Object: &Cluster{},
Object: &cluster.Cluster{},
}
err = json.Unmarshal(re.Object, ev.Object)
if err != nil {
+91 -29
View File
@@ -21,6 +21,7 @@ import (
"github.com/rook/rook/pkg/cephmgr/client"
"github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/pkg/api/errors"
@@ -53,6 +54,7 @@ type Cluster struct {
clientset kubernetes.Interface
factory client.ConnectionFactory
retryDelay int
clusterInfo *mon.ClusterInfo
dataDirHostPath string
}
@@ -77,7 +79,7 @@ func New(clientset kubernetes.Interface, factory client.ConnectionFactory, names
func (c *Cluster) Start() (*mon.ClusterInfo, error) {
logger.Infof("start running mons")
clusterInfo, err := c.initClusterInfo()
err := c.initClusterInfo()
if err != nil {
return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err)
}
@@ -86,50 +88,110 @@ func (c *Cluster) Start() (*mon.ClusterInfo, error) {
mons = append(mons, &MonConfig{Name: fmt.Sprintf("mon%d", i), Port: int32(mon.Port)})
}
err = c.startPods(clusterInfo, mons)
err = c.startPods(mons)
if err != nil {
return nil, fmt.Errorf("failed to start mon pods. %+v", err)
}
return clusterInfo, nil
context := &clusterd.Context{ConfigDir: k8sutil.DataDir}
err = mon.WaitForQuorum(c.factory, context, c.clusterInfo)
if err != nil {
return nil, fmt.Errorf("failed to wait for mon quorum. %+v", err)
}
return c.clusterInfo, nil
}
func (c *Cluster) CheckHealth() error {
context := &clusterd.Context{ConfigDir: k8sutil.DataDir}
conn, err := mon.ConnectToClusterAsAdmin(context, c.factory, c.clusterInfo)
if err != nil {
return fmt.Errorf("cannot connect to cluster. %+v", err)
}
defer conn.Shutdown()
status, err := client.GetMonStatus(conn)
if err != nil {
return fmt.Errorf("failed to get mon status. %+v", err)
}
for _, monitor := range status.MonMap.Mons {
inQuorum := monInQuorum(monitor, status.Quorum)
if inQuorum {
logger.Infof("mon %s found in quorum", monitor.Name)
} else {
logger.Infof("mon %s NOT found in quroum. %+v", monitor.Name, status.Quorum)
}
pod, err := c.clientset.CoreV1().Pods(c.Namespace).Get(monitor.Name)
if err != nil {
logger.Infof("Get mon %s pod failed. %+v", monitor.Name, err)
} else {
logger.Infof("mon %s pod status: %+v", monitor.Name, pod.Status.Phase)
}
if !inQuorum {
err = c.failoverMon(monitor.Name)
if err != nil {
logger.Errorf("failed to failover mon %s. %+v", monitor.Name, err)
}
}
}
return nil
}
func monInQuorum(monitor client.MonMapEntry, quorum []int) bool {
for _, rank := range quorum {
if rank == monitor.Rank {
return true
}
}
return false
}
func (c *Cluster) failoverMon(name string) error {
//logger.Infof("Failing over monitor %s", name)
return nil
}
// Retrieve the ceph cluster info if it already exists.
// If a new cluster create new keys.
func (c *Cluster) initClusterInfo() (*mon.ClusterInfo, error) {
func (c *Cluster) initClusterInfo() error {
secrets, err := c.clientset.CoreV1().Secrets(c.Namespace).Get(appName)
if err != nil {
if !errors.IsNotFound(err) {
return nil, fmt.Errorf("failed to get mon secrets. %+v", err)
return fmt.Errorf("failed to get mon secrets. %+v", err)
}
return c.createMonSecretsAndSave()
}
info := &mon.ClusterInfo{
c.clusterInfo = &mon.ClusterInfo{
Name: string(secrets.Data[clusterSecretName]),
FSID: string(secrets.Data[fsidSecretName]),
MonitorSecret: string(secrets.Data[monSecretName]),
AdminSecret: string(secrets.Data[adminSecretName]),
Monitors: map[string]*mon.CephMonitorConfig{},
}
logger.Infof("found existing monitor secrets for cluster %s with fsid %s", info.Name, info.FSID)
return info, nil
logger.Infof("found existing monitor secrets for cluster %s", c.clusterInfo.Name)
return nil
}
func (c *Cluster) createMonSecretsAndSave() (*mon.ClusterInfo, error) {
func (c *Cluster) createMonSecretsAndSave() error {
logger.Infof("creating mon secrets for a new cluster")
info, err := mon.CreateNamedClusterInfo(c.factory, "", c.Namespace)
var err error
c.clusterInfo, err = mon.CreateNamedClusterInfo(c.factory, "", c.Namespace)
if err != nil {
return nil, fmt.Errorf("failed to create mon secrets. %+v", err)
return fmt.Errorf("failed to create mon secrets. %+v", err)
}
// store the secrets for internal usage of the rook pods
secrets := map[string]string{
clusterSecretName: info.Name,
fsidSecretName: info.FSID,
monSecretName: info.MonitorSecret,
adminSecretName: info.AdminSecret,
clusterSecretName: c.clusterInfo.Name,
fsidSecretName: c.clusterInfo.FSID,
monSecretName: c.clusterInfo.MonitorSecret,
adminSecretName: c.clusterInfo.AdminSecret,
}
secret := &v1.Secret{
ObjectMeta: v1.ObjectMeta{Name: appName, Namespace: c.Namespace},
@@ -138,12 +200,12 @@ func (c *Cluster) createMonSecretsAndSave() (*mon.ClusterInfo, error) {
}
_, err = c.clientset.CoreV1().Secrets(c.Namespace).Create(secret)
if err != nil {
return nil, fmt.Errorf("failed to save mon secrets. %+v", err)
return fmt.Errorf("failed to save mon secrets. %+v", err)
}
// store the secret for usage by the storage class
storageClassSecret := map[string]string{
"key": info.AdminSecret,
"key": c.clusterInfo.AdminSecret,
}
secret = &v1.Secret{
ObjectMeta: v1.ObjectMeta{Name: "rook-admin", Namespace: c.Namespace},
@@ -153,17 +215,17 @@ func (c *Cluster) createMonSecretsAndSave() (*mon.ClusterInfo, error) {
_, err = c.clientset.CoreV1().Secrets(c.Namespace).Create(secret)
if err != nil {
if !errors.IsAlreadyExists(err) {
return nil, fmt.Errorf("failed to save rook-admin secret. %+v", err)
return fmt.Errorf("failed to save rook-admin secret. %+v", err)
}
logger.Infof("rook-admin secret already exists")
} else {
logger.Infof("saved rook-admin secret")
}
return info, nil
return nil
}
func (c *Cluster) startPods(clusterInfo *mon.ClusterInfo, mons []*MonConfig) error {
func (c *Cluster) startPods(mons []*MonConfig) error {
// schedule the mons on different nodes if we have enough nodes to be unique
antiAffinity, err := c.getAntiAffinity()
if err != nil {
@@ -176,14 +238,14 @@ func (c *Cluster) startPods(clusterInfo *mon.ClusterInfo, mons []*MonConfig) err
}
logger.Infof("%d running, %d pending pods", len(running), len(pending))
clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{}
c.clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{}
for _, m := range running {
clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, m.Status.PodIP)
c.clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, m.Status.PodIP)
}
err = c.saveMonEndpoints(clusterInfo)
err = c.saveMonEndpoints()
if err != nil {
return fmt.Errorf("failed to save endpoints for %d running mons. %+v", len(clusterInfo.Monitors), err)
return fmt.Errorf("failed to save endpoints for %d running mons. %+v", len(c.clusterInfo.Monitors), err)
}
if len(running) == c.Size {
@@ -194,7 +256,7 @@ func (c *Cluster) startPods(clusterInfo *mon.ClusterInfo, mons []*MonConfig) err
started := 0
alreadyRunning := 0
for _, m := range mons {
monPod := c.makeMonPod(m, clusterInfo, antiAffinity)
monPod := c.makeMonPod(m, antiAffinity)
name := monPod.Name
logger.Debugf("Starting pod: %+v", monPod)
_, err = c.clientset.CoreV1().Pods(c.Namespace).Create(monPod)
@@ -212,9 +274,9 @@ func (c *Cluster) startPods(clusterInfo *mon.ClusterInfo, mons []*MonConfig) err
if err != nil {
return fmt.Errorf("failed to start pod %s. %+v", name, err)
}
clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, podIP)
c.clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, podIP)
err = c.saveMonEndpoints(clusterInfo)
err = c.saveMonEndpoints()
if err != nil {
return fmt.Errorf("failed to save endpoints after starting mon %s. %+v", m.Name, err)
}
@@ -248,7 +310,7 @@ func (c *Cluster) waitForPodToStart(name string) (string, error) {
return "", fmt.Errorf("timed out waiting for pod %s to start", name)
}
func (c *Cluster) saveMonEndpoints(clusterInfo *mon.ClusterInfo) error {
func (c *Cluster) saveMonEndpoints() error {
configMap := &v1.ConfigMap{
ObjectMeta: v1.ObjectMeta{
Name: monConfigMapName,
@@ -257,7 +319,7 @@ func (c *Cluster) saveMonEndpoints(clusterInfo *mon.ClusterInfo) error {
},
}
configMap.Data = map[string]string{
monEndpointKey: mon.FlattenMonEndpoints(clusterInfo.Monitors),
monEndpointKey: mon.FlattenMonEndpoints(c.clusterInfo.Monitors),
}
_, err := c.clientset.CoreV1().ConfigMaps(c.Namespace).Create(configMap)
+2 -3
View File
@@ -18,7 +18,6 @@ package mon
import (
"fmt"
"github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/client-go/pkg/api/v1"
"k8s.io/client-go/pkg/labels"
@@ -51,7 +50,7 @@ func (c *Cluster) getLabels() map[string]string {
}
}
func (c *Cluster) makeMonPod(config *MonConfig, clusterInfo *mon.ClusterInfo, antiAffinity bool) *v1.Pod {
func (c *Cluster) makeMonPod(config *MonConfig, antiAffinity bool) *v1.Pod {
container := c.monContainer(config, clusterInfo.FSID)
dataDirSource := v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}
@@ -78,7 +77,7 @@ func (c *Cluster) makeMonPod(config *MonConfig, clusterInfo *mon.ClusterInfo, an
k8sutil.SetPodVersion(pod, k8sutil.VersionAttr, c.Version)
if antiAffinity {
k8sutil.PodWithAntiAffinity(pod, monClusterAttr, clusterInfo.Name)
k8sutil.PodWithAntiAffinity(pod, monClusterAttr, c.clusterInfo.Name)
}
return pod
}
+17 -18
View File
@@ -32,6 +32,7 @@ import (
"k8s.io/client-go/rest"
"github.com/rook/rook/pkg/cephmgr/client"
"github.com/rook/rook/pkg/operator/cluster"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/client-go/pkg/api"
"k8s.io/client-go/pkg/api/unversioned"
@@ -48,10 +49,6 @@ var (
ErrVersionOutdated = errors.New("requested version is outdated in apiserver")
)
const (
defaultPool = "rook"
)
type Operator struct {
Namespace string
MasterHost string
@@ -61,7 +58,7 @@ type Operator struct {
waitCluster sync.WaitGroup
factory client.ConnectionFactory
stopChMap map[string]chan struct{}
clusters map[string]*Cluster
clusters map[string]*cluster.Cluster
kubeHttpCli *http.Client
// Kubernetes resource version of the clusters
clusterRVs map[string]string
@@ -73,7 +70,7 @@ func New(host, namespace string, factory client.ConnectionFactory, clientset kub
MasterHost: host,
factory: factory,
clientset: clientset,
clusters: make(map[string]*Cluster),
clusters: make(map[string]*cluster.Cluster),
clusterRVs: make(map[string]string),
stopChMap: map[string]chan struct{}{},
retryDelay: 3,
@@ -147,7 +144,7 @@ func (o *Operator) watchTPR(watchVersion string) error {
continue
}
newCluster := newCluster(c.Spec, o.factory, o.clientset)
newCluster := cluster.New(c.Spec, o.factory, o.clientset)
stopCh := make(chan struct{})
o.stopChMap[ns] = stopCh
o.clusters[ns] = newCluster
@@ -170,21 +167,23 @@ func (o *Operator) watchTPR(watchVersion string) error {
}
func (o *Operator) startCluster(cluster *Cluster) {
if o.devicesInUse && cluster.Spec.UseAllDevices {
logger.Warningf("devices in more than one namespace not supported. ignoring devices in namespace %s", cluster.Spec.Namespace)
cluster.Spec.UseAllDevices = false
func (o *Operator) startCluster(c *cluster.Cluster) {
if o.devicesInUse && c.Spec.UseAllDevices {
logger.Warningf("devices in more than one namespace not supported. ignoring devices in namespace %s", c.Spec.Namespace)
c.Spec.UseAllDevices = false
}
if cluster.Spec.UseAllDevices {
if c.Spec.UseAllDevices {
o.devicesInUse = true
}
go func() {
err := cluster.CreateInstance()
err := c.CreateInstance()
if err != nil {
logger.Errorf("failed to create cluster in namespace %s. %+v", cluster.Spec.Namespace, err)
logger.Errorf("failed to create cluster in namespace %s. %+v", c.Spec.Namespace, err)
return
}
c.Monitor(o.stopChMap[c.Spec.Namespace])
}()
}
@@ -200,7 +199,7 @@ func (o *Operator) findAllClusters() (string, error) {
stopCh := make(chan struct{})
ns := c.Spec.Namespace
existingCluster := newCluster(c.Spec, o.factory, o.clientset)
existingCluster := cluster.New(c.Spec, o.factory, o.clientset)
o.stopChMap[ns] = stopCh
o.clusters[ns] = existingCluster
o.clusterRVs[ns] = c.Metadata.ResourceVersion
@@ -285,7 +284,7 @@ func (o *Operator) watch(watchVersion string) (<-chan *Event, <-chan error) {
return eventCh, errCh
}
func (o *Operator) isClustersCacheStale(currentClusters []Cluster) bool {
func (o *Operator) isClustersCacheStale(currentClusters []cluster.Cluster) bool {
if len(o.clusterRVs) != len(currentClusters) {
return true
}
@@ -305,13 +304,13 @@ func watchClusters(host, ns string, httpClient *http.Client, resourceVersion str
host, tprGroup, tprVersion, ns, resourceVersion))
}
func getClusterList(restcli rest.Interface, ns string) (*ClusterList, error) {
func getClusterList(restcli rest.Interface, ns string) (*cluster.ClusterList, error) {
b, err := restcli.Get().RequestURI(listClustersURI(ns)).DoRaw()
if err != nil {
return nil, err
}
clusters := &ClusterList{}
clusters := &cluster.ClusterList{}
if err := json.Unmarshal(b, clusters); err != nil {
return nil, err
}