failover a monitor when unhealthy

This commit is contained in:
Travis Nielsen committed 2017-03-23 11:00:31 -07:00
1 parent 2d1c150539
commit a16f6fc9dc
8 files changed
+370 -104

No files matched your search

+1 -1
View File
@@ -251,7 +251,7 @@ func ConnectToCluster(context *clusterd.Context, factory client.ConnectionFactor
return nil, fmt.Errorf("failed to connect to cluster %s: %+v", cluster.Name, err)
}
logger.Infof("successfully connected to cluster %s", cluster.Name)
logger.Infof("successfully connected to cluster %s with user %s", cluster.Name, user)
return conn, nil
}
+32 -16
View File
@@ -109,21 +109,28 @@ func removeMonitorsFromQuorum(factory client.ConnectionFactory, context *cluster
defer conn.Shutdown()
logger.Infof("removing %d monitors from quorum", len(monitors))
for monID, mon := range monitors {
logger.Debugf("removing monitor %s (%v)", monID, mon)
cmd := map[string]interface{}{"prefix": "mon remove", "name": mon.Name}
_, err = client.ExecuteMonCommand(conn, cmd, "mon remove")
if err != nil {
return fmt.Errorf("mon remove failed: %+v", err)
for _, mon := range monitors {
if err := RemoveMonitorFromQuorum(conn, mon.Name); err != nil {
return err
}
logger.Infof("removed monitor %s from node %s", mon.Name, monID)
}
return nil
}
func RemoveMonitorFromQuorum(conn client.Connection, name string) error {
logger.Debugf("removing monitor %s", name)
cmd := map[string]interface{}{"prefix": "mon remove", "name": name}
_, err := client.ExecuteMonCommand(conn, cmd, "mon remove")
if err != nil {
return fmt.Errorf("mon %s remove failed: %+v", name, err)
}
logger.Infof("removed monitor %s", name)
return nil
}
// extract the nodeIDs from the mon map
func monIDs(mons map[string]*CephMonitorConfig) []string {
nodes := []string{}
@@ -357,8 +364,6 @@ func calculateMonitorCount(nodeCount int) int {
}
func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context, cluster *ClusterInfo) error {
logger.Infof("waiting for mon quorum...")
// open an admin connection to the cluster
adminConn, err := ConnectToClusterAsAdmin(context, factory, cluster)
if err != nil {
@@ -366,6 +371,17 @@ func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context,
}
defer adminConn.Shutdown()
mons := []string{}
for _, m := range cluster.Monitors {
mons = append(mons, m.Name)
}
return WaitForQuorumWithConnection(adminConn, mons)
}
// Wait for all of the given monitors to form quorum
func WaitForQuorumWithConnection(conn client.Connection, mons []string) error {
logger.Infof("waiting for mon quorum")
// wait for monitors to establish quorum
retryCount := 0
retryMax := 20
@@ -383,7 +399,7 @@ func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context,
// get the mon_status response that contains info about all monitors in the mon map and
// their quorum status
monStatusResp, err := client.GetMonStatus(adminConn)
monStatusResp, err := client.GetMonStatus(conn)
if err != nil {
logger.Debugf("failed to get mon_status, err: %+v", err)
continue
@@ -391,11 +407,11 @@ func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context,
// check if each of the initial monitors is in quorum
allInQuorum := true
for _, im := range cluster.Monitors {
for _, name := range mons {
// first get the initial monitors corresponding mon map entry
var monMapEntry *client.MonMapEntry
for i := range monStatusResp.MonMap.Mons {
if im.Name == monStatusResp.MonMap.Mons[i].Name {
if name == monStatusResp.MonMap.Mons[i].Name {
monMapEntry = &monStatusResp.MonMap.Mons[i]
break
}
@@ -403,7 +419,7 @@ func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context,
if monMapEntry == nil {
// found an initial monitor that is not in the mon map, bail out of this retry
logger.Warningf("failed to find initial monitor %s in mon map", im.Name)
logger.Warningf("failed to find initial monitor %s in mon map", name)
allInQuorum = false
break
}
@@ -420,7 +436,7 @@ func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context,
if !inQuorumList {
// found an initial monitor that is not in quorum, bail out of this retry
logger.Warningf("initial monitor %s is not in quorum list", im.Name)
logger.Warningf("initial monitor %s is not in quorum list", name)
allInQuorum = false
break
}
+13 -16
View File
@@ -17,6 +17,7 @@ package api
import (
"fmt"
"strings"
"github.com/rook/rook/pkg/model"
"github.com/rook/rook/pkg/operator/k8sutil"
@@ -98,15 +99,17 @@ func (c *Cluster) makeDeployment() *extensions.Deployment {
}
func (c *Cluster) apiContainer() v1.Container {
// need a different prefix on the env vars
monSecretVar := opmon.MonSecretEnvVar()
adminSecretVar := opmon.AdminSecretEnvVar()
endpointVar := opmon.MonEndpointEnvVar()
clusterNameVar := opmon.ClusterNameEnvVar()
monSecretVar.Name = "ROOK_OPERATOR_MON_SECRET"
adminSecretVar.Name = "ROOK_OPERATOR_ADMIN_SECRET"
endpointVar.Name = "ROOK_OPERATOR_MON_ENDPOINTS"
clusterNameVar.Name = "ROOK_OPERATOR_CLUSTER_NAME"
envVars := []v1.EnvVar{
k8sutil.NamespaceEnvVar(),
opmon.MonSecretEnvVar(),
opmon.AdminSecretEnvVar(),
opmon.MonEndpointEnvVar(),
opmon.ClusterNameEnvVar(),
}
// replace the ROOKD prefix with ROOK_OPERATOR
for i := range envVars {
envVars[i].Name = strings.Replace(envVars[i].Name, "ROOKD_", "ROOK_OPERATOR_", 1)
}
command := fmt.Sprintf("/usr/bin/rook-operator api --data-dir=%s --api-port=%d --container-version=%s",
k8sutil.DataDir, model.Port, c.Version)
@@ -119,13 +122,7 @@ func (c *Cluster) apiContainer() v1.Container {
VolumeMounts: []v1.VolumeMount{
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
},
Env: []v1.EnvVar{
k8sutil.NamespaceEnvVar(),
clusterNameVar,
endpointVar,
monSecretVar,
adminSecretVar,
},
Env: envVars,
}
}
+3 -1
View File
@@ -75,7 +75,9 @@ func TestPodSpecs(t *testing.T) {
assert.Equal(t, "quay.io/rook/rook-operator:myversion", cont.Image)
assert.Equal(t, 1, len(cont.VolumeMounts))
assert.Equal(t, 5, len(cont.Env))
for _, v := range cont.Env {
assert.True(t, strings.HasPrefix(v.Name, "ROOK_OPERATOR_"))
}
expectedCommand := fmt.Sprintf("/usr/bin/rook-operator api --data-dir=/var/lib/rook --api-port=%d --container-version=%s",
model.Port, c.Version)
+2 -2
View File
@@ -42,7 +42,7 @@ const (
)
var (
healthCheckInterval = 8 * time.Second
healthCheckInterval = 10 * time.Second
logger = capnslog.NewPackageLogger("github.com/rook/rook-operator", "op-cluster")
)
@@ -115,7 +115,7 @@ func (c *Cluster) Monitor(stopCh <-chan struct{}) {
return
case <-time.After(healthCheckInterval):
logger.Infof("checking health of mons")
logger.Debugf("checking health of mons")
err := c.mons.CheckHealth()
if err != nil {
logger.Infof("failed to check mon health. %+v", err)
+246 -65
View File
@@ -17,6 +17,8 @@ package mon
import (
"fmt"
"strconv"
"strings"
"time"
"github.com/rook/rook/pkg/cephmgr/client"
@@ -39,6 +41,7 @@ const (
clusterSecretName = "cluster-name"
monConfigMapName = "mon-config"
monEndpointKey = "endpoints"
maxMonIDKey = "maxMonId"
)
type Cluster struct {
@@ -54,7 +57,11 @@ type Cluster struct {
clientset kubernetes.Interface
factory client.ConnectionFactory
retryDelay int
maxRetries int
clusterInfo *mon.ClusterInfo
maxMonID int
configDir string
waitForStart bool
dataDirHostPath string
}
@@ -73,6 +80,10 @@ func New(clientset kubernetes.Interface, factory client.ConnectionFactory, names
Size: 3,
AntiAffinity: true,
retryDelay: 6,
maxRetries: 15,
maxMonID: -1,
configDir: k8sutil.DataDir,
waitForStart: true,
}
}
@@ -83,33 +94,48 @@ func (c *Cluster) Start() (*mon.ClusterInfo, error) {
if err != nil {
return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err)
}
mons := []*MonConfig{}
for i := 0; i < c.Size; i++ {
mons = append(mons, &MonConfig{Name: fmt.Sprintf("mon%d", i), Port: int32(mon.Port)})
}
err = c.startPods(mons)
if err != nil {
return nil, fmt.Errorf("failed to start mon pods. %+v", err)
}
context := &clusterd.Context{ConfigDir: k8sutil.DataDir}
err = mon.WaitForQuorum(c.factory, context, c.clusterInfo)
if err != nil {
return nil, fmt.Errorf("failed to wait for mon quorum. %+v", err)
if len(c.clusterInfo.Monitors) == 0 {
// Start the initial monitors at startup
mons := c.getExpectedMonConfig()
err = c.startPods(nil, mons)
if err != nil {
return nil, fmt.Errorf("failed to start mon pods. %+v", err)
}
} else {
// Check the health of a previously started cluster
err = c.CheckHealth()
if err != nil {
logger.Warningf("failed to check mon health %+v. %+v", c.clusterInfo.Monitors, err)
}
}
return c.clusterInfo, nil
}
func (c *Cluster) CheckHealth() error {
context := &clusterd.Context{ConfigDir: k8sutil.DataDir}
// update the config map if the pod ips changed
// must retry since during startup of pods they might take some time to initialize
k8sutil.Retry(time.Duration(c.retryDelay)*time.Second, c.maxRetries, func() (bool, error) {
// TODO: There is more work to get the reboot functional. The mons are not
// happy if their ip address changes. They expect a constant id.
err := c.updateConfigMapIfPodIPsChanged()
if err != nil {
logger.Infof("unable to check on mon pods. %+v", err)
return false, nil
}
return true, nil
})
// connect to the mons
context := &clusterd.Context{ConfigDir: c.configDir}
conn, err := mon.ConnectToClusterAsAdmin(context, c.factory, c.clusterInfo)
if err != nil {
return fmt.Errorf("cannot connect to cluster. %+v", err)
}
defer conn.Shutdown()
// get the status and check for quorum
status, err := client.GetMonStatus(conn)
if err != nil {
return fmt.Errorf("failed to get mon status. %+v", err)
@@ -118,20 +144,11 @@ func (c *Cluster) CheckHealth() error {
for _, monitor := range status.MonMap.Mons {
inQuorum := monInQuorum(monitor, status.Quorum)
if inQuorum {
logger.Infof("mon %s found in quorum", monitor.Name)
logger.Debugf("mon %s found in quorum", monitor.Name)
} else {
logger.Infof("mon %s NOT found in quroum. %+v", monitor.Name, status.Quorum)
}
logger.Warningf("mon %s NOT found in quroum. %+v", monitor.Name, status.Quorum)
pod, err := c.clientset.CoreV1().Pods(c.Namespace).Get(monitor.Name)
if err != nil {
logger.Infof("Get mon %s pod failed. %+v", monitor.Name, err)
} else {
logger.Infof("mon %s pod status: %+v", monitor.Name, pod.Status.Phase)
}
if !inQuorum {
err = c.failoverMon(monitor.Name)
err = c.failoverMon(conn, monitor.Name)
if err != nil {
logger.Errorf("failed to failover mon %s. %+v", monitor.Name, err)
}
@@ -141,6 +158,45 @@ func (c *Cluster) CheckHealth() error {
return nil
}
func (c *Cluster) updateConfigMapIfPodIPsChanged() error {
pods, err := c.getPods()
if err != nil {
return fmt.Errorf("failed to check if pod ips changed. %+v", err)
}
if len(pods.Items) == 0 {
return fmt.Errorf("where are the mon pods?")
}
logger.Debugf("there are %d mon pods. checking the podIPs.", len(pods.Items))
updated := false
for _, pod := range pods.Items {
if pod.Status.PodIP == "" {
return fmt.Errorf("no podIP for mon %s", pod.Name)
}
if pod.Status.Phase != v1.PodRunning {
return fmt.Errorf("pod %s is not running. phase=%v", pod.Name, pod.Status.Phase)
}
m := mon.ToCephMon(pod.Name, pod.Status.PodIP)
existing, ok := c.clusterInfo.Monitors[pod.Name]
if !ok || existing.Endpoint == m.Endpoint {
// the endpoint does not need to be updated
logger.Debugf("Did not need to update pod %s with endpoint %+v. ok=%t, m=%+v", pod.Name, pod.Status.PodIP, ok, m)
continue
}
logger.Infof("updating mon %s endpoint from %s to %s", pod.Name, existing.Endpoint, m.Endpoint)
c.clusterInfo.Monitors[pod.Name] = m
updated = true
}
if updated {
return c.saveMonConfig()
}
logger.Debugf("no update to mon pod ips")
return nil
}
func monInQuorum(monitor client.MonMapEntry, quorum []int) bool {
for _, rank := range quorum {
if rank == monitor.Rank {
@@ -150,34 +206,105 @@ func monInQuorum(monitor client.MonMapEntry, quorum []int) bool {
return false
}
func (c *Cluster) failoverMon(name string) error {
//logger.Infof("Failing over monitor %s", name)
func (c *Cluster) failoverMon(conn client.Connection, name string) error {
logger.Infof("Failing over monitor %s", name)
// Start a new monitor
c.maxMonID++
mons := []*MonConfig{&MonConfig{Name: fmt.Sprintf("mon%d", c.maxMonID), Port: int32(mon.Port)}}
logger.Infof("starting new mon %s", mons[0].Name)
err := c.startPods(conn, mons)
if err != nil {
return fmt.Errorf("failed to start new mon %s. %+v", mons[0].Name, err)
}
// Remove the mon pod if it is still there
var gracePeriod int64
options := &v1.DeleteOptions{GracePeriodSeconds: &gracePeriod}
err = c.clientset.CoreV1().Pods(c.Namespace).Delete(name, options)
if err != nil {
if errors.IsNotFound(err) {
logger.Infof("dead mon pod %s was already gone", name)
} else {
return fmt.Errorf("failed to remove dead mon pod %s. %+v", name, err)
}
}
// Remove the bad monitor from quorum
err = mon.RemoveMonitorFromQuorum(conn, name)
if err != nil {
return fmt.Errorf("failed to remove mon %s from quorum. %+v", name, err)
}
delete(c.clusterInfo.Monitors, name)
err = c.saveMonConfig()
if err != nil {
return fmt.Errorf("failed to save mon config after failing mon %s. %+v", name, err)
}
return nil
}
// Retrieve the ceph cluster info if it already exists.
// If a new cluster create new keys.
func (c *Cluster) initClusterInfo() error {
// get the cluster secrets
secrets, err := c.clientset.CoreV1().Secrets(c.Namespace).Get(appName)
if err != nil {
if !errors.IsNotFound(err) {
return fmt.Errorf("failed to get mon secrets. %+v", err)
}
return c.createMonSecretsAndSave()
err = c.createMonSecretsAndSave()
if err != nil {
return err
}
} else {
c.clusterInfo = &mon.ClusterInfo{
Name: string(secrets.Data[clusterSecretName]),
FSID: string(secrets.Data[fsidSecretName]),
MonitorSecret: string(secrets.Data[monSecretName]),
AdminSecret: string(secrets.Data[adminSecretName]),
}
logger.Debugf("found existing monitor secrets for cluster %s", c.clusterInfo.Name)
}
c.clusterInfo = &mon.ClusterInfo{
Name: string(secrets.Data[clusterSecretName]),
FSID: string(secrets.Data[fsidSecretName]),
MonitorSecret: string(secrets.Data[monSecretName]),
AdminSecret: string(secrets.Data[adminSecretName]),
Monitors: map[string]*mon.CephMonitorConfig{},
// get the existing monitor config
err = c.loadMonConfig()
if err != nil {
return fmt.Errorf("failed to get mon config. %+v", err)
}
logger.Infof("found existing monitor secrets for cluster %s", c.clusterInfo.Name)
return nil
}
func (c *Cluster) getExpectedMonConfig() []*MonConfig {
mons := []*MonConfig{}
// initialize the mon pod info for mons that have been previously created
for _, monitor := range c.clusterInfo.Monitors {
mons = append(mons, &MonConfig{Name: monitor.Name, Port: int32(mon.Port)})
}
// initialize mon info if we don't have enough mons (at first startup)
for i := len(c.clusterInfo.Monitors); i < c.Size; i++ {
c.maxMonID++
mons = append(mons, &MonConfig{Name: fmt.Sprintf("mon%d", c.maxMonID), Port: int32(mon.Port)})
}
return mons
}
// get the ID of a monitor from its name
func getMonID(name string) (int, error) {
if strings.Index(name, "mon") != 0 || len(name) < 4 {
return -1, fmt.Errorf("unexpected mon name")
}
id, err := strconv.Atoi(name[3:])
if err != nil {
return -1, err
}
return id, nil
}
func (c *Cluster) createMonSecretsAndSave() error {
logger.Infof("creating mon secrets for a new cluster")
var err error
@@ -225,40 +352,20 @@ func (c *Cluster) createMonSecretsAndSave() error {
return nil
}
func (c *Cluster) startPods(mons []*MonConfig) error {
func (c *Cluster) startPods(conn client.Connection, mons []*MonConfig) error {
// schedule the mons on different nodes if we have enough nodes to be unique
antiAffinity, err := c.getAntiAffinity()
if err != nil {
return fmt.Errorf("failed to get antiaffinity. %+v", err)
}
running, pending, err := c.pollPods()
if err != nil {
return fmt.Errorf("failed to get mon pods. %+v", err)
}
logger.Infof("%d running, %d pending pods", len(running), len(pending))
c.clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{}
for _, m := range running {
c.clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, m.Status.PodIP)
}
err = c.saveMonEndpoints()
if err != nil {
return fmt.Errorf("failed to save endpoints for %d running mons. %+v", len(c.clusterInfo.Monitors), err)
}
if len(running) == c.Size {
logger.Infof("pods are already running")
return nil
}
started := 0
preexisted := len(c.clusterInfo.Monitors)
created := 0
alreadyRunning := 0
for _, m := range mons {
monPod := c.makeMonPod(m, antiAffinity)
name := monPod.Name
logger.Debugf("Starting pod: %+v", monPod)
name := monPod.Name
_, err = c.clientset.CoreV1().Pods(c.Namespace).Create(monPod)
if err != nil {
if !errors.IsAlreadyExists(err) {
@@ -267,7 +374,7 @@ func (c *Cluster) startPods(mons []*MonConfig) error {
logger.Infof("pod %s already exists", name)
alreadyRunning++
} else {
started++
created++
}
podIP, err := c.waitForPodToStart(name)
@@ -276,21 +383,55 @@ func (c *Cluster) startPods(mons []*MonConfig) error {
}
c.clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, podIP)
err = c.saveMonEndpoints()
err = c.saveMonConfig()
if err != nil {
return fmt.Errorf("failed to save endpoints after starting mon %s. %+v", m.Name, err)
}
}
logger.Infof("started %d/%d mons (%d already running)", (started + alreadyRunning), c.Size, alreadyRunning)
logger.Infof("mons created: %d, alreadyRunning: %d, preexisted: %d", created, alreadyRunning, preexisted)
return c.waitForMonsToJoin(conn, mons)
}
func (c *Cluster) waitForMonsToJoin(conn client.Connection, mons []*MonConfig) error {
if !c.waitForStart {
return nil
}
// initialize a connection if it is not already connected
if conn == nil {
context := &clusterd.Context{ConfigDir: k8sutil.DataDir}
var err error
conn, err = mon.ConnectToClusterAsAdmin(context, c.factory, c.clusterInfo)
if err != nil {
return fmt.Errorf("cannot connect to cluster. %+v", err)
}
defer conn.Shutdown()
}
starting := []string{}
for _, m := range mons {
starting = append(starting, m.Name)
}
// wait for the monitors to join quorum
err := mon.WaitForQuorumWithConnection(conn, starting)
if err != nil {
return fmt.Errorf("failed to wait for mon quorum. %+v", err)
}
return nil
}
func (c *Cluster) waitForPodToStart(name string) (string, error) {
if !c.waitForStart {
return "", nil
}
// Poll the status of the pods to see if they are ready
status := ""
for i := 0; i < 15; i++ {
for i := 0; i < c.maxRetries; i++ {
// wait and try again
logger.Infof("waiting %ds for pod %s to start. status=%s", c.retryDelay, name, status)
<-time.After(time.Duration(c.retryDelay) * time.Second)
@@ -310,7 +451,7 @@ func (c *Cluster) waitForPodToStart(name string) (string, error) {
return "", fmt.Errorf("timed out waiting for pod %s to start", name)
}
func (c *Cluster) saveMonEndpoints() error {
func (c *Cluster) saveMonConfig() error {
configMap := &v1.ConfigMap{
ObjectMeta: v1.ObjectMeta{
Name: monConfigMapName,
@@ -320,6 +461,7 @@ func (c *Cluster) saveMonEndpoints() error {
}
configMap.Data = map[string]string{
monEndpointKey: mon.FlattenMonEndpoints(c.clusterInfo.Monitors),
maxMonIDKey: strconv.Itoa(c.maxMonID),
}
_, err := c.clientset.CoreV1().ConfigMaps(c.Namespace).Create(configMap)
@@ -338,6 +480,45 @@ func (c *Cluster) saveMonEndpoints() error {
return nil
}
func (c *Cluster) loadMonConfig() error {
cm, err := c.clientset.CoreV1().ConfigMaps(c.Namespace).Get(monConfigMapName)
if err != nil {
if !errors.IsNotFound(err) {
return err
}
// If the config map was not found, initialize the empty set of monitors
c.maxMonID = -1
c.clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{}
return c.saveMonConfig()
}
// Parse the monitor List
if info, ok := cm.Data[monEndpointKey]; ok {
c.clusterInfo.Monitors = mon.ParseMonEndpoints(info)
} else {
c.clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{}
}
// Parse the max monitor id
if id, ok := cm.Data[maxMonIDKey]; ok {
c.maxMonID, err = strconv.Atoi(id)
if err != nil {
logger.Errorf("invalid max mon id %s. %+v", id, err)
}
}
// Make sure the max id is consistent with the current monitors
for _, m := range c.clusterInfo.Monitors {
id, _ := getMonID(m.Name)
if c.maxMonID < id {
c.maxMonID = id
}
}
logger.Infof("loaded: maxMonID=%d, mons=%+v", c.maxMonID, c.clusterInfo.Monitors)
return nil
}
// detect whether we have a big enough cluster to run services on different nodes.
// the anti-affinity will prevent pods of the same type of running on the same node.
func (c *Cluster) getAntiAffinity() (bool, error) {
+67 -1
View File
@@ -20,9 +20,12 @@ import (
"k8s.io/client-go/pkg/api/v1"
"github.com/rook/rook/pkg/cephmgr/client"
testclient "github.com/rook/rook/pkg/cephmgr/client/test"
"github.com/rook/rook/pkg/operator/test"
"os"
"github.com/stretchr/testify/assert"
)
@@ -30,6 +33,7 @@ func TestStartMonPods(t *testing.T) {
clientset := test.New(3)
factory := &testclient.MockConnectionFactory{Fsid: "fsid", SecretKey: "mysecret"}
c := New(clientset, factory, "ns", "", "myversion")
c.maxRetries = 1
c.retryDelay = 0
// start a basic cluster
@@ -75,7 +79,7 @@ func validateStart(t *testing.T, c *Cluster) {
func TestSaveMonEndpoints(t *testing.T) {
clientset := test.New(1)
c := New(clientset, nil, "ns", "myversion")
c := New(clientset, nil, "ns", "", "myversion")
c.clusterInfo = test.CreateClusterInfo(1)
// create the initial config map
@@ -94,5 +98,67 @@ func TestSaveMonEndpoints(t *testing.T) {
cm, err = c.clientset.CoreV1().ConfigMaps(c.Namespace).Get("mon-config")
assert.Nil(t, err)
assert.Equal(t, "mon1=2.3.4.5:6790", cm.Data["endpoints"])
}
func TestCheckHealth(t *testing.T) {
clientset := test.New(1)
factory := &testclient.MockConnectionFactory{Fsid: "fsid", SecretKey: "mysecret"}
c := New(clientset, factory, "ns", "", "myversion")
c.retryDelay = 1
c.maxRetries = 1
c.clusterInfo = test.CreateClusterInfo(1)
c.configDir = "/tmp/healthtest"
c.waitForStart = false
defer os.RemoveAll(c.configDir)
err := c.CheckHealth()
assert.Nil(t, err)
c.maxMonID = 10
conn, err := factory.NewConnWithClusterAndUser(c.Namespace, "admin")
defer conn.Shutdown()
err = c.failoverMon(conn, "mon1")
assert.Nil(t, err)
cm, err := c.clientset.CoreV1().ConfigMaps(c.Namespace).Get("mon-config")
assert.Nil(t, err)
assert.Equal(t, "mon11=:6790", cm.Data["endpoints"])
}
func TestMonInQuourm(t *testing.T) {
entry := client.MonMapEntry{Name: "foo", Rank: 23}
quorum := []int{}
// Nothing in quorum
assert.False(t, monInQuorum(entry, quorum))
// One or more members in quorum
quorum = []int{23}
assert.True(t, monInQuorum(entry, quorum))
quorum = []int{5, 6, 7, 23, 8}
assert.True(t, monInQuorum(entry, quorum))
// Not in quorum
entry.Rank = 1
assert.False(t, monInQuorum(entry, quorum))
}
func TestMonID(t *testing.T) {
// invalid
id, err := getMonID("m")
assert.NotNil(t, err)
assert.Equal(t, -1, id)
id, err = getMonID("mon")
assert.NotNil(t, err)
assert.Equal(t, -1, id)
id, err = getMonID("monitor0")
assert.NotNil(t, err)
assert.Equal(t, -1, id)
// valid
id, err = getMonID("mon0")
assert.Nil(t, err)
assert.Equal(t, 0, id)
id, err = getMonID("mon123")
assert.Nil(t, err)
assert.Equal(t, 123, id)
}
+6 -2
View File
@@ -52,7 +52,7 @@ func (c *Cluster) getLabels() map[string]string {
func (c *Cluster) makeMonPod(config *MonConfig, antiAffinity bool) *v1.Pod {
container := c.monContainer(config, clusterInfo.FSID)
container := c.monContainer(config, c.clusterInfo.FSID)
dataDirSource := v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}
if c.dataDirHostPath != "" {
dataDirSource = v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: c.dataDirHostPath}}
@@ -112,8 +112,12 @@ func (c *Cluster) monContainer(config *MonConfig, fsid string) v1.Container {
}
}
func (c *Cluster) getPods() (*v1.PodList, error) {
return c.clientset.CoreV1().Pods(c.Namespace).List(c.listOptions())
}
func (c *Cluster) pollPods() ([]*v1.Pod, []*v1.Pod, error) {
podList, err := c.clientset.CoreV1().Pods(c.Namespace).List(c.listOptions())
podList, err := c.getPods()
if err != nil {
return nil, nil, fmt.Errorf("failed to list running pods: %v", err)
}