failover a monitor when unhealthy
This commit is contained in:
1 parent
2d1c150539
commit
a16f6fc9dc
8 files changed
+370
-104
No files matched your search
@@ -251,7 +251,7 @@ func ConnectToCluster(context *clusterd.Context, factory client.ConnectionFactor
|
||||
return nil, fmt.Errorf("failed to connect to cluster %s: %+v", cluster.Name, err)
|
||||
}
|
||||
|
||||
logger.Infof("successfully connected to cluster %s", cluster.Name)
|
||||
logger.Infof("successfully connected to cluster %s with user %s", cluster.Name, user)
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
|
||||
+32
-16
@@ -109,21 +109,28 @@ func removeMonitorsFromQuorum(factory client.ConnectionFactory, context *cluster
|
||||
defer conn.Shutdown()
|
||||
|
||||
logger.Infof("removing %d monitors from quorum", len(monitors))
|
||||
for monID, mon := range monitors {
|
||||
logger.Debugf("removing monitor %s (%v)", monID, mon)
|
||||
|
||||
cmd := map[string]interface{}{"prefix": "mon remove", "name": mon.Name}
|
||||
_, err = client.ExecuteMonCommand(conn, cmd, "mon remove")
|
||||
if err != nil {
|
||||
return fmt.Errorf("mon remove failed: %+v", err)
|
||||
for _, mon := range monitors {
|
||||
if err := RemoveMonitorFromQuorum(conn, mon.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Infof("removed monitor %s from node %s", mon.Name, monID)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func RemoveMonitorFromQuorum(conn client.Connection, name string) error {
|
||||
logger.Debugf("removing monitor %s", name)
|
||||
|
||||
cmd := map[string]interface{}{"prefix": "mon remove", "name": name}
|
||||
_, err := client.ExecuteMonCommand(conn, cmd, "mon remove")
|
||||
if err != nil {
|
||||
return fmt.Errorf("mon %s remove failed: %+v", name, err)
|
||||
}
|
||||
|
||||
logger.Infof("removed monitor %s", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// extract the nodeIDs from the mon map
|
||||
func monIDs(mons map[string]*CephMonitorConfig) []string {
|
||||
nodes := []string{}
|
||||
@@ -357,8 +364,6 @@ func calculateMonitorCount(nodeCount int) int {
|
||||
}
|
||||
|
||||
func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context, cluster *ClusterInfo) error {
|
||||
logger.Infof("waiting for mon quorum...")
|
||||
|
||||
// open an admin connection to the cluster
|
||||
adminConn, err := ConnectToClusterAsAdmin(context, factory, cluster)
|
||||
if err != nil {
|
||||
@@ -366,6 +371,17 @@ func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context,
|
||||
}
|
||||
defer adminConn.Shutdown()
|
||||
|
||||
mons := []string{}
|
||||
for _, m := range cluster.Monitors {
|
||||
mons = append(mons, m.Name)
|
||||
}
|
||||
return WaitForQuorumWithConnection(adminConn, mons)
|
||||
}
|
||||
|
||||
// Wait for all of the given monitors to form quorum
|
||||
func WaitForQuorumWithConnection(conn client.Connection, mons []string) error {
|
||||
logger.Infof("waiting for mon quorum")
|
||||
|
||||
// wait for monitors to establish quorum
|
||||
retryCount := 0
|
||||
retryMax := 20
|
||||
@@ -383,7 +399,7 @@ func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context,
|
||||
|
||||
// get the mon_status response that contains info about all monitors in the mon map and
|
||||
// their quorum status
|
||||
monStatusResp, err := client.GetMonStatus(adminConn)
|
||||
monStatusResp, err := client.GetMonStatus(conn)
|
||||
if err != nil {
|
||||
logger.Debugf("failed to get mon_status, err: %+v", err)
|
||||
continue
|
||||
@@ -391,11 +407,11 @@ func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context,
|
||||
|
||||
// check if each of the initial monitors is in quorum
|
||||
allInQuorum := true
|
||||
for _, im := range cluster.Monitors {
|
||||
for _, name := range mons {
|
||||
// first get the initial monitors corresponding mon map entry
|
||||
var monMapEntry *client.MonMapEntry
|
||||
for i := range monStatusResp.MonMap.Mons {
|
||||
if im.Name == monStatusResp.MonMap.Mons[i].Name {
|
||||
if name == monStatusResp.MonMap.Mons[i].Name {
|
||||
monMapEntry = &monStatusResp.MonMap.Mons[i]
|
||||
break
|
||||
}
|
||||
@@ -403,7 +419,7 @@ func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context,
|
||||
|
||||
if monMapEntry == nil {
|
||||
// found an initial monitor that is not in the mon map, bail out of this retry
|
||||
logger.Warningf("failed to find initial monitor %s in mon map", im.Name)
|
||||
logger.Warningf("failed to find initial monitor %s in mon map", name)
|
||||
allInQuorum = false
|
||||
break
|
||||
}
|
||||
@@ -420,7 +436,7 @@ func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context,
|
||||
|
||||
if !inQuorumList {
|
||||
// found an initial monitor that is not in quorum, bail out of this retry
|
||||
logger.Warningf("initial monitor %s is not in quorum list", im.Name)
|
||||
logger.Warningf("initial monitor %s is not in quorum list", name)
|
||||
allInQuorum = false
|
||||
break
|
||||
}
|
||||
|
||||
+13
-16
@@ -17,6 +17,7 @@ package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/rook/rook/pkg/model"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
@@ -98,15 +99,17 @@ func (c *Cluster) makeDeployment() *extensions.Deployment {
|
||||
}
|
||||
|
||||
func (c *Cluster) apiContainer() v1.Container {
|
||||
// need a different prefix on the env vars
|
||||
monSecretVar := opmon.MonSecretEnvVar()
|
||||
adminSecretVar := opmon.AdminSecretEnvVar()
|
||||
endpointVar := opmon.MonEndpointEnvVar()
|
||||
clusterNameVar := opmon.ClusterNameEnvVar()
|
||||
monSecretVar.Name = "ROOK_OPERATOR_MON_SECRET"
|
||||
adminSecretVar.Name = "ROOK_OPERATOR_ADMIN_SECRET"
|
||||
endpointVar.Name = "ROOK_OPERATOR_MON_ENDPOINTS"
|
||||
clusterNameVar.Name = "ROOK_OPERATOR_CLUSTER_NAME"
|
||||
envVars := []v1.EnvVar{
|
||||
k8sutil.NamespaceEnvVar(),
|
||||
opmon.MonSecretEnvVar(),
|
||||
opmon.AdminSecretEnvVar(),
|
||||
opmon.MonEndpointEnvVar(),
|
||||
opmon.ClusterNameEnvVar(),
|
||||
}
|
||||
// replace the ROOKD prefix with ROOK_OPERATOR
|
||||
for i := range envVars {
|
||||
envVars[i].Name = strings.Replace(envVars[i].Name, "ROOKD_", "ROOK_OPERATOR_", 1)
|
||||
}
|
||||
|
||||
command := fmt.Sprintf("/usr/bin/rook-operator api --data-dir=%s --api-port=%d --container-version=%s",
|
||||
k8sutil.DataDir, model.Port, c.Version)
|
||||
@@ -119,13 +122,7 @@ func (c *Cluster) apiContainer() v1.Container {
|
||||
VolumeMounts: []v1.VolumeMount{
|
||||
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
|
||||
},
|
||||
Env: []v1.EnvVar{
|
||||
k8sutil.NamespaceEnvVar(),
|
||||
clusterNameVar,
|
||||
endpointVar,
|
||||
monSecretVar,
|
||||
adminSecretVar,
|
||||
},
|
||||
Env: envVars,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -75,7 +75,9 @@ func TestPodSpecs(t *testing.T) {
|
||||
assert.Equal(t, "quay.io/rook/rook-operator:myversion", cont.Image)
|
||||
assert.Equal(t, 1, len(cont.VolumeMounts))
|
||||
assert.Equal(t, 5, len(cont.Env))
|
||||
|
||||
for _, v := range cont.Env {
|
||||
assert.True(t, strings.HasPrefix(v.Name, "ROOK_OPERATOR_"))
|
||||
}
|
||||
expectedCommand := fmt.Sprintf("/usr/bin/rook-operator api --data-dir=/var/lib/rook --api-port=%d --container-version=%s",
|
||||
model.Port, c.Version)
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
healthCheckInterval = 8 * time.Second
|
||||
healthCheckInterval = 10 * time.Second
|
||||
logger = capnslog.NewPackageLogger("github.com/rook/rook-operator", "op-cluster")
|
||||
)
|
||||
|
||||
@@ -115,7 +115,7 @@ func (c *Cluster) Monitor(stopCh <-chan struct{}) {
|
||||
return
|
||||
|
||||
case <-time.After(healthCheckInterval):
|
||||
logger.Infof("checking health of mons")
|
||||
logger.Debugf("checking health of mons")
|
||||
err := c.mons.CheckHealth()
|
||||
if err != nil {
|
||||
logger.Infof("failed to check mon health. %+v", err)
|
||||
|
||||
+246
-65
@@ -17,6 +17,8 @@ package mon
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rook/rook/pkg/cephmgr/client"
|
||||
@@ -39,6 +41,7 @@ const (
|
||||
clusterSecretName = "cluster-name"
|
||||
monConfigMapName = "mon-config"
|
||||
monEndpointKey = "endpoints"
|
||||
maxMonIDKey = "maxMonId"
|
||||
)
|
||||
|
||||
type Cluster struct {
|
||||
@@ -54,7 +57,11 @@ type Cluster struct {
|
||||
clientset kubernetes.Interface
|
||||
factory client.ConnectionFactory
|
||||
retryDelay int
|
||||
maxRetries int
|
||||
clusterInfo *mon.ClusterInfo
|
||||
maxMonID int
|
||||
configDir string
|
||||
waitForStart bool
|
||||
dataDirHostPath string
|
||||
}
|
||||
|
||||
@@ -73,6 +80,10 @@ func New(clientset kubernetes.Interface, factory client.ConnectionFactory, names
|
||||
Size: 3,
|
||||
AntiAffinity: true,
|
||||
retryDelay: 6,
|
||||
maxRetries: 15,
|
||||
maxMonID: -1,
|
||||
configDir: k8sutil.DataDir,
|
||||
waitForStart: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,33 +94,48 @@ func (c *Cluster) Start() (*mon.ClusterInfo, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err)
|
||||
}
|
||||
mons := []*MonConfig{}
|
||||
for i := 0; i < c.Size; i++ {
|
||||
mons = append(mons, &MonConfig{Name: fmt.Sprintf("mon%d", i), Port: int32(mon.Port)})
|
||||
}
|
||||
|
||||
err = c.startPods(mons)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to start mon pods. %+v", err)
|
||||
}
|
||||
|
||||
context := &clusterd.Context{ConfigDir: k8sutil.DataDir}
|
||||
err = mon.WaitForQuorum(c.factory, context, c.clusterInfo)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to wait for mon quorum. %+v", err)
|
||||
if len(c.clusterInfo.Monitors) == 0 {
|
||||
// Start the initial monitors at startup
|
||||
mons := c.getExpectedMonConfig()
|
||||
err = c.startPods(nil, mons)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to start mon pods. %+v", err)
|
||||
}
|
||||
} else {
|
||||
// Check the health of a previously started cluster
|
||||
err = c.CheckHealth()
|
||||
if err != nil {
|
||||
logger.Warningf("failed to check mon health %+v. %+v", c.clusterInfo.Monitors, err)
|
||||
}
|
||||
}
|
||||
|
||||
return c.clusterInfo, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) CheckHealth() error {
|
||||
context := &clusterd.Context{ConfigDir: k8sutil.DataDir}
|
||||
// update the config map if the pod ips changed
|
||||
// must retry since during startup of pods they might take some time to initialize
|
||||
k8sutil.Retry(time.Duration(c.retryDelay)*time.Second, c.maxRetries, func() (bool, error) {
|
||||
// TODO: There is more work to get the reboot functional. The mons are not
|
||||
// happy if their ip address changes. They expect a constant id.
|
||||
err := c.updateConfigMapIfPodIPsChanged()
|
||||
if err != nil {
|
||||
logger.Infof("unable to check on mon pods. %+v", err)
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
|
||||
// connect to the mons
|
||||
context := &clusterd.Context{ConfigDir: c.configDir}
|
||||
conn, err := mon.ConnectToClusterAsAdmin(context, c.factory, c.clusterInfo)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot connect to cluster. %+v", err)
|
||||
}
|
||||
defer conn.Shutdown()
|
||||
|
||||
// get the status and check for quorum
|
||||
status, err := client.GetMonStatus(conn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get mon status. %+v", err)
|
||||
@@ -118,20 +144,11 @@ func (c *Cluster) CheckHealth() error {
|
||||
for _, monitor := range status.MonMap.Mons {
|
||||
inQuorum := monInQuorum(monitor, status.Quorum)
|
||||
if inQuorum {
|
||||
logger.Infof("mon %s found in quorum", monitor.Name)
|
||||
logger.Debugf("mon %s found in quorum", monitor.Name)
|
||||
} else {
|
||||
logger.Infof("mon %s NOT found in quroum. %+v", monitor.Name, status.Quorum)
|
||||
}
|
||||
logger.Warningf("mon %s NOT found in quroum. %+v", monitor.Name, status.Quorum)
|
||||
|
||||
pod, err := c.clientset.CoreV1().Pods(c.Namespace).Get(monitor.Name)
|
||||
if err != nil {
|
||||
logger.Infof("Get mon %s pod failed. %+v", monitor.Name, err)
|
||||
} else {
|
||||
logger.Infof("mon %s pod status: %+v", monitor.Name, pod.Status.Phase)
|
||||
}
|
||||
|
||||
if !inQuorum {
|
||||
err = c.failoverMon(monitor.Name)
|
||||
err = c.failoverMon(conn, monitor.Name)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to failover mon %s. %+v", monitor.Name, err)
|
||||
}
|
||||
@@ -141,6 +158,45 @@ func (c *Cluster) CheckHealth() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) updateConfigMapIfPodIPsChanged() error {
|
||||
pods, err := c.getPods()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check if pod ips changed. %+v", err)
|
||||
}
|
||||
if len(pods.Items) == 0 {
|
||||
return fmt.Errorf("where are the mon pods?")
|
||||
}
|
||||
|
||||
logger.Debugf("there are %d mon pods. checking the podIPs.", len(pods.Items))
|
||||
updated := false
|
||||
for _, pod := range pods.Items {
|
||||
if pod.Status.PodIP == "" {
|
||||
return fmt.Errorf("no podIP for mon %s", pod.Name)
|
||||
}
|
||||
if pod.Status.Phase != v1.PodRunning {
|
||||
return fmt.Errorf("pod %s is not running. phase=%v", pod.Name, pod.Status.Phase)
|
||||
}
|
||||
m := mon.ToCephMon(pod.Name, pod.Status.PodIP)
|
||||
existing, ok := c.clusterInfo.Monitors[pod.Name]
|
||||
if !ok || existing.Endpoint == m.Endpoint {
|
||||
// the endpoint does not need to be updated
|
||||
logger.Debugf("Did not need to update pod %s with endpoint %+v. ok=%t, m=%+v", pod.Name, pod.Status.PodIP, ok, m)
|
||||
continue
|
||||
}
|
||||
|
||||
logger.Infof("updating mon %s endpoint from %s to %s", pod.Name, existing.Endpoint, m.Endpoint)
|
||||
c.clusterInfo.Monitors[pod.Name] = m
|
||||
updated = true
|
||||
}
|
||||
|
||||
if updated {
|
||||
return c.saveMonConfig()
|
||||
}
|
||||
|
||||
logger.Debugf("no update to mon pod ips")
|
||||
return nil
|
||||
}
|
||||
|
||||
func monInQuorum(monitor client.MonMapEntry, quorum []int) bool {
|
||||
for _, rank := range quorum {
|
||||
if rank == monitor.Rank {
|
||||
@@ -150,34 +206,105 @@ func monInQuorum(monitor client.MonMapEntry, quorum []int) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *Cluster) failoverMon(name string) error {
|
||||
//logger.Infof("Failing over monitor %s", name)
|
||||
func (c *Cluster) failoverMon(conn client.Connection, name string) error {
|
||||
logger.Infof("Failing over monitor %s", name)
|
||||
|
||||
// Start a new monitor
|
||||
c.maxMonID++
|
||||
mons := []*MonConfig{&MonConfig{Name: fmt.Sprintf("mon%d", c.maxMonID), Port: int32(mon.Port)}}
|
||||
logger.Infof("starting new mon %s", mons[0].Name)
|
||||
err := c.startPods(conn, mons)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start new mon %s. %+v", mons[0].Name, err)
|
||||
}
|
||||
|
||||
// Remove the mon pod if it is still there
|
||||
var gracePeriod int64
|
||||
options := &v1.DeleteOptions{GracePeriodSeconds: &gracePeriod}
|
||||
err = c.clientset.CoreV1().Pods(c.Namespace).Delete(name, options)
|
||||
if err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
logger.Infof("dead mon pod %s was already gone", name)
|
||||
} else {
|
||||
return fmt.Errorf("failed to remove dead mon pod %s. %+v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Remove the bad monitor from quorum
|
||||
err = mon.RemoveMonitorFromQuorum(conn, name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to remove mon %s from quorum. %+v", name, err)
|
||||
}
|
||||
delete(c.clusterInfo.Monitors, name)
|
||||
err = c.saveMonConfig()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save mon config after failing mon %s. %+v", name, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Retrieve the ceph cluster info if it already exists.
|
||||
// If a new cluster create new keys.
|
||||
func (c *Cluster) initClusterInfo() error {
|
||||
// get the cluster secrets
|
||||
secrets, err := c.clientset.CoreV1().Secrets(c.Namespace).Get(appName)
|
||||
if err != nil {
|
||||
if !errors.IsNotFound(err) {
|
||||
return fmt.Errorf("failed to get mon secrets. %+v", err)
|
||||
}
|
||||
|
||||
return c.createMonSecretsAndSave()
|
||||
err = c.createMonSecretsAndSave()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
c.clusterInfo = &mon.ClusterInfo{
|
||||
Name: string(secrets.Data[clusterSecretName]),
|
||||
FSID: string(secrets.Data[fsidSecretName]),
|
||||
MonitorSecret: string(secrets.Data[monSecretName]),
|
||||
AdminSecret: string(secrets.Data[adminSecretName]),
|
||||
}
|
||||
logger.Debugf("found existing monitor secrets for cluster %s", c.clusterInfo.Name)
|
||||
}
|
||||
|
||||
c.clusterInfo = &mon.ClusterInfo{
|
||||
Name: string(secrets.Data[clusterSecretName]),
|
||||
FSID: string(secrets.Data[fsidSecretName]),
|
||||
MonitorSecret: string(secrets.Data[monSecretName]),
|
||||
AdminSecret: string(secrets.Data[adminSecretName]),
|
||||
Monitors: map[string]*mon.CephMonitorConfig{},
|
||||
// get the existing monitor config
|
||||
err = c.loadMonConfig()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get mon config. %+v", err)
|
||||
}
|
||||
logger.Infof("found existing monitor secrets for cluster %s", c.clusterInfo.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) getExpectedMonConfig() []*MonConfig {
|
||||
mons := []*MonConfig{}
|
||||
|
||||
// initialize the mon pod info for mons that have been previously created
|
||||
for _, monitor := range c.clusterInfo.Monitors {
|
||||
mons = append(mons, &MonConfig{Name: monitor.Name, Port: int32(mon.Port)})
|
||||
}
|
||||
|
||||
// initialize mon info if we don't have enough mons (at first startup)
|
||||
for i := len(c.clusterInfo.Monitors); i < c.Size; i++ {
|
||||
c.maxMonID++
|
||||
mons = append(mons, &MonConfig{Name: fmt.Sprintf("mon%d", c.maxMonID), Port: int32(mon.Port)})
|
||||
}
|
||||
|
||||
return mons
|
||||
}
|
||||
|
||||
// get the ID of a monitor from its name
|
||||
func getMonID(name string) (int, error) {
|
||||
if strings.Index(name, "mon") != 0 || len(name) < 4 {
|
||||
return -1, fmt.Errorf("unexpected mon name")
|
||||
}
|
||||
id, err := strconv.Atoi(name[3:])
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) createMonSecretsAndSave() error {
|
||||
logger.Infof("creating mon secrets for a new cluster")
|
||||
var err error
|
||||
@@ -225,40 +352,20 @@ func (c *Cluster) createMonSecretsAndSave() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) startPods(mons []*MonConfig) error {
|
||||
func (c *Cluster) startPods(conn client.Connection, mons []*MonConfig) error {
|
||||
// schedule the mons on different nodes if we have enough nodes to be unique
|
||||
antiAffinity, err := c.getAntiAffinity()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get antiaffinity. %+v", err)
|
||||
}
|
||||
|
||||
running, pending, err := c.pollPods()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get mon pods. %+v", err)
|
||||
}
|
||||
logger.Infof("%d running, %d pending pods", len(running), len(pending))
|
||||
|
||||
c.clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{}
|
||||
for _, m := range running {
|
||||
c.clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, m.Status.PodIP)
|
||||
}
|
||||
|
||||
err = c.saveMonEndpoints()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save endpoints for %d running mons. %+v", len(c.clusterInfo.Monitors), err)
|
||||
}
|
||||
|
||||
if len(running) == c.Size {
|
||||
logger.Infof("pods are already running")
|
||||
return nil
|
||||
}
|
||||
|
||||
started := 0
|
||||
preexisted := len(c.clusterInfo.Monitors)
|
||||
created := 0
|
||||
alreadyRunning := 0
|
||||
for _, m := range mons {
|
||||
monPod := c.makeMonPod(m, antiAffinity)
|
||||
name := monPod.Name
|
||||
logger.Debugf("Starting pod: %+v", monPod)
|
||||
name := monPod.Name
|
||||
_, err = c.clientset.CoreV1().Pods(c.Namespace).Create(monPod)
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
@@ -267,7 +374,7 @@ func (c *Cluster) startPods(mons []*MonConfig) error {
|
||||
logger.Infof("pod %s already exists", name)
|
||||
alreadyRunning++
|
||||
} else {
|
||||
started++
|
||||
created++
|
||||
}
|
||||
|
||||
podIP, err := c.waitForPodToStart(name)
|
||||
@@ -276,21 +383,55 @@ func (c *Cluster) startPods(mons []*MonConfig) error {
|
||||
}
|
||||
c.clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, podIP)
|
||||
|
||||
err = c.saveMonEndpoints()
|
||||
err = c.saveMonConfig()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save endpoints after starting mon %s. %+v", m.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("started %d/%d mons (%d already running)", (started + alreadyRunning), c.Size, alreadyRunning)
|
||||
logger.Infof("mons created: %d, alreadyRunning: %d, preexisted: %d", created, alreadyRunning, preexisted)
|
||||
|
||||
return c.waitForMonsToJoin(conn, mons)
|
||||
}
|
||||
|
||||
func (c *Cluster) waitForMonsToJoin(conn client.Connection, mons []*MonConfig) error {
|
||||
if !c.waitForStart {
|
||||
return nil
|
||||
}
|
||||
|
||||
// initialize a connection if it is not already connected
|
||||
if conn == nil {
|
||||
context := &clusterd.Context{ConfigDir: k8sutil.DataDir}
|
||||
var err error
|
||||
conn, err = mon.ConnectToClusterAsAdmin(context, c.factory, c.clusterInfo)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot connect to cluster. %+v", err)
|
||||
}
|
||||
defer conn.Shutdown()
|
||||
}
|
||||
|
||||
starting := []string{}
|
||||
for _, m := range mons {
|
||||
starting = append(starting, m.Name)
|
||||
}
|
||||
|
||||
// wait for the monitors to join quorum
|
||||
err := mon.WaitForQuorumWithConnection(conn, starting)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to wait for mon quorum. %+v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) waitForPodToStart(name string) (string, error) {
|
||||
if !c.waitForStart {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// Poll the status of the pods to see if they are ready
|
||||
status := ""
|
||||
for i := 0; i < 15; i++ {
|
||||
for i := 0; i < c.maxRetries; i++ {
|
||||
// wait and try again
|
||||
logger.Infof("waiting %ds for pod %s to start. status=%s", c.retryDelay, name, status)
|
||||
<-time.After(time.Duration(c.retryDelay) * time.Second)
|
||||
@@ -310,7 +451,7 @@ func (c *Cluster) waitForPodToStart(name string) (string, error) {
|
||||
return "", fmt.Errorf("timed out waiting for pod %s to start", name)
|
||||
}
|
||||
|
||||
func (c *Cluster) saveMonEndpoints() error {
|
||||
func (c *Cluster) saveMonConfig() error {
|
||||
configMap := &v1.ConfigMap{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: monConfigMapName,
|
||||
@@ -320,6 +461,7 @@ func (c *Cluster) saveMonEndpoints() error {
|
||||
}
|
||||
configMap.Data = map[string]string{
|
||||
monEndpointKey: mon.FlattenMonEndpoints(c.clusterInfo.Monitors),
|
||||
maxMonIDKey: strconv.Itoa(c.maxMonID),
|
||||
}
|
||||
|
||||
_, err := c.clientset.CoreV1().ConfigMaps(c.Namespace).Create(configMap)
|
||||
@@ -338,6 +480,45 @@ func (c *Cluster) saveMonEndpoints() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) loadMonConfig() error {
|
||||
cm, err := c.clientset.CoreV1().ConfigMaps(c.Namespace).Get(monConfigMapName)
|
||||
if err != nil {
|
||||
if !errors.IsNotFound(err) {
|
||||
return err
|
||||
}
|
||||
// If the config map was not found, initialize the empty set of monitors
|
||||
c.maxMonID = -1
|
||||
c.clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{}
|
||||
return c.saveMonConfig()
|
||||
}
|
||||
|
||||
// Parse the monitor List
|
||||
if info, ok := cm.Data[monEndpointKey]; ok {
|
||||
c.clusterInfo.Monitors = mon.ParseMonEndpoints(info)
|
||||
} else {
|
||||
c.clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{}
|
||||
}
|
||||
|
||||
// Parse the max monitor id
|
||||
if id, ok := cm.Data[maxMonIDKey]; ok {
|
||||
c.maxMonID, err = strconv.Atoi(id)
|
||||
if err != nil {
|
||||
logger.Errorf("invalid max mon id %s. %+v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Make sure the max id is consistent with the current monitors
|
||||
for _, m := range c.clusterInfo.Monitors {
|
||||
id, _ := getMonID(m.Name)
|
||||
if c.maxMonID < id {
|
||||
c.maxMonID = id
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("loaded: maxMonID=%d, mons=%+v", c.maxMonID, c.clusterInfo.Monitors)
|
||||
return nil
|
||||
}
|
||||
|
||||
// detect whether we have a big enough cluster to run services on different nodes.
|
||||
// the anti-affinity will prevent pods of the same type of running on the same node.
|
||||
func (c *Cluster) getAntiAffinity() (bool, error) {
|
||||
|
||||
@@ -20,9 +20,12 @@ import (
|
||||
|
||||
"k8s.io/client-go/pkg/api/v1"
|
||||
|
||||
"github.com/rook/rook/pkg/cephmgr/client"
|
||||
testclient "github.com/rook/rook/pkg/cephmgr/client/test"
|
||||
"github.com/rook/rook/pkg/operator/test"
|
||||
|
||||
"os"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
@@ -30,6 +33,7 @@ func TestStartMonPods(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
factory := &testclient.MockConnectionFactory{Fsid: "fsid", SecretKey: "mysecret"}
|
||||
c := New(clientset, factory, "ns", "", "myversion")
|
||||
c.maxRetries = 1
|
||||
c.retryDelay = 0
|
||||
|
||||
// start a basic cluster
|
||||
@@ -75,7 +79,7 @@ func validateStart(t *testing.T, c *Cluster) {
|
||||
|
||||
func TestSaveMonEndpoints(t *testing.T) {
|
||||
clientset := test.New(1)
|
||||
c := New(clientset, nil, "ns", "myversion")
|
||||
c := New(clientset, nil, "ns", "", "myversion")
|
||||
c.clusterInfo = test.CreateClusterInfo(1)
|
||||
|
||||
// create the initial config map
|
||||
@@ -94,5 +98,67 @@ func TestSaveMonEndpoints(t *testing.T) {
|
||||
cm, err = c.clientset.CoreV1().ConfigMaps(c.Namespace).Get("mon-config")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "mon1=2.3.4.5:6790", cm.Data["endpoints"])
|
||||
}
|
||||
|
||||
func TestCheckHealth(t *testing.T) {
|
||||
clientset := test.New(1)
|
||||
factory := &testclient.MockConnectionFactory{Fsid: "fsid", SecretKey: "mysecret"}
|
||||
c := New(clientset, factory, "ns", "", "myversion")
|
||||
c.retryDelay = 1
|
||||
c.maxRetries = 1
|
||||
c.clusterInfo = test.CreateClusterInfo(1)
|
||||
c.configDir = "/tmp/healthtest"
|
||||
c.waitForStart = false
|
||||
defer os.RemoveAll(c.configDir)
|
||||
|
||||
err := c.CheckHealth()
|
||||
assert.Nil(t, err)
|
||||
|
||||
c.maxMonID = 10
|
||||
conn, err := factory.NewConnWithClusterAndUser(c.Namespace, "admin")
|
||||
defer conn.Shutdown()
|
||||
err = c.failoverMon(conn, "mon1")
|
||||
assert.Nil(t, err)
|
||||
|
||||
cm, err := c.clientset.CoreV1().ConfigMaps(c.Namespace).Get("mon-config")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "mon11=:6790", cm.Data["endpoints"])
|
||||
}
|
||||
|
||||
func TestMonInQuourm(t *testing.T) {
|
||||
entry := client.MonMapEntry{Name: "foo", Rank: 23}
|
||||
quorum := []int{}
|
||||
// Nothing in quorum
|
||||
assert.False(t, monInQuorum(entry, quorum))
|
||||
|
||||
// One or more members in quorum
|
||||
quorum = []int{23}
|
||||
assert.True(t, monInQuorum(entry, quorum))
|
||||
quorum = []int{5, 6, 7, 23, 8}
|
||||
assert.True(t, monInQuorum(entry, quorum))
|
||||
|
||||
// Not in quorum
|
||||
entry.Rank = 1
|
||||
assert.False(t, monInQuorum(entry, quorum))
|
||||
}
|
||||
|
||||
func TestMonID(t *testing.T) {
|
||||
// invalid
|
||||
id, err := getMonID("m")
|
||||
assert.NotNil(t, err)
|
||||
assert.Equal(t, -1, id)
|
||||
id, err = getMonID("mon")
|
||||
assert.NotNil(t, err)
|
||||
assert.Equal(t, -1, id)
|
||||
id, err = getMonID("monitor0")
|
||||
assert.NotNil(t, err)
|
||||
assert.Equal(t, -1, id)
|
||||
|
||||
// valid
|
||||
id, err = getMonID("mon0")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, id)
|
||||
id, err = getMonID("mon123")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 123, id)
|
||||
}
|
||||
@@ -52,7 +52,7 @@ func (c *Cluster) getLabels() map[string]string {
|
||||
|
||||
func (c *Cluster) makeMonPod(config *MonConfig, antiAffinity bool) *v1.Pod {
|
||||
|
||||
container := c.monContainer(config, clusterInfo.FSID)
|
||||
container := c.monContainer(config, c.clusterInfo.FSID)
|
||||
dataDirSource := v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}
|
||||
if c.dataDirHostPath != "" {
|
||||
dataDirSource = v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: c.dataDirHostPath}}
|
||||
@@ -112,8 +112,12 @@ func (c *Cluster) monContainer(config *MonConfig, fsid string) v1.Container {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) getPods() (*v1.PodList, error) {
|
||||
return c.clientset.CoreV1().Pods(c.Namespace).List(c.listOptions())
|
||||
}
|
||||
|
||||
func (c *Cluster) pollPods() ([]*v1.Pod, []*v1.Pod, error) {
|
||||
podList, err := c.clientset.CoreV1().Pods(c.Namespace).List(c.listOptions())
|
||||
podList, err := c.getPods()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to list running pods: %v", err)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user