diff --git a/pkg/cephmgr/mon/config.go b/pkg/cephmgr/mon/config.go index 88ad8f206..7a79a9f61 100644 --- a/pkg/cephmgr/mon/config.go +++ b/pkg/cephmgr/mon/config.go @@ -251,7 +251,7 @@ func ConnectToCluster(context *clusterd.Context, factory client.ConnectionFactor return nil, fmt.Errorf("failed to connect to cluster %s: %+v", cluster.Name, err) } - logger.Infof("successfully connected to cluster %s", cluster.Name) + logger.Infof("successfully connected to cluster %s with user %s", cluster.Name, user) return conn, nil } diff --git a/pkg/cephmgr/mon/leader.go b/pkg/cephmgr/mon/leader.go index b7c2189cb..a49578ea8 100644 --- a/pkg/cephmgr/mon/leader.go +++ b/pkg/cephmgr/mon/leader.go @@ -109,21 +109,28 @@ func removeMonitorsFromQuorum(factory client.ConnectionFactory, context *cluster defer conn.Shutdown() logger.Infof("removing %d monitors from quorum", len(monitors)) - for monID, mon := range monitors { - logger.Debugf("removing monitor %s (%v)", monID, mon) - - cmd := map[string]interface{}{"prefix": "mon remove", "name": mon.Name} - _, err = client.ExecuteMonCommand(conn, cmd, "mon remove") - if err != nil { - return fmt.Errorf("mon remove failed: %+v", err) + for _, mon := range monitors { + if err := RemoveMonitorFromQuorum(conn, mon.Name); err != nil { + return err } - - logger.Infof("removed monitor %s from node %s", mon.Name, monID) } return nil } +func RemoveMonitorFromQuorum(conn client.Connection, name string) error { + logger.Debugf("removing monitor %s", name) + + cmd := map[string]interface{}{"prefix": "mon remove", "name": name} + _, err := client.ExecuteMonCommand(conn, cmd, "mon remove") + if err != nil { + return fmt.Errorf("mon %s remove failed: %+v", name, err) + } + + logger.Infof("removed monitor %s", name) + return nil +} + // extract the nodeIDs from the mon map func monIDs(mons map[string]*CephMonitorConfig) []string { nodes := []string{} @@ -357,8 +364,6 @@ func calculateMonitorCount(nodeCount int) int { } func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context, cluster *ClusterInfo) error { - logger.Infof("waiting for mon quorum...") - // open an admin connection to the cluster adminConn, err := ConnectToClusterAsAdmin(context, factory, cluster) if err != nil { @@ -366,6 +371,17 @@ func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context, } defer adminConn.Shutdown() + mons := []string{} + for _, m := range cluster.Monitors { + mons = append(mons, m.Name) + } + return WaitForQuorumWithConnection(adminConn, mons) +} + +// Wait for all of the given monitors to form quorum +func WaitForQuorumWithConnection(conn client.Connection, mons []string) error { + logger.Infof("waiting for mon quorum") + // wait for monitors to establish quorum retryCount := 0 retryMax := 20 @@ -383,7 +399,7 @@ func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context, // get the mon_status response that contains info about all monitors in the mon map and // their quorum status - monStatusResp, err := client.GetMonStatus(adminConn) + monStatusResp, err := client.GetMonStatus(conn) if err != nil { logger.Debugf("failed to get mon_status, err: %+v", err) continue @@ -391,11 +407,11 @@ func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context, // check if each of the initial monitors is in quorum allInQuorum := true - for _, im := range cluster.Monitors { + for _, name := range mons { // first get the initial monitors corresponding mon map entry var monMapEntry *client.MonMapEntry for i := range monStatusResp.MonMap.Mons { - if im.Name == monStatusResp.MonMap.Mons[i].Name { + if name == monStatusResp.MonMap.Mons[i].Name { monMapEntry = &monStatusResp.MonMap.Mons[i] break } @@ -403,7 +419,7 @@ func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context, if monMapEntry == nil { // found an initial monitor that is not in the mon map, bail out of this retry - logger.Warningf("failed to find initial monitor %s in mon map", im.Name) + logger.Warningf("failed to find initial monitor %s in mon map", name) allInQuorum = false break } @@ -420,7 +436,7 @@ func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context, if !inQuorumList { // found an initial monitor that is not in quorum, bail out of this retry - logger.Warningf("initial monitor %s is not in quorum list", im.Name) + logger.Warningf("initial monitor %s is not in quorum list", name) allInQuorum = false break } diff --git a/pkg/operator/api/api.go b/pkg/operator/api/api.go index 6006b6651..a74be39c0 100644 --- a/pkg/operator/api/api.go +++ b/pkg/operator/api/api.go @@ -17,6 +17,7 @@ package api import ( "fmt" + "strings" "github.com/rook/rook/pkg/model" "github.com/rook/rook/pkg/operator/k8sutil" @@ -98,15 +99,17 @@ func (c *Cluster) makeDeployment() *extensions.Deployment { } func (c *Cluster) apiContainer() v1.Container { - // need a different prefix on the env vars - monSecretVar := opmon.MonSecretEnvVar() - adminSecretVar := opmon.AdminSecretEnvVar() - endpointVar := opmon.MonEndpointEnvVar() - clusterNameVar := opmon.ClusterNameEnvVar() - monSecretVar.Name = "ROOK_OPERATOR_MON_SECRET" - adminSecretVar.Name = "ROOK_OPERATOR_ADMIN_SECRET" - endpointVar.Name = "ROOK_OPERATOR_MON_ENDPOINTS" - clusterNameVar.Name = "ROOK_OPERATOR_CLUSTER_NAME" + envVars := []v1.EnvVar{ + k8sutil.NamespaceEnvVar(), + opmon.MonSecretEnvVar(), + opmon.AdminSecretEnvVar(), + opmon.MonEndpointEnvVar(), + opmon.ClusterNameEnvVar(), + } + // replace the ROOKD prefix with ROOK_OPERATOR + for i := range envVars { + envVars[i].Name = strings.Replace(envVars[i].Name, "ROOKD_", "ROOK_OPERATOR_", 1) + } command := fmt.Sprintf("/usr/bin/rook-operator api --data-dir=%s --api-port=%d --container-version=%s", k8sutil.DataDir, model.Port, c.Version) @@ -119,13 +122,7 @@ func (c *Cluster) apiContainer() v1.Container { VolumeMounts: []v1.VolumeMount{ {Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir}, }, - Env: []v1.EnvVar{ - k8sutil.NamespaceEnvVar(), - clusterNameVar, - endpointVar, - monSecretVar, - adminSecretVar, - }, + Env: envVars, } } diff --git a/pkg/operator/api/api_test.go b/pkg/operator/api/api_test.go index 05cc67e33..4929946fb 100644 --- a/pkg/operator/api/api_test.go +++ b/pkg/operator/api/api_test.go @@ -75,7 +75,9 @@ func TestPodSpecs(t *testing.T) { assert.Equal(t, "quay.io/rook/rook-operator:myversion", cont.Image) assert.Equal(t, 1, len(cont.VolumeMounts)) assert.Equal(t, 5, len(cont.Env)) - + for _, v := range cont.Env { + assert.True(t, strings.HasPrefix(v.Name, "ROOK_OPERATOR_")) + } expectedCommand := fmt.Sprintf("/usr/bin/rook-operator api --data-dir=/var/lib/rook --api-port=%d --container-version=%s", model.Port, c.Version) diff --git a/pkg/operator/cluster/cluster.go b/pkg/operator/cluster/cluster.go index 11d4df34e..afbfb58a5 100644 --- a/pkg/operator/cluster/cluster.go +++ b/pkg/operator/cluster/cluster.go @@ -42,7 +42,7 @@ const ( ) var ( - healthCheckInterval = 8 * time.Second + healthCheckInterval = 10 * time.Second logger = capnslog.NewPackageLogger("github.com/rook/rook-operator", "op-cluster") ) @@ -115,7 +115,7 @@ func (c *Cluster) Monitor(stopCh <-chan struct{}) { return case <-time.After(healthCheckInterval): - logger.Infof("checking health of mons") + logger.Debugf("checking health of mons") err := c.mons.CheckHealth() if err != nil { logger.Infof("failed to check mon health. %+v", err) diff --git a/pkg/operator/mon/mon.go b/pkg/operator/mon/mon.go index 60e177df8..a951ded79 100644 --- a/pkg/operator/mon/mon.go +++ b/pkg/operator/mon/mon.go @@ -17,6 +17,8 @@ package mon import ( "fmt" + "strconv" + "strings" "time" "github.com/rook/rook/pkg/cephmgr/client" @@ -39,6 +41,7 @@ const ( clusterSecretName = "cluster-name" monConfigMapName = "mon-config" monEndpointKey = "endpoints" + maxMonIDKey = "maxMonId" ) type Cluster struct { @@ -54,7 +57,11 @@ type Cluster struct { clientset kubernetes.Interface factory client.ConnectionFactory retryDelay int + maxRetries int clusterInfo *mon.ClusterInfo + maxMonID int + configDir string + waitForStart bool dataDirHostPath string } @@ -73,6 +80,10 @@ func New(clientset kubernetes.Interface, factory client.ConnectionFactory, names Size: 3, AntiAffinity: true, retryDelay: 6, + maxRetries: 15, + maxMonID: -1, + configDir: k8sutil.DataDir, + waitForStart: true, } } @@ -83,33 +94,48 @@ func (c *Cluster) Start() (*mon.ClusterInfo, error) { if err != nil { return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err) } - mons := []*MonConfig{} - for i := 0; i < c.Size; i++ { - mons = append(mons, &MonConfig{Name: fmt.Sprintf("mon%d", i), Port: int32(mon.Port)}) - } - err = c.startPods(mons) - if err != nil { - return nil, fmt.Errorf("failed to start mon pods. %+v", err) - } - - context := &clusterd.Context{ConfigDir: k8sutil.DataDir} - err = mon.WaitForQuorum(c.factory, context, c.clusterInfo) - if err != nil { - return nil, fmt.Errorf("failed to wait for mon quorum. %+v", err) + if len(c.clusterInfo.Monitors) == 0 { + // Start the initial monitors at startup + mons := c.getExpectedMonConfig() + err = c.startPods(nil, mons) + if err != nil { + return nil, fmt.Errorf("failed to start mon pods. %+v", err) + } + } else { + // Check the health of a previously started cluster + err = c.CheckHealth() + if err != nil { + logger.Warningf("failed to check mon health %+v. %+v", c.clusterInfo.Monitors, err) + } } return c.clusterInfo, nil } func (c *Cluster) CheckHealth() error { - context := &clusterd.Context{ConfigDir: k8sutil.DataDir} + // update the config map if the pod ips changed + // must retry since during startup of pods they might take some time to initialize + k8sutil.Retry(time.Duration(c.retryDelay)*time.Second, c.maxRetries, func() (bool, error) { + // TODO: There is more work to get the reboot functional. The mons are not + // happy if their ip address changes. They expect a constant id. + err := c.updateConfigMapIfPodIPsChanged() + if err != nil { + logger.Infof("unable to check on mon pods. %+v", err) + return false, nil + } + return true, nil + }) + + // connect to the mons + context := &clusterd.Context{ConfigDir: c.configDir} conn, err := mon.ConnectToClusterAsAdmin(context, c.factory, c.clusterInfo) if err != nil { return fmt.Errorf("cannot connect to cluster. %+v", err) } defer conn.Shutdown() + // get the status and check for quorum status, err := client.GetMonStatus(conn) if err != nil { return fmt.Errorf("failed to get mon status. %+v", err) @@ -118,20 +144,11 @@ func (c *Cluster) CheckHealth() error { for _, monitor := range status.MonMap.Mons { inQuorum := monInQuorum(monitor, status.Quorum) if inQuorum { - logger.Infof("mon %s found in quorum", monitor.Name) + logger.Debugf("mon %s found in quorum", monitor.Name) } else { - logger.Infof("mon %s NOT found in quroum. %+v", monitor.Name, status.Quorum) - } + logger.Warningf("mon %s NOT found in quroum. %+v", monitor.Name, status.Quorum) - pod, err := c.clientset.CoreV1().Pods(c.Namespace).Get(monitor.Name) - if err != nil { - logger.Infof("Get mon %s pod failed. %+v", monitor.Name, err) - } else { - logger.Infof("mon %s pod status: %+v", monitor.Name, pod.Status.Phase) - } - - if !inQuorum { - err = c.failoverMon(monitor.Name) + err = c.failoverMon(conn, monitor.Name) if err != nil { logger.Errorf("failed to failover mon %s. %+v", monitor.Name, err) } @@ -141,6 +158,45 @@ func (c *Cluster) CheckHealth() error { return nil } +func (c *Cluster) updateConfigMapIfPodIPsChanged() error { + pods, err := c.getPods() + if err != nil { + return fmt.Errorf("failed to check if pod ips changed. %+v", err) + } + if len(pods.Items) == 0 { + return fmt.Errorf("where are the mon pods?") + } + + logger.Debugf("there are %d mon pods. checking the podIPs.", len(pods.Items)) + updated := false + for _, pod := range pods.Items { + if pod.Status.PodIP == "" { + return fmt.Errorf("no podIP for mon %s", pod.Name) + } + if pod.Status.Phase != v1.PodRunning { + return fmt.Errorf("pod %s is not running. phase=%v", pod.Name, pod.Status.Phase) + } + m := mon.ToCephMon(pod.Name, pod.Status.PodIP) + existing, ok := c.clusterInfo.Monitors[pod.Name] + if !ok || existing.Endpoint == m.Endpoint { + // the endpoint does not need to be updated + logger.Debugf("Did not need to update pod %s with endpoint %+v. ok=%t, m=%+v", pod.Name, pod.Status.PodIP, ok, m) + continue + } + + logger.Infof("updating mon %s endpoint from %s to %s", pod.Name, existing.Endpoint, m.Endpoint) + c.clusterInfo.Monitors[pod.Name] = m + updated = true + } + + if updated { + return c.saveMonConfig() + } + + logger.Debugf("no update to mon pod ips") + return nil +} + func monInQuorum(monitor client.MonMapEntry, quorum []int) bool { for _, rank := range quorum { if rank == monitor.Rank { @@ -150,34 +206,105 @@ func monInQuorum(monitor client.MonMapEntry, quorum []int) bool { return false } -func (c *Cluster) failoverMon(name string) error { - //logger.Infof("Failing over monitor %s", name) +func (c *Cluster) failoverMon(conn client.Connection, name string) error { + logger.Infof("Failing over monitor %s", name) + + // Start a new monitor + c.maxMonID++ + mons := []*MonConfig{&MonConfig{Name: fmt.Sprintf("mon%d", c.maxMonID), Port: int32(mon.Port)}} + logger.Infof("starting new mon %s", mons[0].Name) + err := c.startPods(conn, mons) + if err != nil { + return fmt.Errorf("failed to start new mon %s. %+v", mons[0].Name, err) + } + + // Remove the mon pod if it is still there + var gracePeriod int64 + options := &v1.DeleteOptions{GracePeriodSeconds: &gracePeriod} + err = c.clientset.CoreV1().Pods(c.Namespace).Delete(name, options) + if err != nil { + if errors.IsNotFound(err) { + logger.Infof("dead mon pod %s was already gone", name) + } else { + return fmt.Errorf("failed to remove dead mon pod %s. %+v", name, err) + } + } + + // Remove the bad monitor from quorum + err = mon.RemoveMonitorFromQuorum(conn, name) + if err != nil { + return fmt.Errorf("failed to remove mon %s from quorum. %+v", name, err) + } + delete(c.clusterInfo.Monitors, name) + err = c.saveMonConfig() + if err != nil { + return fmt.Errorf("failed to save mon config after failing mon %s. %+v", name, err) + } + return nil } // Retrieve the ceph cluster info if it already exists. // If a new cluster create new keys. func (c *Cluster) initClusterInfo() error { + // get the cluster secrets secrets, err := c.clientset.CoreV1().Secrets(c.Namespace).Get(appName) if err != nil { if !errors.IsNotFound(err) { return fmt.Errorf("failed to get mon secrets. %+v", err) } - return c.createMonSecretsAndSave() + err = c.createMonSecretsAndSave() + if err != nil { + return err + } + } else { + c.clusterInfo = &mon.ClusterInfo{ + Name: string(secrets.Data[clusterSecretName]), + FSID: string(secrets.Data[fsidSecretName]), + MonitorSecret: string(secrets.Data[monSecretName]), + AdminSecret: string(secrets.Data[adminSecretName]), + } + logger.Debugf("found existing monitor secrets for cluster %s", c.clusterInfo.Name) } - c.clusterInfo = &mon.ClusterInfo{ - Name: string(secrets.Data[clusterSecretName]), - FSID: string(secrets.Data[fsidSecretName]), - MonitorSecret: string(secrets.Data[monSecretName]), - AdminSecret: string(secrets.Data[adminSecretName]), - Monitors: map[string]*mon.CephMonitorConfig{}, + // get the existing monitor config + err = c.loadMonConfig() + if err != nil { + return fmt.Errorf("failed to get mon config. %+v", err) } - logger.Infof("found existing monitor secrets for cluster %s", c.clusterInfo.Name) return nil } +func (c *Cluster) getExpectedMonConfig() []*MonConfig { + mons := []*MonConfig{} + + // initialize the mon pod info for mons that have been previously created + for _, monitor := range c.clusterInfo.Monitors { + mons = append(mons, &MonConfig{Name: monitor.Name, Port: int32(mon.Port)}) + } + + // initialize mon info if we don't have enough mons (at first startup) + for i := len(c.clusterInfo.Monitors); i < c.Size; i++ { + c.maxMonID++ + mons = append(mons, &MonConfig{Name: fmt.Sprintf("mon%d", c.maxMonID), Port: int32(mon.Port)}) + } + + return mons +} + +// get the ID of a monitor from its name +func getMonID(name string) (int, error) { + if strings.Index(name, "mon") != 0 || len(name) < 4 { + return -1, fmt.Errorf("unexpected mon name") + } + id, err := strconv.Atoi(name[3:]) + if err != nil { + return -1, err + } + return id, nil +} + func (c *Cluster) createMonSecretsAndSave() error { logger.Infof("creating mon secrets for a new cluster") var err error @@ -225,40 +352,20 @@ func (c *Cluster) createMonSecretsAndSave() error { return nil } -func (c *Cluster) startPods(mons []*MonConfig) error { +func (c *Cluster) startPods(conn client.Connection, mons []*MonConfig) error { // schedule the mons on different nodes if we have enough nodes to be unique antiAffinity, err := c.getAntiAffinity() if err != nil { return fmt.Errorf("failed to get antiaffinity. %+v", err) } - running, pending, err := c.pollPods() - if err != nil { - return fmt.Errorf("failed to get mon pods. %+v", err) - } - logger.Infof("%d running, %d pending pods", len(running), len(pending)) - - c.clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{} - for _, m := range running { - c.clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, m.Status.PodIP) - } - - err = c.saveMonEndpoints() - if err != nil { - return fmt.Errorf("failed to save endpoints for %d running mons. %+v", len(c.clusterInfo.Monitors), err) - } - - if len(running) == c.Size { - logger.Infof("pods are already running") - return nil - } - - started := 0 + preexisted := len(c.clusterInfo.Monitors) + created := 0 alreadyRunning := 0 for _, m := range mons { monPod := c.makeMonPod(m, antiAffinity) - name := monPod.Name logger.Debugf("Starting pod: %+v", monPod) + name := monPod.Name _, err = c.clientset.CoreV1().Pods(c.Namespace).Create(monPod) if err != nil { if !errors.IsAlreadyExists(err) { @@ -267,7 +374,7 @@ func (c *Cluster) startPods(mons []*MonConfig) error { logger.Infof("pod %s already exists", name) alreadyRunning++ } else { - started++ + created++ } podIP, err := c.waitForPodToStart(name) @@ -276,21 +383,55 @@ func (c *Cluster) startPods(mons []*MonConfig) error { } c.clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, podIP) - err = c.saveMonEndpoints() + err = c.saveMonConfig() if err != nil { return fmt.Errorf("failed to save endpoints after starting mon %s. %+v", m.Name, err) } } - logger.Infof("started %d/%d mons (%d already running)", (started + alreadyRunning), c.Size, alreadyRunning) + logger.Infof("mons created: %d, alreadyRunning: %d, preexisted: %d", created, alreadyRunning, preexisted) + + return c.waitForMonsToJoin(conn, mons) +} + +func (c *Cluster) waitForMonsToJoin(conn client.Connection, mons []*MonConfig) error { + if !c.waitForStart { + return nil + } + + // initialize a connection if it is not already connected + if conn == nil { + context := &clusterd.Context{ConfigDir: k8sutil.DataDir} + var err error + conn, err = mon.ConnectToClusterAsAdmin(context, c.factory, c.clusterInfo) + if err != nil { + return fmt.Errorf("cannot connect to cluster. %+v", err) + } + defer conn.Shutdown() + } + + starting := []string{} + for _, m := range mons { + starting = append(starting, m.Name) + } + + // wait for the monitors to join quorum + err := mon.WaitForQuorumWithConnection(conn, starting) + if err != nil { + return fmt.Errorf("failed to wait for mon quorum. %+v", err) + } + return nil } func (c *Cluster) waitForPodToStart(name string) (string, error) { + if !c.waitForStart { + return "", nil + } // Poll the status of the pods to see if they are ready status := "" - for i := 0; i < 15; i++ { + for i := 0; i < c.maxRetries; i++ { // wait and try again logger.Infof("waiting %ds for pod %s to start. status=%s", c.retryDelay, name, status) <-time.After(time.Duration(c.retryDelay) * time.Second) @@ -310,7 +451,7 @@ func (c *Cluster) waitForPodToStart(name string) (string, error) { return "", fmt.Errorf("timed out waiting for pod %s to start", name) } -func (c *Cluster) saveMonEndpoints() error { +func (c *Cluster) saveMonConfig() error { configMap := &v1.ConfigMap{ ObjectMeta: v1.ObjectMeta{ Name: monConfigMapName, @@ -320,6 +461,7 @@ func (c *Cluster) saveMonEndpoints() error { } configMap.Data = map[string]string{ monEndpointKey: mon.FlattenMonEndpoints(c.clusterInfo.Monitors), + maxMonIDKey: strconv.Itoa(c.maxMonID), } _, err := c.clientset.CoreV1().ConfigMaps(c.Namespace).Create(configMap) @@ -338,6 +480,45 @@ func (c *Cluster) saveMonEndpoints() error { return nil } +func (c *Cluster) loadMonConfig() error { + cm, err := c.clientset.CoreV1().ConfigMaps(c.Namespace).Get(monConfigMapName) + if err != nil { + if !errors.IsNotFound(err) { + return err + } + // If the config map was not found, initialize the empty set of monitors + c.maxMonID = -1 + c.clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{} + return c.saveMonConfig() + } + + // Parse the monitor List + if info, ok := cm.Data[monEndpointKey]; ok { + c.clusterInfo.Monitors = mon.ParseMonEndpoints(info) + } else { + c.clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{} + } + + // Parse the max monitor id + if id, ok := cm.Data[maxMonIDKey]; ok { + c.maxMonID, err = strconv.Atoi(id) + if err != nil { + logger.Errorf("invalid max mon id %s. %+v", id, err) + } + } + + // Make sure the max id is consistent with the current monitors + for _, m := range c.clusterInfo.Monitors { + id, _ := getMonID(m.Name) + if c.maxMonID < id { + c.maxMonID = id + } + } + + logger.Infof("loaded: maxMonID=%d, mons=%+v", c.maxMonID, c.clusterInfo.Monitors) + return nil +} + // detect whether we have a big enough cluster to run services on different nodes. // the anti-affinity will prevent pods of the same type of running on the same node. func (c *Cluster) getAntiAffinity() (bool, error) { diff --git a/pkg/operator/mon/mon_test.go b/pkg/operator/mon/mon_test.go index ea1fa9a97..1f1e46c90 100644 --- a/pkg/operator/mon/mon_test.go +++ b/pkg/operator/mon/mon_test.go @@ -20,9 +20,12 @@ import ( "k8s.io/client-go/pkg/api/v1" + "github.com/rook/rook/pkg/cephmgr/client" testclient "github.com/rook/rook/pkg/cephmgr/client/test" "github.com/rook/rook/pkg/operator/test" + "os" + "github.com/stretchr/testify/assert" ) @@ -30,6 +33,7 @@ func TestStartMonPods(t *testing.T) { clientset := test.New(3) factory := &testclient.MockConnectionFactory{Fsid: "fsid", SecretKey: "mysecret"} c := New(clientset, factory, "ns", "", "myversion") + c.maxRetries = 1 c.retryDelay = 0 // start a basic cluster @@ -75,7 +79,7 @@ func validateStart(t *testing.T, c *Cluster) { func TestSaveMonEndpoints(t *testing.T) { clientset := test.New(1) - c := New(clientset, nil, "ns", "myversion") + c := New(clientset, nil, "ns", "", "myversion") c.clusterInfo = test.CreateClusterInfo(1) // create the initial config map @@ -94,5 +98,67 @@ func TestSaveMonEndpoints(t *testing.T) { cm, err = c.clientset.CoreV1().ConfigMaps(c.Namespace).Get("mon-config") assert.Nil(t, err) assert.Equal(t, "mon1=2.3.4.5:6790", cm.Data["endpoints"]) +} +func TestCheckHealth(t *testing.T) { + clientset := test.New(1) + factory := &testclient.MockConnectionFactory{Fsid: "fsid", SecretKey: "mysecret"} + c := New(clientset, factory, "ns", "", "myversion") + c.retryDelay = 1 + c.maxRetries = 1 + c.clusterInfo = test.CreateClusterInfo(1) + c.configDir = "/tmp/healthtest" + c.waitForStart = false + defer os.RemoveAll(c.configDir) + + err := c.CheckHealth() + assert.Nil(t, err) + + c.maxMonID = 10 + conn, err := factory.NewConnWithClusterAndUser(c.Namespace, "admin") + defer conn.Shutdown() + err = c.failoverMon(conn, "mon1") + assert.Nil(t, err) + + cm, err := c.clientset.CoreV1().ConfigMaps(c.Namespace).Get("mon-config") + assert.Nil(t, err) + assert.Equal(t, "mon11=:6790", cm.Data["endpoints"]) +} + +func TestMonInQuourm(t *testing.T) { + entry := client.MonMapEntry{Name: "foo", Rank: 23} + quorum := []int{} + // Nothing in quorum + assert.False(t, monInQuorum(entry, quorum)) + + // One or more members in quorum + quorum = []int{23} + assert.True(t, monInQuorum(entry, quorum)) + quorum = []int{5, 6, 7, 23, 8} + assert.True(t, monInQuorum(entry, quorum)) + + // Not in quorum + entry.Rank = 1 + assert.False(t, monInQuorum(entry, quorum)) +} + +func TestMonID(t *testing.T) { + // invalid + id, err := getMonID("m") + assert.NotNil(t, err) + assert.Equal(t, -1, id) + id, err = getMonID("mon") + assert.NotNil(t, err) + assert.Equal(t, -1, id) + id, err = getMonID("monitor0") + assert.NotNil(t, err) + assert.Equal(t, -1, id) + + // valid + id, err = getMonID("mon0") + assert.Nil(t, err) + assert.Equal(t, 0, id) + id, err = getMonID("mon123") + assert.Nil(t, err) + assert.Equal(t, 123, id) } diff --git a/pkg/operator/mon/pod.go b/pkg/operator/mon/pod.go index 2305e23fb..1f12c9b49 100644 --- a/pkg/operator/mon/pod.go +++ b/pkg/operator/mon/pod.go @@ -52,7 +52,7 @@ func (c *Cluster) getLabels() map[string]string { func (c *Cluster) makeMonPod(config *MonConfig, antiAffinity bool) *v1.Pod { - container := c.monContainer(config, clusterInfo.FSID) + container := c.monContainer(config, c.clusterInfo.FSID) dataDirSource := v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}} if c.dataDirHostPath != "" { dataDirSource = v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: c.dataDirHostPath}} @@ -112,8 +112,12 @@ func (c *Cluster) monContainer(config *MonConfig, fsid string) v1.Container { } } +func (c *Cluster) getPods() (*v1.PodList, error) { + return c.clientset.CoreV1().Pods(c.Namespace).List(c.listOptions()) +} + func (c *Cluster) pollPods() ([]*v1.Pod, []*v1.Pod, error) { - podList, err := c.clientset.CoreV1().Pods(c.Namespace).List(c.listOptions()) + podList, err := c.getPods() if err != nil { return nil, nil, fmt.Errorf("failed to list running pods: %v", err) }