detect if mons are healthy
This commit is contained in:
1 parent
175003ed6a
commit
2d1c150539
10 files changed
+206
-90
No files matched your search
@@ -251,6 +251,7 @@ func ConnectToCluster(context *clusterd.Context, factory client.ConnectionFactor
|
||||
return nil, fmt.Errorf("failed to connect to cluster %s: %+v", cluster.Name, err)
|
||||
}
|
||||
|
||||
logger.Infof("successfully connected to cluster %s", cluster.Name)
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ type Leader struct {
|
||||
}
|
||||
|
||||
func NewLeader() *Leader {
|
||||
return &Leader{waitForQuorum: waitForQuorum}
|
||||
return &Leader{waitForQuorum: WaitForQuorum}
|
||||
}
|
||||
|
||||
// Apply the desired state to the cluster. The context provides all the information needed to make changes to the service.
|
||||
@@ -356,7 +356,8 @@ func calculateMonitorCount(nodeCount int) int {
|
||||
}
|
||||
}
|
||||
|
||||
func waitForQuorum(factory client.ConnectionFactory, context *clusterd.Context, cluster *ClusterInfo) error {
|
||||
func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context, cluster *ClusterInfo) error {
|
||||
logger.Infof("waiting for mon quorum...")
|
||||
|
||||
// open an admin connection to the cluster
|
||||
adminConn, err := ConnectToClusterAsAdmin(context, factory, cluster)
|
||||
|
||||
@@ -16,13 +16,15 @@ limitations under the License.
|
||||
Some of the code below came from https://github.com/coreos/etcd-operator
|
||||
which also has the apache 2.0 license.
|
||||
*/
|
||||
package operator
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"k8s.io/client-go/pkg/api/errors"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/rook/rook/pkg/cephmgr/client"
|
||||
cephmon "github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
@@ -30,41 +32,33 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/operator/mon"
|
||||
"github.com/rook/rook/pkg/operator/osd"
|
||||
"github.com/rook/rook/pkg/operator/rgw"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/pkg/api/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultPool = "rook"
|
||||
)
|
||||
|
||||
var (
|
||||
healthCheckInterval = 8 * time.Second
|
||||
logger = capnslog.NewPackageLogger("github.com/rook/rook-operator", "op-cluster")
|
||||
)
|
||||
|
||||
type Cluster struct {
|
||||
factory client.ConnectionFactory
|
||||
clientset kubernetes.Interface
|
||||
Metadata v1.ObjectMeta `json:"metadata,omitempty"`
|
||||
Spec ClusterSpec `json:"spec"`
|
||||
Spec `json:"spec"`
|
||||
dataDir string
|
||||
mons *mon.Cluster
|
||||
osds *osd.Cluster
|
||||
apis *api.Cluster
|
||||
rgws *rgw.Cluster
|
||||
}
|
||||
|
||||
type ClusterSpec struct {
|
||||
// The namespace where the the rook resources will all be created.
|
||||
Namespace string `json:"namespace"`
|
||||
|
||||
// Version is the expected version of the rook container to run in the cluster.
|
||||
// The rook-operator will eventually make the rook cluster version
|
||||
// equal to the expected version.
|
||||
Version string `json:"version"`
|
||||
|
||||
// Paused is to pause the control of the operator for the rook cluster.
|
||||
Paused bool `json:"paused,omitempty"`
|
||||
|
||||
// Whether to consume all the storage devices found on a machine
|
||||
UseAllDevices bool `json:"useAllDevices"`
|
||||
|
||||
// A regular expression to allow more fine-grained selection of devices on nodes across the cluster
|
||||
DeviceFilter string `json:"deviceFilter"`
|
||||
|
||||
// The path on the host where config and data can be persisted.
|
||||
DataDirHostPath string `json:"dataDirHostPath"`
|
||||
}
|
||||
|
||||
func newCluster(spec ClusterSpec, factory client.ConnectionFactory, clientset kubernetes.Interface) *Cluster {
|
||||
func New(spec Spec, factory client.ConnectionFactory, clientset kubernetes.Interface) *Cluster {
|
||||
return &Cluster{
|
||||
Spec: spec,
|
||||
factory: factory,
|
||||
@@ -85,26 +79,26 @@ func (c *Cluster) CreateInstance() error {
|
||||
}
|
||||
|
||||
// Start the mon pods
|
||||
m := mon.New(c.clientset, c.factory, c.Spec.Namespace, c.Spec.DataDirHostPath, c.Spec.Version)
|
||||
cluster, err := m.Start()
|
||||
c.mons = mon.New(c.clientset, c.factory, c.Spec.Namespace, c.Spec.DataDirHostPath, c.Spec.Version)
|
||||
clusterInfo, err := c.mons.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the mons. %+v", err)
|
||||
}
|
||||
|
||||
a := api.New(c.clientset, c.Spec.Namespace, c.Spec.Version)
|
||||
err = a.Start()
|
||||
c.apis = api.New(c.clientset, c.Spec.Namespace, c.Spec.Version)
|
||||
err = c.apis.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the REST api. %+v", err)
|
||||
}
|
||||
|
||||
// Start the OSDs
|
||||
osds := osd.New(c.clientset, c.Spec.Namespace, c.Spec.Version, c.Spec.DeviceFilter, c.Spec.DataDirHostPath, c.Spec.UseAllDevices)
|
||||
err = osds.Start()
|
||||
c.osds = osd.New(c.clientset, c.Spec.Namespace, c.Spec.Version, c.Spec.DeviceFilter, c.Spec.DataDirHostPath, c.Spec.UseAllDevices)
|
||||
err = c.osds.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the osds. %+v", err)
|
||||
}
|
||||
|
||||
err = c.createClientAccess(cluster)
|
||||
err = c.createClientAccess(clusterInfo)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create client access. %+v", err)
|
||||
}
|
||||
@@ -113,6 +107,23 @@ func (c *Cluster) CreateInstance() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) Monitor(stopCh <-chan struct{}) {
|
||||
for {
|
||||
select {
|
||||
case <-stopCh:
|
||||
logger.Infof("Stopping monitoring of cluster %s", c.Spec.Namespace)
|
||||
return
|
||||
|
||||
case <-time.After(healthCheckInterval):
|
||||
logger.Infof("checking health of mons")
|
||||
err := c.mons.CheckHealth()
|
||||
if err != nil {
|
||||
logger.Infof("failed to check mon health. %+v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) createClientAccess(clusterInfo *cephmon.ClusterInfo) error {
|
||||
context := &clusterd.Context{ConfigDir: c.dataDir}
|
||||
conn, err := cephmon.ConnectToClusterAsAdmin(context, c.factory, clusterInfo)
|
||||
@@ -16,7 +16,7 @@ limitations under the License.
|
||||
Some of the code below came from https://github.com/coreos/etcd-operator
|
||||
which also has the apache 2.0 license.
|
||||
*/
|
||||
package operator
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
@@ -16,7 +16,7 @@ limitations under the License.
|
||||
Some of the code below came from https://github.com/coreos/etcd-operator
|
||||
which also has the apache 2.0 license.
|
||||
*/
|
||||
package operator
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -39,8 +39,8 @@ func TestCreateSecrets(t *testing.T) {
|
||||
response := "{\"key\":\"mysecurekey\"}"
|
||||
return []byte(response), "", nil
|
||||
}
|
||||
spec := ClusterSpec{Namespace: "ns", Version: "myversion"}
|
||||
c := newCluster(spec, factory, clientset)
|
||||
spec := Spec{Namespace: "ns", Version: "myversion"}
|
||||
c := New(spec, factory, clientset)
|
||||
c.dataDir = "/tmp/testdir"
|
||||
defer os.RemoveAll(c.dataDir)
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
Some of the code below came from https://github.com/coreos/etcd-operator
|
||||
which also has the apache 2.0 license.
|
||||
*/
|
||||
package cluster
|
||||
|
||||
type Spec struct {
|
||||
// The namespace where the the rook resources will all be created.
|
||||
Namespace string `json:"namespace"`
|
||||
|
||||
// Version is the expected version of the rook container to run in the cluster.
|
||||
// The rook-operator will eventually make the rook cluster version
|
||||
// equal to the expected version.
|
||||
Version string `json:"version"`
|
||||
|
||||
// Paused is to pause the control of the operator for the rook cluster.
|
||||
Paused bool `json:"paused,omitempty"`
|
||||
|
||||
// Whether to consume all the storage devices found on a machine
|
||||
UseAllDevices bool `json:"useAllDevices"`
|
||||
|
||||
// A regular expression to allow more fine-grained selection of devices on nodes across the cluster
|
||||
DeviceFilter string `json:"deviceFilter"`
|
||||
|
||||
// The path on the host where config and data can be persisted.
|
||||
DataDirHostPath string `json:"dataDirHostPath"`
|
||||
}
|
||||
@@ -23,13 +23,15 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/cluster"
|
||||
|
||||
"k8s.io/client-go/pkg/api/unversioned"
|
||||
kwatch "k8s.io/client-go/pkg/watch"
|
||||
)
|
||||
|
||||
type Event struct {
|
||||
Type kwatch.EventType
|
||||
Object *Cluster
|
||||
Object *cluster.Cluster
|
||||
}
|
||||
|
||||
type rawEvent struct {
|
||||
@@ -58,7 +60,7 @@ func pollEvent(decoder *json.Decoder) (*Event, *unversioned.Status, error) {
|
||||
|
||||
ev := &Event{
|
||||
Type: re.Type,
|
||||
Object: &Cluster{},
|
||||
Object: &cluster.Cluster{},
|
||||
}
|
||||
err = json.Unmarshal(re.Object, ev.Object)
|
||||
if err != nil {
|
||||
|
||||
+91
-29
@@ -21,6 +21,7 @@ import (
|
||||
|
||||
"github.com/rook/rook/pkg/cephmgr/client"
|
||||
"github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/pkg/api/errors"
|
||||
@@ -53,6 +54,7 @@ type Cluster struct {
|
||||
clientset kubernetes.Interface
|
||||
factory client.ConnectionFactory
|
||||
retryDelay int
|
||||
clusterInfo *mon.ClusterInfo
|
||||
dataDirHostPath string
|
||||
}
|
||||
|
||||
@@ -77,7 +79,7 @@ func New(clientset kubernetes.Interface, factory client.ConnectionFactory, names
|
||||
func (c *Cluster) Start() (*mon.ClusterInfo, error) {
|
||||
logger.Infof("start running mons")
|
||||
|
||||
clusterInfo, err := c.initClusterInfo()
|
||||
err := c.initClusterInfo()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err)
|
||||
}
|
||||
@@ -86,50 +88,110 @@ func (c *Cluster) Start() (*mon.ClusterInfo, error) {
|
||||
mons = append(mons, &MonConfig{Name: fmt.Sprintf("mon%d", i), Port: int32(mon.Port)})
|
||||
}
|
||||
|
||||
err = c.startPods(clusterInfo, mons)
|
||||
err = c.startPods(mons)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to start mon pods. %+v", err)
|
||||
}
|
||||
|
||||
return clusterInfo, nil
|
||||
context := &clusterd.Context{ConfigDir: k8sutil.DataDir}
|
||||
err = mon.WaitForQuorum(c.factory, context, c.clusterInfo)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to wait for mon quorum. %+v", err)
|
||||
}
|
||||
|
||||
return c.clusterInfo, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) CheckHealth() error {
|
||||
context := &clusterd.Context{ConfigDir: k8sutil.DataDir}
|
||||
conn, err := mon.ConnectToClusterAsAdmin(context, c.factory, c.clusterInfo)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot connect to cluster. %+v", err)
|
||||
}
|
||||
defer conn.Shutdown()
|
||||
|
||||
status, err := client.GetMonStatus(conn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get mon status. %+v", err)
|
||||
}
|
||||
|
||||
for _, monitor := range status.MonMap.Mons {
|
||||
inQuorum := monInQuorum(monitor, status.Quorum)
|
||||
if inQuorum {
|
||||
logger.Infof("mon %s found in quorum", monitor.Name)
|
||||
} else {
|
||||
logger.Infof("mon %s NOT found in quroum. %+v", monitor.Name, status.Quorum)
|
||||
}
|
||||
|
||||
pod, err := c.clientset.CoreV1().Pods(c.Namespace).Get(monitor.Name)
|
||||
if err != nil {
|
||||
logger.Infof("Get mon %s pod failed. %+v", monitor.Name, err)
|
||||
} else {
|
||||
logger.Infof("mon %s pod status: %+v", monitor.Name, pod.Status.Phase)
|
||||
}
|
||||
|
||||
if !inQuorum {
|
||||
err = c.failoverMon(monitor.Name)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to failover mon %s. %+v", monitor.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func monInQuorum(monitor client.MonMapEntry, quorum []int) bool {
|
||||
for _, rank := range quorum {
|
||||
if rank == monitor.Rank {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *Cluster) failoverMon(name string) error {
|
||||
//logger.Infof("Failing over monitor %s", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Retrieve the ceph cluster info if it already exists.
|
||||
// If a new cluster create new keys.
|
||||
func (c *Cluster) initClusterInfo() (*mon.ClusterInfo, error) {
|
||||
func (c *Cluster) initClusterInfo() error {
|
||||
secrets, err := c.clientset.CoreV1().Secrets(c.Namespace).Get(appName)
|
||||
if err != nil {
|
||||
if !errors.IsNotFound(err) {
|
||||
return nil, fmt.Errorf("failed to get mon secrets. %+v", err)
|
||||
return fmt.Errorf("failed to get mon secrets. %+v", err)
|
||||
}
|
||||
|
||||
return c.createMonSecretsAndSave()
|
||||
}
|
||||
|
||||
info := &mon.ClusterInfo{
|
||||
c.clusterInfo = &mon.ClusterInfo{
|
||||
Name: string(secrets.Data[clusterSecretName]),
|
||||
FSID: string(secrets.Data[fsidSecretName]),
|
||||
MonitorSecret: string(secrets.Data[monSecretName]),
|
||||
AdminSecret: string(secrets.Data[adminSecretName]),
|
||||
Monitors: map[string]*mon.CephMonitorConfig{},
|
||||
}
|
||||
logger.Infof("found existing monitor secrets for cluster %s with fsid %s", info.Name, info.FSID)
|
||||
return info, nil
|
||||
logger.Infof("found existing monitor secrets for cluster %s", c.clusterInfo.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) createMonSecretsAndSave() (*mon.ClusterInfo, error) {
|
||||
func (c *Cluster) createMonSecretsAndSave() error {
|
||||
logger.Infof("creating mon secrets for a new cluster")
|
||||
info, err := mon.CreateNamedClusterInfo(c.factory, "", c.Namespace)
|
||||
var err error
|
||||
c.clusterInfo, err = mon.CreateNamedClusterInfo(c.factory, "", c.Namespace)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create mon secrets. %+v", err)
|
||||
return fmt.Errorf("failed to create mon secrets. %+v", err)
|
||||
}
|
||||
|
||||
// store the secrets for internal usage of the rook pods
|
||||
secrets := map[string]string{
|
||||
clusterSecretName: info.Name,
|
||||
fsidSecretName: info.FSID,
|
||||
monSecretName: info.MonitorSecret,
|
||||
adminSecretName: info.AdminSecret,
|
||||
clusterSecretName: c.clusterInfo.Name,
|
||||
fsidSecretName: c.clusterInfo.FSID,
|
||||
monSecretName: c.clusterInfo.MonitorSecret,
|
||||
adminSecretName: c.clusterInfo.AdminSecret,
|
||||
}
|
||||
secret := &v1.Secret{
|
||||
ObjectMeta: v1.ObjectMeta{Name: appName, Namespace: c.Namespace},
|
||||
@@ -138,12 +200,12 @@ func (c *Cluster) createMonSecretsAndSave() (*mon.ClusterInfo, error) {
|
||||
}
|
||||
_, err = c.clientset.CoreV1().Secrets(c.Namespace).Create(secret)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to save mon secrets. %+v", err)
|
||||
return fmt.Errorf("failed to save mon secrets. %+v", err)
|
||||
}
|
||||
|
||||
// store the secret for usage by the storage class
|
||||
storageClassSecret := map[string]string{
|
||||
"key": info.AdminSecret,
|
||||
"key": c.clusterInfo.AdminSecret,
|
||||
}
|
||||
secret = &v1.Secret{
|
||||
ObjectMeta: v1.ObjectMeta{Name: "rook-admin", Namespace: c.Namespace},
|
||||
@@ -153,17 +215,17 @@ func (c *Cluster) createMonSecretsAndSave() (*mon.ClusterInfo, error) {
|
||||
_, err = c.clientset.CoreV1().Secrets(c.Namespace).Create(secret)
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return nil, fmt.Errorf("failed to save rook-admin secret. %+v", err)
|
||||
return fmt.Errorf("failed to save rook-admin secret. %+v", err)
|
||||
}
|
||||
logger.Infof("rook-admin secret already exists")
|
||||
} else {
|
||||
logger.Infof("saved rook-admin secret")
|
||||
}
|
||||
|
||||
return info, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) startPods(clusterInfo *mon.ClusterInfo, mons []*MonConfig) error {
|
||||
func (c *Cluster) startPods(mons []*MonConfig) error {
|
||||
// schedule the mons on different nodes if we have enough nodes to be unique
|
||||
antiAffinity, err := c.getAntiAffinity()
|
||||
if err != nil {
|
||||
@@ -176,14 +238,14 @@ func (c *Cluster) startPods(clusterInfo *mon.ClusterInfo, mons []*MonConfig) err
|
||||
}
|
||||
logger.Infof("%d running, %d pending pods", len(running), len(pending))
|
||||
|
||||
clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{}
|
||||
c.clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{}
|
||||
for _, m := range running {
|
||||
clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, m.Status.PodIP)
|
||||
c.clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, m.Status.PodIP)
|
||||
}
|
||||
|
||||
err = c.saveMonEndpoints(clusterInfo)
|
||||
err = c.saveMonEndpoints()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save endpoints for %d running mons. %+v", len(clusterInfo.Monitors), err)
|
||||
return fmt.Errorf("failed to save endpoints for %d running mons. %+v", len(c.clusterInfo.Monitors), err)
|
||||
}
|
||||
|
||||
if len(running) == c.Size {
|
||||
@@ -194,7 +256,7 @@ func (c *Cluster) startPods(clusterInfo *mon.ClusterInfo, mons []*MonConfig) err
|
||||
started := 0
|
||||
alreadyRunning := 0
|
||||
for _, m := range mons {
|
||||
monPod := c.makeMonPod(m, clusterInfo, antiAffinity)
|
||||
monPod := c.makeMonPod(m, antiAffinity)
|
||||
name := monPod.Name
|
||||
logger.Debugf("Starting pod: %+v", monPod)
|
||||
_, err = c.clientset.CoreV1().Pods(c.Namespace).Create(monPod)
|
||||
@@ -212,9 +274,9 @@ func (c *Cluster) startPods(clusterInfo *mon.ClusterInfo, mons []*MonConfig) err
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start pod %s. %+v", name, err)
|
||||
}
|
||||
clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, podIP)
|
||||
c.clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, podIP)
|
||||
|
||||
err = c.saveMonEndpoints(clusterInfo)
|
||||
err = c.saveMonEndpoints()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save endpoints after starting mon %s. %+v", m.Name, err)
|
||||
}
|
||||
@@ -248,7 +310,7 @@ func (c *Cluster) waitForPodToStart(name string) (string, error) {
|
||||
return "", fmt.Errorf("timed out waiting for pod %s to start", name)
|
||||
}
|
||||
|
||||
func (c *Cluster) saveMonEndpoints(clusterInfo *mon.ClusterInfo) error {
|
||||
func (c *Cluster) saveMonEndpoints() error {
|
||||
configMap := &v1.ConfigMap{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: monConfigMapName,
|
||||
@@ -257,7 +319,7 @@ func (c *Cluster) saveMonEndpoints(clusterInfo *mon.ClusterInfo) error {
|
||||
},
|
||||
}
|
||||
configMap.Data = map[string]string{
|
||||
monEndpointKey: mon.FlattenMonEndpoints(clusterInfo.Monitors),
|
||||
monEndpointKey: mon.FlattenMonEndpoints(c.clusterInfo.Monitors),
|
||||
}
|
||||
|
||||
_, err := c.clientset.CoreV1().ConfigMaps(c.Namespace).Create(configMap)
|
||||
|
||||
@@ -18,7 +18,6 @@ package mon
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/client-go/pkg/api/v1"
|
||||
"k8s.io/client-go/pkg/labels"
|
||||
@@ -51,7 +50,7 @@ func (c *Cluster) getLabels() map[string]string {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) makeMonPod(config *MonConfig, clusterInfo *mon.ClusterInfo, antiAffinity bool) *v1.Pod {
|
||||
func (c *Cluster) makeMonPod(config *MonConfig, antiAffinity bool) *v1.Pod {
|
||||
|
||||
container := c.monContainer(config, clusterInfo.FSID)
|
||||
dataDirSource := v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}
|
||||
@@ -78,7 +77,7 @@ func (c *Cluster) makeMonPod(config *MonConfig, clusterInfo *mon.ClusterInfo, an
|
||||
k8sutil.SetPodVersion(pod, k8sutil.VersionAttr, c.Version)
|
||||
|
||||
if antiAffinity {
|
||||
k8sutil.PodWithAntiAffinity(pod, monClusterAttr, clusterInfo.Name)
|
||||
k8sutil.PodWithAntiAffinity(pod, monClusterAttr, c.clusterInfo.Name)
|
||||
}
|
||||
return pod
|
||||
}
|
||||
|
||||
+17
-18
@@ -32,6 +32,7 @@ import (
|
||||
"k8s.io/client-go/rest"
|
||||
|
||||
"github.com/rook/rook/pkg/cephmgr/client"
|
||||
"github.com/rook/rook/pkg/operator/cluster"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/client-go/pkg/api"
|
||||
"k8s.io/client-go/pkg/api/unversioned"
|
||||
@@ -48,10 +49,6 @@ var (
|
||||
ErrVersionOutdated = errors.New("requested version is outdated in apiserver")
|
||||
)
|
||||
|
||||
const (
|
||||
defaultPool = "rook"
|
||||
)
|
||||
|
||||
type Operator struct {
|
||||
Namespace string
|
||||
MasterHost string
|
||||
@@ -61,7 +58,7 @@ type Operator struct {
|
||||
waitCluster sync.WaitGroup
|
||||
factory client.ConnectionFactory
|
||||
stopChMap map[string]chan struct{}
|
||||
clusters map[string]*Cluster
|
||||
clusters map[string]*cluster.Cluster
|
||||
kubeHttpCli *http.Client
|
||||
// Kubernetes resource version of the clusters
|
||||
clusterRVs map[string]string
|
||||
@@ -73,7 +70,7 @@ func New(host, namespace string, factory client.ConnectionFactory, clientset kub
|
||||
MasterHost: host,
|
||||
factory: factory,
|
||||
clientset: clientset,
|
||||
clusters: make(map[string]*Cluster),
|
||||
clusters: make(map[string]*cluster.Cluster),
|
||||
clusterRVs: make(map[string]string),
|
||||
stopChMap: map[string]chan struct{}{},
|
||||
retryDelay: 3,
|
||||
@@ -147,7 +144,7 @@ func (o *Operator) watchTPR(watchVersion string) error {
|
||||
continue
|
||||
}
|
||||
|
||||
newCluster := newCluster(c.Spec, o.factory, o.clientset)
|
||||
newCluster := cluster.New(c.Spec, o.factory, o.clientset)
|
||||
stopCh := make(chan struct{})
|
||||
o.stopChMap[ns] = stopCh
|
||||
o.clusters[ns] = newCluster
|
||||
@@ -170,21 +167,23 @@ func (o *Operator) watchTPR(watchVersion string) error {
|
||||
|
||||
}
|
||||
|
||||
func (o *Operator) startCluster(cluster *Cluster) {
|
||||
if o.devicesInUse && cluster.Spec.UseAllDevices {
|
||||
logger.Warningf("devices in more than one namespace not supported. ignoring devices in namespace %s", cluster.Spec.Namespace)
|
||||
cluster.Spec.UseAllDevices = false
|
||||
func (o *Operator) startCluster(c *cluster.Cluster) {
|
||||
if o.devicesInUse && c.Spec.UseAllDevices {
|
||||
logger.Warningf("devices in more than one namespace not supported. ignoring devices in namespace %s", c.Spec.Namespace)
|
||||
c.Spec.UseAllDevices = false
|
||||
}
|
||||
|
||||
if cluster.Spec.UseAllDevices {
|
||||
if c.Spec.UseAllDevices {
|
||||
o.devicesInUse = true
|
||||
}
|
||||
|
||||
go func() {
|
||||
err := cluster.CreateInstance()
|
||||
err := c.CreateInstance()
|
||||
if err != nil {
|
||||
logger.Errorf("failed to create cluster in namespace %s. %+v", cluster.Spec.Namespace, err)
|
||||
logger.Errorf("failed to create cluster in namespace %s. %+v", c.Spec.Namespace, err)
|
||||
return
|
||||
}
|
||||
c.Monitor(o.stopChMap[c.Spec.Namespace])
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -200,7 +199,7 @@ func (o *Operator) findAllClusters() (string, error) {
|
||||
|
||||
stopCh := make(chan struct{})
|
||||
ns := c.Spec.Namespace
|
||||
existingCluster := newCluster(c.Spec, o.factory, o.clientset)
|
||||
existingCluster := cluster.New(c.Spec, o.factory, o.clientset)
|
||||
o.stopChMap[ns] = stopCh
|
||||
o.clusters[ns] = existingCluster
|
||||
o.clusterRVs[ns] = c.Metadata.ResourceVersion
|
||||
@@ -285,7 +284,7 @@ func (o *Operator) watch(watchVersion string) (<-chan *Event, <-chan error) {
|
||||
return eventCh, errCh
|
||||
}
|
||||
|
||||
func (o *Operator) isClustersCacheStale(currentClusters []Cluster) bool {
|
||||
func (o *Operator) isClustersCacheStale(currentClusters []cluster.Cluster) bool {
|
||||
if len(o.clusterRVs) != len(currentClusters) {
|
||||
return true
|
||||
}
|
||||
@@ -305,13 +304,13 @@ func watchClusters(host, ns string, httpClient *http.Client, resourceVersion str
|
||||
host, tprGroup, tprVersion, ns, resourceVersion))
|
||||
}
|
||||
|
||||
func getClusterList(restcli rest.Interface, ns string) (*ClusterList, error) {
|
||||
func getClusterList(restcli rest.Interface, ns string) (*cluster.ClusterList, error) {
|
||||
b, err := restcli.Get().RequestURI(listClustersURI(ns)).DoRaw()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clusters := &ClusterList{}
|
||||
clusters := &cluster.ClusterList{}
|
||||
if err := json.Unmarshal(b, clusters); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user