From f6a96af9bb0f15b96ada15bc01e0c80c3494506d Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Thu, 1 Sep 2016 16:48:17 -0700 Subject: [PATCH 1/2] monitors and osd agents check desired state --- glide.lock | 2 +- pkg/castled/monagent.go | 34 +++++++++++++++++++++------------- pkg/castled/osdagent.go | 15 +++++++++++++-- pkg/castled/osdleader.go | 15 ++++++++++++++- 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/glide.lock b/glide.lock index f36a76dda..2da3745da 100644 --- a/glide.lock +++ b/glide.lock @@ -15,7 +15,7 @@ imports: - name: github.com/inconshreveable/mousetrap version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 - name: github.com/quantum/clusterd - version: 7409e697f6eee196a05dce7c86c1499dd878eb71 + version: d124d39e6d7b5cc80968d62d65a6ce8d372f8c36 subpackages: - pkg/inventory - pkg/orchestrator diff --git a/pkg/castled/monagent.go b/pkg/castled/monagent.go index 5003da4db..99a5f5146 100644 --- a/pkg/castled/monagent.go +++ b/pkg/castled/monagent.go @@ -5,12 +5,14 @@ import ( "io/ioutil" "log" "os" + "path" "path/filepath" "time" ctx "golang.org/x/net/context" "github.com/quantum/clusterd/pkg/orchestrator" + "github.com/quantum/clusterd/pkg/util" ) const ( @@ -19,17 +21,24 @@ const ( type monAgent struct { cluster *clusterInfo - context *orchestrator.ClusterContext } -func (a *monAgent) GetName() string { +func (a *monAgent) Name() string { return monitorAgentName } func (a *monAgent) ConfigureLocalService(context *orchestrator.ClusterContext) error { - a.context = context - var err error + // check if the monitor is in the desired state for this node + key := path.Join(cephKey, monitorAgentName, desiredKey, context.NodeID) + monDesired, err := util.EtcdDirExists(context.EtcdClient, key) + if err != nil { + return err + } + if !monDesired { + return nil + } + a.cluster, err = loadClusterInfo(context.EtcdClient) if err != nil { return fmt.Errorf("failed to load cluster info: %+v", err) @@ -42,12 +51,12 @@ func (a *monAgent) ConfigureLocalService(context *orchestrator.ClusterContext) e } // initialze the file systems for the monitors - if err := a.makeMonitorFileSystem(monitor.Name); err != nil { + if err := a.makeMonitorFileSystem(context, monitor.Name); err != nil { return fmt.Errorf("failed to make monitor filesystems: %+v", err) } // run the monitors - err = a.runMonitor(monitor) + err = a.runMonitor(context, monitor) if err != nil { return fmt.Errorf("failed to run monitors: %+v", err) } @@ -58,19 +67,18 @@ func (a *monAgent) ConfigureLocalService(context *orchestrator.ClusterContext) e } func (a *monAgent) DestroyLocalService(context *orchestrator.ClusterContext) error { - a.context = context return nil } // wait for all expected initial monitors to register (report their names/endpoints) -func (a *monAgent) waitForMonitorRegistration(monName string) error { +func (a *monAgent) waitForMonitorRegistration(context *orchestrator.ClusterContext, monName string) error { key := getMonitorEndpointKey(monName) retryCount := 0 retryMax := 40 sleepTime := 5 for { - resp, err := a.context.EtcdClient.Get(ctx.Background(), key, nil) + resp, err := context.EtcdClient.Get(ctx.Background(), key, nil) if err == nil && resp != nil && resp.Node != nil && resp.Node.Value != "" { log.Printf("monitor %s registered at %s", monName, resp.Node.Value) break @@ -88,7 +96,7 @@ func (a *monAgent) waitForMonitorRegistration(monName string) error { } // creates and initializes the given monitors file systems -func (a *monAgent) makeMonitorFileSystem(monName string) error { +func (a *monAgent) makeMonitorFileSystem(context *orchestrator.ClusterContext, monName string) error { // write the keyring to disk if err := writeMonitorKeyring(monName, a.cluster); err != nil { return err @@ -107,7 +115,7 @@ func (a *monAgent) makeMonitorFileSystem(monName string) error { } // call mon --mkfs in a child process - err = a.context.ProcMan.Run( + err = context.ProcMan.Run( "mon", "--mkfs", fmt.Sprintf("--cluster=%s", a.cluster.Name), @@ -123,14 +131,14 @@ func (a *monAgent) makeMonitorFileSystem(monName string) error { } // runs the monitor in a child process -func (a *monAgent) runMonitor(monitor *CephMonitorConfig) error { +func (a *monAgent) runMonitor(context *orchestrator.ClusterContext, monitor *CephMonitorConfig) error { if monitor.Endpoint == "" { return fmt.Errorf("missing endpoint for mon %s", monitor.Name) } // start the monitor daemon in the foreground with the given config log.Printf("starting monitor %s", monitor.Name) - err := a.context.ProcMan.Start( + err := context.ProcMan.Start( "mon", "--foreground", fmt.Sprintf("--cluster=%v", a.cluster.Name), diff --git a/pkg/castled/osdagent.go b/pkg/castled/osdagent.go index 6ead4016e..cbcbc9cb1 100644 --- a/pkg/castled/osdagent.go +++ b/pkg/castled/osdagent.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "os" + "path" "path/filepath" "strconv" "strings" @@ -12,6 +13,7 @@ import ( "github.com/quantum/castle/pkg/cephd" "github.com/quantum/clusterd/pkg/orchestrator" + "github.com/quantum/clusterd/pkg/util" ) const ( @@ -29,13 +31,22 @@ func newOSDAgent(rawDevices string, forceFormat bool) *osdAgent { return &osdAgent{devices: devices, forceFormat: forceFormat} } -func (a *osdAgent) GetName() string { +func (a *osdAgent) Name() string { return osdAgentName } func (a *osdAgent) ConfigureLocalService(context *orchestrator.ClusterContext) error { - var err error + // check if the osd is in the desired state for this node + key := path.Join(cephKey, osdAgentName, desiredKey, context.NodeID) + osdDesired, err := util.EtcdDirExists(context.EtcdClient, key) + if err != nil { + return err + } + if !osdDesired { + return nil + } + a.cluster, err = loadClusterInfo(context.EtcdClient) if err != nil { return fmt.Errorf("failed to load cluster info: %v", err) diff --git a/pkg/castled/osdleader.go b/pkg/castled/osdleader.go index bd724a613..3825224ea 100644 --- a/pkg/castled/osdleader.go +++ b/pkg/castled/osdleader.go @@ -1,6 +1,12 @@ package castled -import "github.com/quantum/clusterd/pkg/orchestrator" +import ( + "log" + "path" + + "github.com/quantum/clusterd/pkg/orchestrator" + "github.com/quantum/clusterd/pkg/util" +) // Load the state of the OSDs from etcd. // Returns whether the service has updates to be applied. @@ -19,6 +25,13 @@ func configureOSDs(context *orchestrator.ClusterContext) error { // Trigger all of the nodes to configure their OSDs osdNodes := []string{} for nodeID := range context.Inventory.Nodes { + key := path.Join(cephKey, osdAgentName, desiredKey, nodeID) + err := util.CreateEtcdDir(context.EtcdClient, key) + if err != nil { + log.Printf("failed to trigger osd %s", nodeID) + continue + } + osdNodes = append(osdNodes, nodeID) } From 66a2b57c25ed45a5370c31eb83ee073945b6f65d Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Thu, 1 Sep 2016 23:49:50 -0700 Subject: [PATCH 2/2] pickup new upstream ceph. fix minor printf issues --- Documentation/dev-setup.md | 2 +- ceph | 2 +- cmd/castlectl/castlectl.go | 2 +- cmd/castled/castled.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/dev-setup.md b/Documentation/dev-setup.md index 382c86ae7..5509ca16a 100644 --- a/Documentation/dev-setup.md +++ b/Documentation/dev-setup.md @@ -7,7 +7,7 @@ cd $GOPATH/src/github.com/quantum git clone https://github.com/quantum/castle.git cd castle git submodule update --init --recursive -sudo apt-get install cmake python-sphinx libudev-dev libaio-dev libblkid-dev libldap2-dev xfslibs-dev libleveldb-dev libexpat1-dev cython libfcgi-dev libatomic-ops-dev libsnappy-dev libgoogle-perftools-dev libjemalloc-dev libkeyutils-dev libcurl4-openssl-dev libcrypto++-dev libssl-dev libboost-dev libboost-thread-dev libboost-system-dev libboost-regex-dev libboost-random-dev libboost-program-options-dev libboost-date-time-dev libboost-iostreams-dev +sudo apt-get install cmake python-sphinx libudev-dev libaio-dev libblkid-dev libldap2-dev xfslibs-dev libleveldb-dev libexpat1-dev cython libfcgi-dev libatomic-ops-dev libsnappy-dev libgoogle-perftools-dev libjemalloc-dev libkeyutils-dev libcurl4-openssl-dev libcrypto++-dev libssl-dev libboost-dev libboost-thread-dev libboost-system-dev libboost-regex-dev libboost-random-dev libboost-program-options-dev libboost-date-time-dev libboost-iostreams-dev python3-all-dev cython3 make -j2 V=1 STATIC=1 ALLOCATOR=tcmalloc build ./bin/castled ``` diff --git a/ceph b/ceph index 56131a554..0cbdba99b 160000 --- a/ceph +++ b/ceph @@ -1 +1 @@ -Subproject commit 56131a5545544369f2cfe5d3c43333082859d69b +Subproject commit 0cbdba99b4ed85cfdf6d4eb8d6bd65013ed26c23 diff --git a/cmd/castlectl/castlectl.go b/cmd/castlectl/castlectl.go index 44601b71d..fd1a876de 100644 --- a/cmd/castlectl/castlectl.go +++ b/cmd/castlectl/castlectl.go @@ -8,6 +8,6 @@ import ( func main() { if err := cmd.Execute(); err != nil { - fmt.Printf("castlectl error: %+v", err) + fmt.Printf("castlectl error: %+v\n", err) } } diff --git a/cmd/castled/castled.go b/cmd/castled/castled.go index e22b2fe4c..2d6604086 100644 --- a/cmd/castled/castled.go +++ b/cmd/castled/castled.go @@ -8,6 +8,6 @@ import ( func main() { if err := cmd.Execute(); err != nil { - fmt.Printf("castled error: %+v", err) + fmt.Printf("castled error: %+v\n", err) } }