diff --git a/cmd/rook/status/status.go b/cmd/rook/status/status.go index 1dc642f01..984a10f78 100644 --- a/cmd/rook/status/status.go +++ b/cmd/rook/status/status.go @@ -91,6 +91,16 @@ func getStatus(c client.RookRestClient) (string, error) { } w.Flush() + // mgrs + buffer.WriteString("\n") + buffer.WriteString("MGRs:\n") + fmt.Fprintln(w, "NAME\tSTATUS") + fmt.Fprintf(w, "%s\t%s\n", statusDetails.Mgrs.ActiveName, "Active") + for _, name := range statusDetails.Mgrs.Standbys { + fmt.Fprintf(w, "%s\t%s\n", name, "Standby") + } + w.Flush() + // OSDs buffer.WriteString("\n") buffer.WriteString("OSDs:\n") diff --git a/cmd/rookd/main.go b/cmd/rookd/main.go index 920f7f96b..8214a51ea 100644 --- a/cmd/rookd/main.go +++ b/cmd/rookd/main.go @@ -98,6 +98,7 @@ func addCommands() { rootCmd.AddCommand(versionCmd) rootCmd.AddCommand(monCmd) rootCmd.AddCommand(osdCmd) + rootCmd.AddCommand(mgrCmd) rootCmd.AddCommand(rgwCmd) rootCmd.AddCommand(mdsCmd) rootCmd.AddCommand(apiCmd) diff --git a/cmd/rookd/mgr.go b/cmd/rookd/mgr.go new file mode 100644 index 000000000..efdf52351 --- /dev/null +++ b/cmd/rookd/mgr.go @@ -0,0 +1,71 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "fmt" + "os" + + "github.com/rook/rook/pkg/ceph/mgr" + "github.com/rook/rook/pkg/ceph/mon" + "github.com/rook/rook/pkg/util/flags" + "github.com/spf13/cobra" +) + +var ( + mgrName string + mgrKeyring string +) + +var mgrCmd = &cobra.Command{ + Use: "mgr", + Short: "Generates mgr config and runs the mgr daemon", + Hidden: true, +} + +func init() { + mgrCmd.Flags().StringVar(&mgrName, "mgr-name", "", "the mgr name") + mgrCmd.Flags().StringVar(&mgrKeyring, "mgr-keyring", "", "the mgr keyring") + addCephFlags(mgrCmd) + + flags.SetFlagsFromEnv(mgrCmd.Flags(), "ROOKD") + + mgrCmd.RunE = startMgr +} + +func startMgr(cmd *cobra.Command, args []string) error { + if err := flags.VerifyRequiredFlags(mgrCmd, []string{"mon-endpoints", "cluster-name", "mon-secret", "admin-secret"}); err != nil { + return err + } + + setLogLevel() + + clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints) + config := &mgr.Config{ + Name: mgrName, + Keyring: mgrKeyring, + ClusterInfo: &clusterInfo, + InProc: true, + } + + err := mgr.Run(createDaemonContext(), config) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + return nil +} diff --git a/pkg/api/k8s/cluster_handler.go b/pkg/api/k8s/cluster_handler.go index e05a5f4c9..bebb7735c 100644 --- a/pkg/api/k8s/cluster_handler.go +++ b/pkg/api/k8s/cluster_handler.go @@ -22,6 +22,7 @@ import ( "github.com/rook/rook/pkg/ceph/rgw" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/model" + "github.com/rook/rook/pkg/operator/k8sutil" k8smds "github.com/rook/rook/pkg/operator/mds" k8srgw "github.com/rook/rook/pkg/operator/rgw" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -46,8 +47,8 @@ func (s *clusterHandler) EnableObjectStore() error { logger.Infof("Starting the Object store") // Passing an empty Placement{} as the api doesn't know about placement // information. This should be resolved with the transition to CRD (TPR). - r := k8srgw.New(s.context, s.clusterInfo.Name, s.namespace, s.versionTag, , k8sutil.Placement{}) - err := r.Start(s.clusterInfo) + r := k8srgw.New(s.context, s.clusterInfo.Name, s.namespace, s.versionTag, k8sutil.Placement{}) + err := r.Start() if err != nil { return fmt.Errorf("failed to start rgw. %+v", err) } @@ -79,7 +80,7 @@ func (s *clusterHandler) StartFileSystem(fs *model.FilesystemRequest) error { // Passing an empty Placement{} as the api doesn't know about placement // information. This should be resolved with the transition to CRD (TPR). c := k8smds.New(s.context, s.clusterInfo.Name, s.namespace, s.versionTag, k8sutil.Placement{}) - return c.Start(s.context.Clientset, s.clusterInfo) + return c.Start() } func (s *clusterHandler) RemoveFileSystem(fs *model.FilesystemRequest) error { diff --git a/pkg/api/status.go b/pkg/api/status.go index c1e2458ea..fbe73756c 100644 --- a/pkg/api/status.go +++ b/pkg/api/status.go @@ -80,6 +80,16 @@ func (h *Handler) GetStatusDetails(w http.ResponseWriter, r *http.Request) { NearFull: osdMap.NearFull, } + // generate the Mgr summarhy + mgrs := model.MgrSummary{ + ActiveName: cephStatus.MgrMap.ActiveName, + ActiveAddr: cephStatus.MgrMap.ActiveAddr, + Available: cephStatus.MgrMap.Available, + } + for _, standby := range cephStatus.MgrMap.Standbys { + mgrs.Standbys = append(mgrs.Standbys, standby.Name) + } + // generate the usage Summary usageSummary := model.UsageSummary{ DataBytes: cephStatus.PgMap.DataBytes, @@ -99,6 +109,7 @@ func (h *Handler) GetStatusDetails(w http.ResponseWriter, r *http.Request) { OverallStatus: ceph.HealthToModelHealthStatus(cephStatus.Health.OverallStatus), SummaryMessages: summaries, Monitors: monitors, + Mgrs: mgrs, OSDs: osds, PGs: pgSummary, Usage: usageSummary, diff --git a/pkg/ceph/client/auth.go b/pkg/ceph/client/auth.go index 29ea33862..25e8f4709 100644 --- a/pkg/ceph/client/auth.go +++ b/pkg/ceph/client/auth.go @@ -40,7 +40,7 @@ func AuthGetOrCreateKey(context *clusterd.Context, clusterName, name string, cap return "", fmt.Errorf("failed get-or-create-key %s: %+v", name, err) } - logger.Infof("Parsing key: %v", buf) + logger.Infof("Parsing key: %v", string(buf)) return parseAuthKey(buf) } diff --git a/pkg/ceph/client/status.go b/pkg/ceph/client/status.go index c78a8d0ea..365c6f923 100644 --- a/pkg/ceph/client/status.go +++ b/pkg/ceph/client/status.go @@ -48,7 +48,8 @@ type CephStatus struct { OsdMap struct { OsdMap OsdMap `json:"osdmap"` } `json:"osdmap"` - PgMap PgMap `json:"pgmap"` + PgMap PgMap `json:"pgmap"` + MgrMap MgrMap `json:"mgrmap"` } type HealthStatus struct { @@ -95,6 +96,20 @@ type MonMap struct { Mons []MonMapEntry `json:"mons"` } +type MgrMap struct { + Epoch int `json:"epoch"` + ActiveGID int `json:"active_gid"` + ActiveName string `json:"active_name"` + ActiveAddr string `json:"active_addr"` + Available bool `json:"available"` + Standbys []MgrStandby `json:"standbys"` +} + +type MgrStandby struct { + GID int `json:"gid"` + Name string `json:"name"` +} + type OsdMap struct { Epoch int `json:"epoch"` NumOsd int `json:"num_osds"` @@ -121,12 +136,13 @@ type PgStateEntry struct { } func Status(context *clusterd.Context, clusterName string) (CephStatus, error) { - args := []string{"status", "--format", "json"} + args := []string{"status"} buf, err := ExecuteCephCommand(context, clusterName, args) if err != nil { return CephStatus{}, fmt.Errorf("failed to get status: %+v", err) } + logger.Infof("STATUS: %s", string(buf)) var status CephStatus if err := json.Unmarshal(buf, &status); err != nil { return CephStatus{}, fmt.Errorf("failed to unmarshal status response: %+v", err) diff --git a/pkg/ceph/mgr/daemon.go b/pkg/ceph/mgr/daemon.go new file mode 100644 index 000000000..4012e6537 --- /dev/null +++ b/pkg/ceph/mgr/daemon.go @@ -0,0 +1,135 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package mgr + +import ( + "fmt" + "path" + "regexp" + + "github.com/coreos/pkg/capnslog" + "github.com/rook/rook/pkg/ceph/mon" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/util" + "github.com/rook/rook/pkg/util/proc" +) + +var ( + logger = capnslog.NewPackageLogger("github.com/rook/rook", "cephmgr") + keyringTemplate = ` +[mgr.%s] + key = %s + caps mon = "allow *" +` +) + +const ( + cephmgr = "ceph-mgr" +) + +type Config struct { + InProc bool + ClusterInfo *mon.ClusterInfo + Name string + Keyring string +} + +func Run(context *clusterd.Context, config *Config) error { + logger.Infof("Starting MGR %s with keyring %s", config.Name, config.Keyring) + err := generateConfigFiles(context, config) + if err != nil { + return fmt.Errorf("failed to generate mgr config files. %+v", err) + } + + _, err = startMgr(context, config) + if err != nil { + return fmt.Errorf("failed to run mgr. %+v", err) + } + + return err +} + +func generateConfigFiles(context *clusterd.Context, config *Config) error { + // write the latest config to the config dir + /*if err := mon.GenerateAdminConnectionConfig(context, config.ClusterInfo); err != nil { + return fmt.Errorf("failed to write connection config. %+v", err) + }*/ + + keyringPath := getMgrKeyringPath(context.ConfigDir, config.Name) + confDir := getMgrConfDir(context.ConfigDir, config.Name) + username := fmt.Sprintf("mgr.%s", config.Name) + settings := map[string]string{ + "mgr data": confDir, + } + logger.Infof("Conf files: dir=%s keyring=%s", confDir, keyringPath) + _, err := mon.GenerateConfigFile(context, config.ClusterInfo, confDir, + username, keyringPath, false, nil, settings) + if err != nil { + return fmt.Errorf("failed to create config file. %+v", err) + } + + keyringEval := func(key string) string { + return fmt.Sprintf(keyringTemplate, config.Name, key) + } + + err = mon.WriteKeyring(keyringPath, config.Keyring, keyringEval) + if err != nil { + return fmt.Errorf("failed to create mds keyring. %+v", err) + } + + return nil +} + +func startMgr(context *clusterd.Context, config *Config) (mgrProc *proc.MonitoredProc, err error) { + + // start the mgr daemon in the foreground with the given config + logger.Infof("starting ceph-mgr") + + confFile := getMgrConfFilePath(context.ConfigDir, config.Name, config.ClusterInfo.Name) + util.WriteFileToLog(logger, confFile) + + keyringPath := getMgrKeyringPath(context.ConfigDir, config.Name) + util.WriteFileToLog(logger, keyringPath) + args := []string{ + "--foreground", + fmt.Sprintf("--cluster=%s", config.ClusterInfo.Name), + fmt.Sprintf("--conf=%s", confFile), + fmt.Sprintf("--keyring=%s", keyringPath), + "-i", config.Name, + } + + if config.InProc { + err = context.ProcMan.Run(cephmgr, cephmgr, args...) + } else { + mgrProc, err = context.ProcMan.Start(cephmgr, cephmgr, regexp.QuoteMeta(cephmgr), proc.ReuseExisting, args...) + } + if err != nil { + err = fmt.Errorf("failed to start mgr: %+v", err) + } + return +} + +func getMgrConfDir(dir, name string) string { + return path.Join(dir, fmt.Sprintf("mgr%s", name)) +} + +func getMgrConfFilePath(dir, name, clusterName string) string { + return path.Join(getMgrConfDir(dir, name), fmt.Sprintf("%s.config", clusterName)) +} + +func getMgrKeyringPath(dir, name string) string { + return path.Join(getMgrConfDir(dir, name), "keyring") +} diff --git a/pkg/ceph/mgr/leader.go b/pkg/ceph/mgr/leader.go new file mode 100644 index 000000000..ef60db522 --- /dev/null +++ b/pkg/ceph/mgr/leader.go @@ -0,0 +1,41 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package mgr + +import ( + "fmt" + + "github.com/rook/rook/pkg/ceph/client" + "github.com/rook/rook/pkg/clusterd" +) + +func getKeyringProperties(name string) (string, []string) { + username := fmt.Sprintf("mgr.%s", name) + access := []string{"mon", "allow *"} + return username, access +} + +// create a keyring for the mds client with a limited set of privileges +func CreateKeyring(context *clusterd.Context, clusterName, name string) (string, error) { + // get-or-create-key for the user account + username, access := getKeyringProperties(name) + keyring, err := client.AuthGetOrCreateKey(context, clusterName, username, access) + if err != nil { + return "", fmt.Errorf("failed to get or create auth key for %s. %+v", username, err) + } + + return keyring, nil +} diff --git a/pkg/ceph/mon/agent.go b/pkg/ceph/mon/agent.go index 5bc3486e6..ddcf06feb 100644 --- a/pkg/ceph/mon/agent.go +++ b/pkg/ceph/mon/agent.go @@ -33,9 +33,9 @@ const ( monitorKeyringTemplate = ` [mon.] key = %s - caps mon = "allow *"` + adminKeyringTemplate + caps mon = "allow *"` + AdminKeyringTemplate - adminKeyringTemplate = ` + AdminKeyringTemplate = ` [client.admin] key = %s auid = 0 diff --git a/pkg/ceph/mon/config.go b/pkg/ceph/mon/config.go index 44fad584e..b547628e3 100644 --- a/pkg/ceph/mon/config.go +++ b/pkg/ceph/mon/config.go @@ -95,7 +95,7 @@ func GetConfFilePath(root, clusterName string) string { func GenerateAdminConnectionConfig(context *clusterd.Context, cluster *ClusterInfo) error { root := path.Join(context.ConfigDir, cluster.Name) - keyring := fmt.Sprintf(adminKeyringTemplate, cluster.AdminSecret) + keyring := fmt.Sprintf(AdminKeyringTemplate, cluster.AdminSecret) keyringPath := path.Join(root, fmt.Sprintf("%s.keyring", client.AdminUsername)) err := writeKeyring(cluster, keyring, keyringPath) if err != nil { @@ -170,7 +170,7 @@ func GenerateConfigFile(context *clusterd.Context, cluster *ClusterInfo, pathRoo // write the entire config to disk filePath := GetConfFilePath(pathRoot, cluster.Name) - logger.Debugf("writing config file %s", filePath) + logger.Infof("writing config file %s", filePath) if err := configFile.SaveTo(filePath); err != nil { return "", fmt.Errorf("failed to save config file %s. %+v", filePath, err) } diff --git a/pkg/model/status.go b/pkg/model/status.go index a632d8d23..d6f036d85 100644 --- a/pkg/model/status.go +++ b/pkg/model/status.go @@ -30,6 +30,7 @@ type StatusDetails struct { Monitors []MonitorSummary `json:"monitors"` OSDs OSDSummary `json:"osd"` PGs PGSummary `json:"pg"` + Mgrs MgrSummary `json:"mgr"` Usage UsageSummary `json:"usage"` } @@ -58,6 +59,13 @@ type PGSummary struct { StateCounts map[string]int `json:"stateCount"` } +type MgrSummary struct { + ActiveName string `json:"active_name"` + ActiveAddr string `json:"active_addr"` + Available bool `json:"available"` + Standbys []string `json:"standbys"` +} + type UsageSummary struct { DataBytes uint64 `json:"data"` UsedBytes uint64 `json:"used"` diff --git a/pkg/operator/api/api.go b/pkg/operator/api/api.go index 1822ca8a8..4494a478c 100644 --- a/pkg/operator/api/api.go +++ b/pkg/operator/api/api.go @@ -18,6 +18,7 @@ package api import ( "fmt" + "github.com/coreos/pkg/capnslog" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/model" "github.com/rook/rook/pkg/operator/k8sutil" @@ -30,6 +31,8 @@ import ( "k8s.io/client-go/pkg/apis/rbac/v1beta1" ) +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-api") + const ( DeploymentName = "rook-api" ) diff --git a/pkg/operator/api/log.go b/pkg/operator/api/log.go deleted file mode 100644 index e2622f613..000000000 --- a/pkg/operator/api/log.go +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package api - -import ( - "github.com/coreos/pkg/capnslog" -) - -var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-api") diff --git a/pkg/operator/cluster/cluster.go b/pkg/operator/cluster/cluster.go index 1e6c146d4..d97a9a853 100644 --- a/pkg/operator/cluster/cluster.go +++ b/pkg/operator/cluster/cluster.go @@ -31,6 +31,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/api" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/operator/mgr" "github.com/rook/rook/pkg/operator/mon" "github.com/rook/rook/pkg/operator/osd" "github.com/rook/rook/pkg/operator/rgw" @@ -50,6 +51,7 @@ type Cluster struct { v1.ObjectMeta `json:"metadata,omitempty"` Spec `json:"spec"` mons *mon.Cluster + mgrs *mgr.Cluster osds *osd.Cluster apis *api.Cluster rgws *rgw.Cluster @@ -90,6 +92,12 @@ func (c *Cluster) CreateInstance() error { return fmt.Errorf("failed to start the mons. %+v", err) } + c.mgrs = mgr.New(c.context, c.Name, c.Namespace, c.Spec.VersionTag) + err = c.mgrs.Start() + if err != nil { + return fmt.Errorf("failed to start the ceph mgr. %+v", err) + } + c.apis = api.New(c.context, c.Name, c.Namespace, c.Spec.VersionTag, c.Spec.Placement.GetAPI()) err = c.apis.Start() if err != nil { diff --git a/pkg/operator/k8sutil/k8sutil.go b/pkg/operator/k8sutil/k8sutil.go index 90de21e7e..15e156c62 100644 --- a/pkg/operator/k8sutil/k8sutil.go +++ b/pkg/operator/k8sutil/k8sutil.go @@ -22,9 +22,12 @@ import ( "fmt" "time" + "github.com/coreos/pkg/capnslog" "github.com/rook/rook/pkg/clusterd" ) +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-k8sutil") + const ( Namespace = "rook" DefaultNamespace = "default" diff --git a/pkg/operator/k8sutil/log.go b/pkg/operator/k8sutil/log.go deleted file mode 100644 index d1b911679..000000000 --- a/pkg/operator/k8sutil/log.go +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package k8sutil - -import ( - "github.com/coreos/pkg/capnslog" -) - -var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-k8sutil") diff --git a/pkg/operator/mds/log.go b/pkg/operator/mds/log.go deleted file mode 100644 index 3a3696a31..000000000 --- a/pkg/operator/mds/log.go +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package mds - -import ( - "github.com/coreos/pkg/capnslog" -) - -var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mds") diff --git a/pkg/operator/mds/mds.go b/pkg/operator/mds/mds.go index 8c13c3a87..287b1d99d 100644 --- a/pkg/operator/mds/mds.go +++ b/pkg/operator/mds/mds.go @@ -18,8 +18,8 @@ package mds import ( "fmt" + "github.com/coreos/pkg/capnslog" cephmds "github.com/rook/rook/pkg/ceph/mds" - "github.com/rook/rook/pkg/ceph/mon" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" opmon "github.com/rook/rook/pkg/operator/mon" @@ -30,6 +30,8 @@ import ( extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1" ) +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mds") + const ( appName = "mds" dataPoolSuffix = "-data" @@ -59,22 +61,18 @@ func New(context *clusterd.Context, name, namespace, version string, placement k } } -func (c *Cluster) Start(clientset kubernetes.Interface, cluster *mon.ClusterInfo) error { +func (c *Cluster) Start() error { logger.Infof("start running mds") - if cluster == nil || len(cluster.Monitors) == 0 { - return fmt.Errorf("missing mons to start mds") - } - id := "mds1" - err := c.createKeyring(clientset, cluster, id) + err := c.createKeyring(c.context.Clientset, id) if err != nil { return fmt.Errorf("failed to create mds keyring. %+v", err) } // start the deployment deployment := c.makeDeployment(id) - _, err = clientset.ExtensionsV1beta1().Deployments(c.Namespace).Create(deployment) + _, err = c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Create(deployment) if err != nil { if !errors.IsAlreadyExists(err) { return fmt.Errorf("failed to create mds deployment. %+v", err) @@ -87,7 +85,7 @@ func (c *Cluster) Start(clientset kubernetes.Interface, cluster *mon.ClusterInfo return nil } -func (c *Cluster) createKeyring(clientset kubernetes.Interface, cluster *mon.ClusterInfo, id string) error { +func (c *Cluster) createKeyring(clientset kubernetes.Interface, id string) error { _, err := clientset.CoreV1().Secrets(c.Namespace).Get(appName, metav1.GetOptions{}) if err == nil { logger.Infof("the mds keyring was already generated") diff --git a/pkg/operator/mds/mds_test.go b/pkg/operator/mds/mds_test.go index 673f5017b..6824c9530 100644 --- a/pkg/operator/mds/mds_test.go +++ b/pkg/operator/mds/mds_test.go @@ -28,12 +28,10 @@ import ( exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/pkg/api/v1" ) func TestStartMDS(t *testing.T) { - info := testop.CreateClusterInfo(1) executor := &exectest.MockExecutor{ MockExecuteCommandWithOutput: func(actionName string, command string, args ...string) (string, error) { return "{\"key\":\"mysecurekey\"}", nil @@ -42,27 +40,29 @@ func TestStartMDS(t *testing.T) { configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) - c := New(&clusterd.Context{Executor: executor, ConfigDir: configDir}, "myname", "ns", "myversion", k8sutil.Placement{}) + context := &clusterd.Context{ + Executor: executor, + ConfigDir: configDir, + KubeContext: clusterd.KubeContext{Clientset: testop.New(3)}} + c := New(context, "myname", "ns", "myversion", k8sutil.Placement{}) defer os.RemoveAll(c.dataDir) - clientset := testop.New(3) - // start a basic cluster - err := c.Start(clientset, info) + err := c.Start() assert.Nil(t, err) - validateStart(t, c, clientset) + validateStart(t, c) // starting again should be a no-op - err = c.Start(clientset, info) + err = c.Start() assert.Nil(t, err) - validateStart(t, c, clientset) + validateStart(t, c) } -func validateStart(t *testing.T, c *Cluster, clientset *fake.Clientset) { +func validateStart(t *testing.T, c *Cluster) { - r, err := clientset.ExtensionsV1beta1().Deployments(c.Namespace).Get("mds", metav1.GetOptions{}) + r, err := c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Get("mds", metav1.GetOptions{}) assert.Nil(t, err) assert.Equal(t, "mds", r.Name) } diff --git a/pkg/operator/mgr/mgr.go b/pkg/operator/mgr/mgr.go new file mode 100644 index 000000000..aa0835547 --- /dev/null +++ b/pkg/operator/mgr/mgr.go @@ -0,0 +1,173 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package mgr + +import ( + "fmt" + + "github.com/coreos/pkg/capnslog" + cephmgr "github.com/rook/rook/pkg/ceph/mgr" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/k8sutil" + opmon "github.com/rook/rook/pkg/operator/mon" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/pkg/api/v1" + extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1" +) + +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mgr") + +const ( + appName = "cephmgr" + keyringName = "keyring" + mgrCount = 2 +) + +type Cluster struct { + Name string + Namespace string + Version string + Replicas int32 + context *clusterd.Context + dataDir string +} + +func New(context *clusterd.Context, name, namespace, version string) *Cluster { + return &Cluster{ + context: context, + Name: name, + Namespace: namespace, + Version: version, + Replicas: 1, + dataDir: k8sutil.DataDir, + } +} + +func (c *Cluster) Start() error { + logger.Infof("start running mgr") + + for i := 1; i <= mgrCount; i++ { + name := fmt.Sprintf("%s%d", appName, i) + err := c.createKeyring(c.Name, name) + if err != nil { + return fmt.Errorf("failed to create mgr keyring. %+v", err) + } + + // start the deployment + deployment := c.makeDeployment(name) + _, err = c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Create(deployment) + if err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create mgr deployment. %+v", err) + } + logger.Infof("%s deployment already exists", name) + } else { + logger.Infof("%s deployment started", name) + } + } + + return nil +} + +func (c *Cluster) makeDeployment(name string) *extensions.Deployment { + deployment := &extensions.Deployment{} + deployment.Name = name + deployment.Namespace = c.Namespace + + podSpec := v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: c.getLabels(), + Annotations: map[string]string{}, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{c.mgrContainer(name)}, + RestartPolicy: v1.RestartPolicyAlways, + Volumes: []v1.Volume{ + {Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}, + k8sutil.ConfigOverrideVolume(), + }, + }, + } + + deployment.Spec = extensions.DeploymentSpec{Template: podSpec, Replicas: &c.Replicas} + return deployment +} + +func (c *Cluster) mgrContainer(name string) v1.Container { + + command := fmt.Sprintf("/usr/local/bin/rookd mgr --config-dir=%s", k8sutil.DataDir) + return v1.Container{ + // FIX: sleep 5 for a flaky network setup + Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)}, + Name: name, + Image: k8sutil.MakeRookImage(c.Version), + VolumeMounts: []v1.VolumeMount{ + {Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir}, + k8sutil.ConfigOverrideMount(), + }, + Env: []v1.EnvVar{ + {Name: "ROOKD_MGR_NAME", Value: name}, + {Name: "ROOKD_MGR_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: name}, Key: keyringName}}}, + opmon.ClusterNameEnvVar(c.Name), + opmon.MonEndpointEnvVar(), + opmon.MonSecretEnvVar(), + opmon.AdminSecretEnvVar(), + k8sutil.ConfigOverrideEnvVar(), + }, + } +} + +func (c *Cluster) getLabels() map[string]string { + return map[string]string{ + k8sutil.AppAttr: appName, + k8sutil.ClusterAttr: c.Namespace, + } +} + +func (c *Cluster) createKeyring(clusterName, name string) error { + _, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(name, metav1.GetOptions{}) + if err == nil { + logger.Infof("the mgr keyring was already generated") + return nil + } + if !errors.IsNotFound(err) { + return fmt.Errorf("failed to get mgr secrets. %+v", err) + } + + // get-or-create-key for the user account + keyring, err := cephmgr.CreateKeyring(c.context, clusterName, name) + if err != nil { + return fmt.Errorf("failed to create mgr keyring. %+v", err) + } + + // Store the keyring in a secret + secrets := map[string]string{ + keyringName: keyring, + } + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: c.Namespace}, + StringData: secrets, + Type: k8sutil.RookType, + } + _, err = c.context.Clientset.CoreV1().Secrets(c.Namespace).Create(secret) + if err != nil { + return fmt.Errorf("failed to save mgr secrets. %+v", err) + } + + return nil +} diff --git a/pkg/operator/mgr/mgr_test.go b/pkg/operator/mgr/mgr_test.go new file mode 100644 index 000000000..624a6fe65 --- /dev/null +++ b/pkg/operator/mgr/mgr_test.go @@ -0,0 +1,92 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package mgr + +import ( + "fmt" + "io/ioutil" + "os" + "strings" + "testing" + + "github.com/rook/rook/pkg/clusterd" + testop "github.com/rook/rook/pkg/operator/test" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/pkg/api/v1" +) + +func TestStartMGR(t *testing.T) { + executor := &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(actionName string, command string, args ...string) (string, error) { + return "{\"key\":\"mysecurekey\"}", nil + }, + } + + configDir, _ := ioutil.TempDir("", "") + defer os.RemoveAll(configDir) + context := &clusterd.Context{ + Executor: executor, + ConfigDir: configDir, + KubeContext: clusterd.KubeContext{Clientset: testop.New(3)}} + c := New(context, "myname", "ns", "myversion") + defer os.RemoveAll(c.dataDir) + + // start a basic service + err := c.Start() + assert.Nil(t, err) + + validateStart(t, c) + + // starting again should be a no-op + err = c.Start() + assert.Nil(t, err) + + validateStart(t, c) +} + +func validateStart(t *testing.T, c *Cluster) { + + r, err := c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Get(appName, metav1.GetOptions{}) + assert.Nil(t, err) + assert.Equal(t, appName, r.Name) +} + +func TestPodSpec(t *testing.T) { + c := New(nil, "myname", "ns", "myversion") + + d := c.makeDeployment("mgr1") + assert.NotNil(t, d) + assert.Equal(t, appName, d.Name) + assert.Equal(t, v1.RestartPolicyAlways, d.Spec.Template.Spec.RestartPolicy) + assert.Equal(t, 2, len(d.Spec.Template.Spec.Volumes)) + assert.Equal(t, "rook-data", d.Spec.Template.Spec.Volumes[0].Name) + + assert.Equal(t, appName, d.ObjectMeta.Name) + assert.Equal(t, appName, d.Spec.Template.ObjectMeta.Labels["app"]) + assert.Equal(t, c.Namespace, d.Spec.Template.ObjectMeta.Labels["rook_cluster"]) + assert.Equal(t, 0, len(d.ObjectMeta.Annotations)) + + cont := d.Spec.Template.Spec.Containers[0] + assert.Equal(t, "quay.io/rook/rookd:myversion", cont.Image) + assert.Equal(t, 2, len(cont.VolumeMounts)) + assert.Equal(t, 7, len(cont.Env)) + + expectedCommand := fmt.Sprintf("/usr/local/bin/rookd mgr --config-dir=/var/lib/rook") + + assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2]) +} diff --git a/pkg/operator/mon/log.go b/pkg/operator/mon/log.go deleted file mode 100644 index 43146c950..000000000 --- a/pkg/operator/mon/log.go +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package mon - -import ( - "github.com/coreos/pkg/capnslog" -) - -var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mon") diff --git a/pkg/operator/mon/mon.go b/pkg/operator/mon/mon.go index 259f18486..9543394cb 100644 --- a/pkg/operator/mon/mon.go +++ b/pkg/operator/mon/mon.go @@ -21,6 +21,7 @@ import ( "strings" "time" + "github.com/coreos/pkg/capnslog" "github.com/rook/rook/pkg/ceph/client" "github.com/rook/rook/pkg/ceph/mon" "github.com/rook/rook/pkg/clusterd" @@ -32,6 +33,8 @@ import ( "k8s.io/client-go/pkg/api/v1" ) +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mon") + const ( appName = "mon" monNodeAttr = "mon_node" @@ -146,6 +149,12 @@ func (c *Cluster) initClusterInfo() error { if err != nil { return fmt.Errorf("failed to get mon config. %+v", err) } + + // make sure we have the connection info generated so connections can happen + err = c.writeConnectionConfig() + if err != nil { + return err + } return nil } @@ -350,13 +359,22 @@ func (c *Cluster) saveMonConfig() error { logger.Infof("saved mon endpoints to config map %+v", configMap.Data) // write the latest config to the config dir - if err := mon.GenerateAdminConnectionConfig(c.context, c.clusterInfo); err != nil { + if err := c.writeConnectionConfig(); err != nil { return fmt.Errorf("failed to write connection config for new mons. %+v", err) } return nil } +func (c *Cluster) writeConnectionConfig() error { + // write the latest config to the config dir + if err := mon.GenerateAdminConnectionConfig(c.context, c.clusterInfo); err != nil { + return fmt.Errorf("failed to write connection config. %+v", err) + } + + return nil +} + func (c *Cluster) loadMonConfig() error { cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(monConfigMapName, metav1.GetOptions{}) if err != nil { diff --git a/pkg/operator/osd/log.go b/pkg/operator/osd/log.go deleted file mode 100644 index 8ba097c44..000000000 --- a/pkg/operator/osd/log.go +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package osd - -import ( - "github.com/coreos/pkg/capnslog" -) - -var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-osd") diff --git a/pkg/operator/osd/osd.go b/pkg/operator/osd/osd.go index cefcab142..13eacd000 100644 --- a/pkg/operator/osd/osd.go +++ b/pkg/operator/osd/osd.go @@ -22,6 +22,7 @@ import ( "strconv" + "github.com/coreos/pkg/capnslog" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" opmon "github.com/rook/rook/pkg/operator/mon" @@ -31,6 +32,8 @@ import ( extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1" ) +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-osd") + const ( appName = "osd" appNameFmt = "osd-%s" diff --git a/pkg/operator/rgw/log.go b/pkg/operator/rgw/log.go deleted file mode 100644 index 0e136bba2..000000000 --- a/pkg/operator/rgw/log.go +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package rgw - -import ( - "github.com/coreos/pkg/capnslog" -) - -var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-rgw") diff --git a/pkg/operator/rgw/rgw.go b/pkg/operator/rgw/rgw.go index c3cdc76c4..dd870a36b 100644 --- a/pkg/operator/rgw/rgw.go +++ b/pkg/operator/rgw/rgw.go @@ -18,7 +18,7 @@ package rgw import ( "fmt" - "github.com/rook/rook/pkg/ceph/mon" + "github.com/coreos/pkg/capnslog" cephrgw "github.com/rook/rook/pkg/ceph/rgw" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" @@ -30,6 +30,8 @@ import ( extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1" ) +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-rgw") + const ( appName = "rgw" keyringName = "keyring" @@ -55,14 +57,10 @@ func New(context *clusterd.Context, name, namespace, version string, placement k } } -func (c *Cluster) Start(cluster *mon.ClusterInfo) error { +func (c *Cluster) Start() error { logger.Infof("start running rgw") - if cluster == nil || len(cluster.Monitors) == 0 { - return fmt.Errorf("missing mons to start rgw") - } - - err := c.createKeyring(cluster) + err := c.createKeyring() if err != nil { return fmt.Errorf("failed to create rgw keyring. %+v", err) } @@ -88,7 +86,7 @@ func (c *Cluster) Start(cluster *mon.ClusterInfo) error { return nil } -func (c *Cluster) createKeyring(cluster *mon.ClusterInfo) error { +func (c *Cluster) createKeyring() error { _, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(appName, metav1.GetOptions{}) if err == nil { logger.Infof("the rgw keyring was already generated") @@ -100,7 +98,7 @@ func (c *Cluster) createKeyring(cluster *mon.ClusterInfo) error { // create the keyring logger.Infof("generating rgw keyring") - keyring, err := cephrgw.CreateKeyring(c.context, cluster.Name) + keyring, err := cephrgw.CreateKeyring(c.context, c.Name) if err != nil { return fmt.Errorf("failed to create keyring. %+v", err) } diff --git a/pkg/operator/rgw/rgw_test.go b/pkg/operator/rgw/rgw_test.go index e1ea23c48..8afde8f88 100644 --- a/pkg/operator/rgw/rgw_test.go +++ b/pkg/operator/rgw/rgw_test.go @@ -35,7 +35,6 @@ import ( func TestStartRGW(t *testing.T) { clientset := testop.New(3) - info := testop.CreateClusterInfo(1) executor := &exectest.MockExecutor{ MockExecuteCommandWithOutput: func(actionName string, command string, args ...string) (string, error) { return "{\"key\":\"mysecurekey\"}", nil @@ -47,13 +46,13 @@ func TestStartRGW(t *testing.T) { c := New(&clusterd.Context{KubeContext: clusterd.KubeContext{Clientset: clientset}, Executor: executor, ConfigDir: configDir}, "myname", "ns", "version", k8sutil.Placement{}) // start a basic cluster - err := c.Start(info) + err := c.Start() assert.Nil(t, err) validateStart(t, c, clientset) // starting again should be a no-op - err = c.Start(info) + err = c.Start() assert.Nil(t, err) validateStart(t, c, clientset)