feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

operator: start a pod for the ceph-mgr

Browse Source
This commit is contained in:
Travis Nielsen authored and Bassam Tabbara committed 2017-06-20 20:57:46 -07:00
1 parent f3ae0a2fa8
commit ef4ea21913
29 files changed
+632 -175
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cmd/rook/status/status.go
+10
View File
@@ -91,6 +91,16 @@ func getStatus(c client.RookRestClient) (string, error) {
}
w.Flush()
// mgrs
buffer.WriteString("\n")
buffer.WriteString("MGRs:\n")
fmt.Fprintln(w, "NAME\tSTATUS")
fmt.Fprintf(w, "%s\t%s\n", statusDetails.Mgrs.ActiveName, "Active")
for _, name := range statusDetails.Mgrs.Standbys {
fmt.Fprintf(w, "%s\t%s\n", name, "Standby")
}
w.Flush()
// OSDs
buffer.WriteString("\n")
buffer.WriteString("OSDs:\n")
cmd/rookd/main.go
+1
View File
@@ -98,6 +98,7 @@ func addCommands() {
rootCmd.AddCommand(versionCmd)
rootCmd.AddCommand(monCmd)
rootCmd.AddCommand(osdCmd)
rootCmd.AddCommand(mgrCmd)
rootCmd.AddCommand(rgwCmd)
rootCmd.AddCommand(mdsCmd)
rootCmd.AddCommand(apiCmd)
cmd/rookd/mgr.go
+71
View File
@@ -0,0 +1,71 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"fmt"
"os"
"github.com/rook/rook/pkg/ceph/mgr"
"github.com/rook/rook/pkg/ceph/mon"
"github.com/rook/rook/pkg/util/flags"
"github.com/spf13/cobra"
)
var (
mgrName string
mgrKeyring string
)
var mgrCmd = &cobra.Command{
Use: "mgr",
Short: "Generates mgr config and runs the mgr daemon",
Hidden: true,
}
func init() {
mgrCmd.Flags().StringVar(&mgrName, "mgr-name", "", "the mgr name")
mgrCmd.Flags().StringVar(&mgrKeyring, "mgr-keyring", "", "the mgr keyring")
addCephFlags(mgrCmd)
flags.SetFlagsFromEnv(mgrCmd.Flags(), "ROOKD")
mgrCmd.RunE = startMgr
}
func startMgr(cmd *cobra.Command, args []string) error {
if err := flags.VerifyRequiredFlags(mgrCmd, []string{"mon-endpoints", "cluster-name", "mon-secret", "admin-secret"}); err != nil {
return err
}
setLogLevel()
clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
config := &mgr.Config{
Name: mgrName,
Keyring: mgrKeyring,
ClusterInfo: &clusterInfo,
InProc: true,
}
err := mgr.Run(createDaemonContext(), config)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
return nil
}
pkg/api/k8s/cluster_handler.go
+4 -3
View File
@@ -22,6 +22,7 @@ import (
"github.com/rook/rook/pkg/ceph/rgw"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/model"
"github.com/rook/rook/pkg/operator/k8sutil"
k8smds "github.com/rook/rook/pkg/operator/mds"
k8srgw "github.com/rook/rook/pkg/operator/rgw"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -46,8 +47,8 @@ func (s *clusterHandler) EnableObjectStore() error {
logger.Infof("Starting the Object store")
// Passing an empty Placement{} as the api doesn't know about placement
// information. This should be resolved with the transition to CRD (TPR).
r := k8srgw.New(s.context, s.clusterInfo.Name, s.namespace, s.versionTag, , k8sutil.Placement{})
err := r.Start(s.clusterInfo)
r := k8srgw.New(s.context, s.clusterInfo.Name, s.namespace, s.versionTag, k8sutil.Placement{})
err := r.Start()
if err != nil {
return fmt.Errorf("failed to start rgw. %+v", err)
}
@@ -79,7 +80,7 @@ func (s *clusterHandler) StartFileSystem(fs *model.FilesystemRequest) error {
// Passing an empty Placement{} as the api doesn't know about placement
// information. This should be resolved with the transition to CRD (TPR).
c := k8smds.New(s.context, s.clusterInfo.Name, s.namespace, s.versionTag, k8sutil.Placement{})
return c.Start(s.context.Clientset, s.clusterInfo)
return c.Start()
}
func (s *clusterHandler) RemoveFileSystem(fs *model.FilesystemRequest) error {
pkg/api/status.go
+11
View File
@@ -80,6 +80,16 @@ func (h *Handler) GetStatusDetails(w http.ResponseWriter, r *http.Request) {
NearFull: osdMap.NearFull,
}
// generate the Mgr summarhy
mgrs := model.MgrSummary{
ActiveName: cephStatus.MgrMap.ActiveName,
ActiveAddr: cephStatus.MgrMap.ActiveAddr,
Available: cephStatus.MgrMap.Available,
}
for _, standby := range cephStatus.MgrMap.Standbys {
mgrs.Standbys = append(mgrs.Standbys, standby.Name)
}
// generate the usage Summary
usageSummary := model.UsageSummary{
DataBytes: cephStatus.PgMap.DataBytes,
@@ -99,6 +109,7 @@ func (h *Handler) GetStatusDetails(w http.ResponseWriter, r *http.Request) {
OverallStatus: ceph.HealthToModelHealthStatus(cephStatus.Health.OverallStatus),
SummaryMessages: summaries,
Monitors: monitors,
Mgrs: mgrs,
OSDs: osds,
PGs: pgSummary,
Usage: usageSummary,
pkg/ceph/client/auth.go
+1 -1
View File
@@ -40,7 +40,7 @@ func AuthGetOrCreateKey(context *clusterd.Context, clusterName, name string, cap
return "", fmt.Errorf("failed get-or-create-key %s: %+v", name, err)
}
logger.Infof("Parsing key: %v", buf)
logger.Infof("Parsing key: %v", string(buf))
return parseAuthKey(buf)
}
pkg/ceph/client/status.go
+18 -2
View File
@@ -48,7 +48,8 @@ type CephStatus struct {
OsdMap struct {
OsdMap OsdMap `json:"osdmap"`
} `json:"osdmap"`
PgMap PgMap `json:"pgmap"`
PgMap PgMap `json:"pgmap"`
MgrMap MgrMap `json:"mgrmap"`
}
type HealthStatus struct {
@@ -95,6 +96,20 @@ type MonMap struct {
Mons []MonMapEntry `json:"mons"`
}
type MgrMap struct {
Epoch int `json:"epoch"`
ActiveGID int `json:"active_gid"`
ActiveName string `json:"active_name"`
ActiveAddr string `json:"active_addr"`
Available bool `json:"available"`
Standbys []MgrStandby `json:"standbys"`
}
type MgrStandby struct {
GID int `json:"gid"`
Name string `json:"name"`
}
type OsdMap struct {
Epoch int `json:"epoch"`
NumOsd int `json:"num_osds"`
@@ -121,12 +136,13 @@ type PgStateEntry struct {
}
func Status(context *clusterd.Context, clusterName string) (CephStatus, error) {
args := []string{"status", "--format", "json"}
args := []string{"status"}
buf, err := ExecuteCephCommand(context, clusterName, args)
if err != nil {
return CephStatus{}, fmt.Errorf("failed to get status: %+v", err)
}
logger.Infof("STATUS: %s", string(buf))
var status CephStatus
if err := json.Unmarshal(buf, &status); err != nil {
return CephStatus{}, fmt.Errorf("failed to unmarshal status response: %+v", err)
pkg/ceph/mgr/daemon.go
+135
View File
@@ -0,0 +1,135 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mgr
import (
"fmt"
"path"
"regexp"
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/pkg/ceph/mon"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/util"
"github.com/rook/rook/pkg/util/proc"
)
var (
logger = capnslog.NewPackageLogger("github.com/rook/rook", "cephmgr")
keyringTemplate = `
[mgr.%s]
key = %s
caps mon = "allow *"
`
)
const (
cephmgr = "ceph-mgr"
)
type Config struct {
InProc bool
ClusterInfo *mon.ClusterInfo
Name string
Keyring string
}
func Run(context *clusterd.Context, config *Config) error {
logger.Infof("Starting MGR %s with keyring %s", config.Name, config.Keyring)
err := generateConfigFiles(context, config)
if err != nil {
return fmt.Errorf("failed to generate mgr config files. %+v", err)
}
_, err = startMgr(context, config)
if err != nil {
return fmt.Errorf("failed to run mgr. %+v", err)
}
return err
}
func generateConfigFiles(context *clusterd.Context, config *Config) error {
// write the latest config to the config dir
/*if err := mon.GenerateAdminConnectionConfig(context, config.ClusterInfo); err != nil {
return fmt.Errorf("failed to write connection config. %+v", err)
}*/
keyringPath := getMgrKeyringPath(context.ConfigDir, config.Name)
confDir := getMgrConfDir(context.ConfigDir, config.Name)
username := fmt.Sprintf("mgr.%s", config.Name)
settings := map[string]string{
"mgr data": confDir,
}
logger.Infof("Conf files: dir=%s keyring=%s", confDir, keyringPath)
_, err := mon.GenerateConfigFile(context, config.ClusterInfo, confDir,
username, keyringPath, false, nil, settings)
if err != nil {
return fmt.Errorf("failed to create config file. %+v", err)
}
keyringEval := func(key string) string {
return fmt.Sprintf(keyringTemplate, config.Name, key)
}
err = mon.WriteKeyring(keyringPath, config.Keyring, keyringEval)
if err != nil {
return fmt.Errorf("failed to create mds keyring. %+v", err)
}
return nil
}
func startMgr(context *clusterd.Context, config *Config) (mgrProc *proc.MonitoredProc, err error) {
// start the mgr daemon in the foreground with the given config
logger.Infof("starting ceph-mgr")
confFile := getMgrConfFilePath(context.ConfigDir, config.Name, config.ClusterInfo.Name)
util.WriteFileToLog(logger, confFile)
keyringPath := getMgrKeyringPath(context.ConfigDir, config.Name)
util.WriteFileToLog(logger, keyringPath)
args := []string{
"--foreground",
fmt.Sprintf("--cluster=%s", config.ClusterInfo.Name),
fmt.Sprintf("--conf=%s", confFile),
fmt.Sprintf("--keyring=%s", keyringPath),
"-i", config.Name,
}
if config.InProc {
err = context.ProcMan.Run(cephmgr, cephmgr, args...)
} else {
mgrProc, err = context.ProcMan.Start(cephmgr, cephmgr, regexp.QuoteMeta(cephmgr), proc.ReuseExisting, args...)
}
if err != nil {
err = fmt.Errorf("failed to start mgr: %+v", err)
}
return
}
func getMgrConfDir(dir, name string) string {
return path.Join(dir, fmt.Sprintf("mgr%s", name))
}
func getMgrConfFilePath(dir, name, clusterName string) string {
return path.Join(getMgrConfDir(dir, name), fmt.Sprintf("%s.config", clusterName))
}
func getMgrKeyringPath(dir, name string) string {
return path.Join(getMgrConfDir(dir, name), "keyring")
}
pkg/ceph/mgr/leader.go
+41
View File
@@ -0,0 +1,41 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mgr
import (
"fmt"
"github.com/rook/rook/pkg/ceph/client"
"github.com/rook/rook/pkg/clusterd"
)
func getKeyringProperties(name string) (string, []string) {
username := fmt.Sprintf("mgr.%s", name)
access := []string{"mon", "allow *"}
return username, access
}
// create a keyring for the mds client with a limited set of privileges
func CreateKeyring(context *clusterd.Context, clusterName, name string) (string, error) {
// get-or-create-key for the user account
username, access := getKeyringProperties(name)
keyring, err := client.AuthGetOrCreateKey(context, clusterName, username, access)
if err != nil {
return "", fmt.Errorf("failed to get or create auth key for %s. %+v", username, err)
}
return keyring, nil
}
pkg/ceph/mon/agent.go
+2 -2
View File
@@ -33,9 +33,9 @@ const (
monitorKeyringTemplate = `
[mon.]
key = %s
caps mon = "allow *"` + adminKeyringTemplate
caps mon = "allow *"` + AdminKeyringTemplate
adminKeyringTemplate = `
AdminKeyringTemplate = `
[client.admin]
key = %s
auid = 0
pkg/ceph/mon/config.go
+2 -2
View File
@@ -95,7 +95,7 @@ func GetConfFilePath(root, clusterName string) string {
func GenerateAdminConnectionConfig(context *clusterd.Context, cluster *ClusterInfo) error {
root := path.Join(context.ConfigDir, cluster.Name)
keyring := fmt.Sprintf(adminKeyringTemplate, cluster.AdminSecret)
keyring := fmt.Sprintf(AdminKeyringTemplate, cluster.AdminSecret)
keyringPath := path.Join(root, fmt.Sprintf("%s.keyring", client.AdminUsername))
err := writeKeyring(cluster, keyring, keyringPath)
if err != nil {
@@ -170,7 +170,7 @@ func GenerateConfigFile(context *clusterd.Context, cluster *ClusterInfo, pathRoo
// write the entire config to disk
filePath := GetConfFilePath(pathRoot, cluster.Name)
logger.Debugf("writing config file %s", filePath)
logger.Infof("writing config file %s", filePath)
if err := configFile.SaveTo(filePath); err != nil {
return "", fmt.Errorf("failed to save config file %s. %+v", filePath, err)
}
pkg/model/status.go
+8
View File
@@ -30,6 +30,7 @@ type StatusDetails struct {
Monitors []MonitorSummary `json:"monitors"`
OSDs OSDSummary `json:"osd"`
PGs PGSummary `json:"pg"`
Mgrs MgrSummary `json:"mgr"`
Usage UsageSummary `json:"usage"`
}
@@ -58,6 +59,13 @@ type PGSummary struct {
StateCounts map[string]int `json:"stateCount"`
}
type MgrSummary struct {
ActiveName string `json:"active_name"`
ActiveAddr string `json:"active_addr"`
Available bool `json:"available"`
Standbys []string `json:"standbys"`
}
type UsageSummary struct {
DataBytes uint64 `json:"data"`
UsedBytes uint64 `json:"used"`
pkg/operator/api/api.go
+3
View File
@@ -18,6 +18,7 @@ package api
import (
"fmt"
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/model"
"github.com/rook/rook/pkg/operator/k8sutil"
@@ -30,6 +31,8 @@ import (
"k8s.io/client-go/pkg/apis/rbac/v1beta1"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-api")
const (
DeploymentName = "rook-api"
)
pkg/operator/api/log.go
-22
View File
@@ -1,22 +0,0 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package api
import (
"github.com/coreos/pkg/capnslog"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-api")
pkg/operator/cluster/cluster.go
+8
View File
@@ -31,6 +31,7 @@ import (
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/api"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/operator/mgr"
"github.com/rook/rook/pkg/operator/mon"
"github.com/rook/rook/pkg/operator/osd"
"github.com/rook/rook/pkg/operator/rgw"
@@ -50,6 +51,7 @@ type Cluster struct {
v1.ObjectMeta `json:"metadata,omitempty"`
Spec `json:"spec"`
mons *mon.Cluster
mgrs *mgr.Cluster
osds *osd.Cluster
apis *api.Cluster
rgws *rgw.Cluster
@@ -90,6 +92,12 @@ func (c *Cluster) CreateInstance() error {
return fmt.Errorf("failed to start the mons. %+v", err)
}
c.mgrs = mgr.New(c.context, c.Name, c.Namespace, c.Spec.VersionTag)
err = c.mgrs.Start()
if err != nil {
return fmt.Errorf("failed to start the ceph mgr. %+v", err)
}
c.apis = api.New(c.context, c.Name, c.Namespace, c.Spec.VersionTag, c.Spec.Placement.GetAPI())
err = c.apis.Start()
if err != nil {
pkg/operator/k8sutil/k8sutil.go
+3
View File
@@ -22,9 +22,12 @@ import (
"fmt"
"time"
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/pkg/clusterd"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-k8sutil")
const (
Namespace = "rook"
DefaultNamespace = "default"
pkg/operator/k8sutil/log.go
-22
View File
@@ -1,22 +0,0 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package k8sutil
import (
"github.com/coreos/pkg/capnslog"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-k8sutil")
pkg/operator/mds/log.go
-22
View File
@@ -1,22 +0,0 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mds
import (
"github.com/coreos/pkg/capnslog"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mds")
pkg/operator/mds/mds.go
+7 -9
View File
@@ -18,8 +18,8 @@ package mds
import (
"fmt"
"github.com/coreos/pkg/capnslog"
cephmds "github.com/rook/rook/pkg/ceph/mds"
"github.com/rook/rook/pkg/ceph/mon"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
opmon "github.com/rook/rook/pkg/operator/mon"
@@ -30,6 +30,8 @@ import (
extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mds")
const (
appName = "mds"
dataPoolSuffix = "-data"
@@ -59,22 +61,18 @@ func New(context *clusterd.Context, name, namespace, version string, placement k
}
}
func (c *Cluster) Start(clientset kubernetes.Interface, cluster *mon.ClusterInfo) error {
func (c *Cluster) Start() error {
logger.Infof("start running mds")
if cluster == nil || len(cluster.Monitors) == 0 {
return fmt.Errorf("missing mons to start mds")
}
id := "mds1"
err := c.createKeyring(clientset, cluster, id)
err := c.createKeyring(c.context.Clientset, id)
if err != nil {
return fmt.Errorf("failed to create mds keyring. %+v", err)
}
// start the deployment
deployment := c.makeDeployment(id)
_, err = clientset.ExtensionsV1beta1().Deployments(c.Namespace).Create(deployment)
_, err = c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Create(deployment)
if err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create mds deployment. %+v", err)
@@ -87,7 +85,7 @@ func (c *Cluster) Start(clientset kubernetes.Interface, cluster *mon.ClusterInfo
return nil
}
func (c *Cluster) createKeyring(clientset kubernetes.Interface, cluster *mon.ClusterInfo, id string) error {
func (c *Cluster) createKeyring(clientset kubernetes.Interface, id string) error {
_, err := clientset.CoreV1().Secrets(c.Namespace).Get(appName, metav1.GetOptions{})
if err == nil {
logger.Infof("the mds keyring was already generated")
pkg/operator/mds/mds_test.go
+11 -11
View File
@@ -28,12 +28,10 @@ import (
exectest "github.com/rook/rook/pkg/util/exec/test"
"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/client-go/pkg/api/v1"
)
func TestStartMDS(t *testing.T) {
info := testop.CreateClusterInfo(1)
executor := &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(actionName string, command string, args ...string) (string, error) {
return "{\"key\":\"mysecurekey\"}", nil
@@ -42,27 +40,29 @@ func TestStartMDS(t *testing.T) {
configDir, _ := ioutil.TempDir("", "")
defer os.RemoveAll(configDir)
c := New(&clusterd.Context{Executor: executor, ConfigDir: configDir}, "myname", "ns", "myversion", k8sutil.Placement{})
context := &clusterd.Context{
Executor: executor,
ConfigDir: configDir,
KubeContext: clusterd.KubeContext{Clientset: testop.New(3)}}
c := New(context, "myname", "ns", "myversion", k8sutil.Placement{})
defer os.RemoveAll(c.dataDir)
clientset := testop.New(3)
// start a basic cluster
err := c.Start(clientset, info)
err := c.Start()
assert.Nil(t, err)
validateStart(t, c, clientset)
validateStart(t, c)
// starting again should be a no-op
err = c.Start(clientset, info)
err = c.Start()
assert.Nil(t, err)
validateStart(t, c, clientset)
validateStart(t, c)
}
func validateStart(t *testing.T, c *Cluster, clientset *fake.Clientset) {
func validateStart(t *testing.T, c *Cluster) {
r, err := clientset.ExtensionsV1beta1().Deployments(c.Namespace).Get("mds", metav1.GetOptions{})
r, err := c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Get("mds", metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, "mds", r.Name)
}
pkg/operator/mgr/mgr.go
+173
View File
@@ -0,0 +1,173 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mgr
import (
"fmt"
"github.com/coreos/pkg/capnslog"
cephmgr "github.com/rook/rook/pkg/ceph/mgr"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
opmon "github.com/rook/rook/pkg/operator/mon"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/pkg/api/v1"
extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mgr")
const (
appName = "cephmgr"
keyringName = "keyring"
mgrCount = 2
)
type Cluster struct {
Name string
Namespace string
Version string
Replicas int32
context *clusterd.Context
dataDir string
}
func New(context *clusterd.Context, name, namespace, version string) *Cluster {
return &Cluster{
context: context,
Name: name,
Namespace: namespace,
Version: version,
Replicas: 1,
dataDir: k8sutil.DataDir,
}
}
func (c *Cluster) Start() error {
logger.Infof("start running mgr")
for i := 1; i <= mgrCount; i++ {
name := fmt.Sprintf("%s%d", appName, i)
err := c.createKeyring(c.Name, name)
if err != nil {
return fmt.Errorf("failed to create mgr keyring. %+v", err)
}
// start the deployment
deployment := c.makeDeployment(name)
_, err = c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Create(deployment)
if err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create mgr deployment. %+v", err)
}
logger.Infof("%s deployment already exists", name)
} else {
logger.Infof("%s deployment started", name)
}
}
return nil
}
func (c *Cluster) makeDeployment(name string) *extensions.Deployment {
deployment := &extensions.Deployment{}
deployment.Name = name
deployment.Namespace = c.Namespace
podSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Labels: c.getLabels(),
Annotations: map[string]string{},
},
Spec: v1.PodSpec{
Containers: []v1.Container{c.mgrContainer(name)},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: []v1.Volume{
{Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}},
k8sutil.ConfigOverrideVolume(),
},
},
}
deployment.Spec = extensions.DeploymentSpec{Template: podSpec, Replicas: &c.Replicas}
return deployment
}
func (c *Cluster) mgrContainer(name string) v1.Container {
command := fmt.Sprintf("/usr/local/bin/rookd mgr --config-dir=%s", k8sutil.DataDir)
return v1.Container{
// FIX: sleep 5 for a flaky network setup
Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)},
Name: name,
Image: k8sutil.MakeRookImage(c.Version),
VolumeMounts: []v1.VolumeMount{
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
k8sutil.ConfigOverrideMount(),
},
Env: []v1.EnvVar{
{Name: "ROOKD_MGR_NAME", Value: name},
{Name: "ROOKD_MGR_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: name}, Key: keyringName}}},
opmon.ClusterNameEnvVar(c.Name),
opmon.MonEndpointEnvVar(),
opmon.MonSecretEnvVar(),
opmon.AdminSecretEnvVar(),
k8sutil.ConfigOverrideEnvVar(),
},
}
}
func (c *Cluster) getLabels() map[string]string {
return map[string]string{
k8sutil.AppAttr: appName,
k8sutil.ClusterAttr: c.Namespace,
}
}
func (c *Cluster) createKeyring(clusterName, name string) error {
_, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(name, metav1.GetOptions{})
if err == nil {
logger.Infof("the mgr keyring was already generated")
return nil
}
if !errors.IsNotFound(err) {
return fmt.Errorf("failed to get mgr secrets. %+v", err)
}
// get-or-create-key for the user account
keyring, err := cephmgr.CreateKeyring(c.context, clusterName, name)
if err != nil {
return fmt.Errorf("failed to create mgr keyring. %+v", err)
}
// Store the keyring in a secret
secrets := map[string]string{
keyringName: keyring,
}
secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: c.Namespace},
StringData: secrets,
Type: k8sutil.RookType,
}
_, err = c.context.Clientset.CoreV1().Secrets(c.Namespace).Create(secret)
if err != nil {
return fmt.Errorf("failed to save mgr secrets. %+v", err)
}
return nil
}
pkg/operator/mgr/mgr_test.go
+92
View File
@@ -0,0 +1,92 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mgr
import (
"fmt"
"io/ioutil"
"os"
"strings"
"testing"
"github.com/rook/rook/pkg/clusterd"
testop "github.com/rook/rook/pkg/operator/test"
exectest "github.com/rook/rook/pkg/util/exec/test"
"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/pkg/api/v1"
)
func TestStartMGR(t *testing.T) {
executor := &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(actionName string, command string, args ...string) (string, error) {
return "{\"key\":\"mysecurekey\"}", nil
},
}
configDir, _ := ioutil.TempDir("", "")
defer os.RemoveAll(configDir)
context := &clusterd.Context{
Executor: executor,
ConfigDir: configDir,
KubeContext: clusterd.KubeContext{Clientset: testop.New(3)}}
c := New(context, "myname", "ns", "myversion")
defer os.RemoveAll(c.dataDir)
// start a basic service
err := c.Start()
assert.Nil(t, err)
validateStart(t, c)
// starting again should be a no-op
err = c.Start()
assert.Nil(t, err)
validateStart(t, c)
}
func validateStart(t *testing.T, c *Cluster) {
r, err := c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Get(appName, metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, appName, r.Name)
}
func TestPodSpec(t *testing.T) {
c := New(nil, "myname", "ns", "myversion")
d := c.makeDeployment("mgr1")
assert.NotNil(t, d)
assert.Equal(t, appName, d.Name)
assert.Equal(t, v1.RestartPolicyAlways, d.Spec.Template.Spec.RestartPolicy)
assert.Equal(t, 2, len(d.Spec.Template.Spec.Volumes))
assert.Equal(t, "rook-data", d.Spec.Template.Spec.Volumes[0].Name)
assert.Equal(t, appName, d.ObjectMeta.Name)
assert.Equal(t, appName, d.Spec.Template.ObjectMeta.Labels["app"])
assert.Equal(t, c.Namespace, d.Spec.Template.ObjectMeta.Labels["rook_cluster"])
assert.Equal(t, 0, len(d.ObjectMeta.Annotations))
cont := d.Spec.Template.Spec.Containers[0]
assert.Equal(t, "quay.io/rook/rookd:myversion", cont.Image)
assert.Equal(t, 2, len(cont.VolumeMounts))
assert.Equal(t, 7, len(cont.Env))
expectedCommand := fmt.Sprintf("/usr/local/bin/rookd mgr --config-dir=/var/lib/rook")
assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2])
}
pkg/operator/mon/log.go
-22
View File
@@ -1,22 +0,0 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mon
import (
"github.com/coreos/pkg/capnslog"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mon")
pkg/operator/mon/mon.go
+19 -1
View File
@@ -21,6 +21,7 @@ import (
"strings"
"time"
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/pkg/ceph/client"
"github.com/rook/rook/pkg/ceph/mon"
"github.com/rook/rook/pkg/clusterd"
@@ -32,6 +33,8 @@ import (
"k8s.io/client-go/pkg/api/v1"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mon")
const (
appName = "mon"
monNodeAttr = "mon_node"
@@ -146,6 +149,12 @@ func (c *Cluster) initClusterInfo() error {
if err != nil {
return fmt.Errorf("failed to get mon config. %+v", err)
}
// make sure we have the connection info generated so connections can happen
err = c.writeConnectionConfig()
if err != nil {
return err
}
return nil
}
@@ -350,13 +359,22 @@ func (c *Cluster) saveMonConfig() error {
logger.Infof("saved mon endpoints to config map %+v", configMap.Data)
// write the latest config to the config dir
if err := mon.GenerateAdminConnectionConfig(c.context, c.clusterInfo); err != nil {
if err := c.writeConnectionConfig(); err != nil {
return fmt.Errorf("failed to write connection config for new mons. %+v", err)
}
return nil
}
func (c *Cluster) writeConnectionConfig() error {
// write the latest config to the config dir
if err := mon.GenerateAdminConnectionConfig(c.context, c.clusterInfo); err != nil {
return fmt.Errorf("failed to write connection config. %+v", err)
}
return nil
}
func (c *Cluster) loadMonConfig() error {
cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(monConfigMapName, metav1.GetOptions{})
if err != nil {
pkg/operator/osd/log.go
-22
View File
@@ -1,22 +0,0 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package osd
import (
"github.com/coreos/pkg/capnslog"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-osd")
pkg/operator/osd/osd.go
+3
View File
@@ -22,6 +22,7 @@ import (
"strconv"
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
opmon "github.com/rook/rook/pkg/operator/mon"
@@ -31,6 +32,8 @@ import (
extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-osd")
const (
appName = "osd"
appNameFmt = "osd-%s"
pkg/operator/rgw/log.go
-22
View File
@@ -1,22 +0,0 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package rgw
import (
"github.com/coreos/pkg/capnslog"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-rgw")
pkg/operator/rgw/rgw.go
+7 -9
View File
@@ -18,7 +18,7 @@ package rgw
import (
"fmt"
"github.com/rook/rook/pkg/ceph/mon"
"github.com/coreos/pkg/capnslog"
cephrgw "github.com/rook/rook/pkg/ceph/rgw"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
@@ -30,6 +30,8 @@ import (
extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-rgw")
const (
appName = "rgw"
keyringName = "keyring"
@@ -55,14 +57,10 @@ func New(context *clusterd.Context, name, namespace, version string, placement k
}
}
func (c *Cluster) Start(cluster *mon.ClusterInfo) error {
func (c *Cluster) Start() error {
logger.Infof("start running rgw")
if cluster == nil || len(cluster.Monitors) == 0 {
return fmt.Errorf("missing mons to start rgw")
}
err := c.createKeyring(cluster)
err := c.createKeyring()
if err != nil {
return fmt.Errorf("failed to create rgw keyring. %+v", err)
}
@@ -88,7 +86,7 @@ func (c *Cluster) Start(cluster *mon.ClusterInfo) error {
return nil
}
func (c *Cluster) createKeyring(cluster *mon.ClusterInfo) error {
func (c *Cluster) createKeyring() error {
_, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(appName, metav1.GetOptions{})
if err == nil {
logger.Infof("the rgw keyring was already generated")
@@ -100,7 +98,7 @@ func (c *Cluster) createKeyring(cluster *mon.ClusterInfo) error {
// create the keyring
logger.Infof("generating rgw keyring")
keyring, err := cephrgw.CreateKeyring(c.context, cluster.Name)
keyring, err := cephrgw.CreateKeyring(c.context, c.Name)
if err != nil {
return fmt.Errorf("failed to create keyring. %+v", err)
}
pkg/operator/rgw/rgw_test.go
+2 -3
View File
@@ -35,7 +35,6 @@ import (
func TestStartRGW(t *testing.T) {
clientset := testop.New(3)
info := testop.CreateClusterInfo(1)
executor := &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(actionName string, command string, args ...string) (string, error) {
return "{\"key\":\"mysecurekey\"}", nil
@@ -47,13 +46,13 @@ func TestStartRGW(t *testing.T) {
c := New(&clusterd.Context{KubeContext: clusterd.KubeContext{Clientset: clientset}, Executor: executor, ConfigDir: configDir}, "myname", "ns", "version", k8sutil.Placement{})
// start a basic cluster
err := c.Start(info)
err := c.Start()
assert.Nil(t, err)
validateStart(t, c, clientset)
// starting again should be a no-op
err = c.Start(info)
err = c.Start()
assert.Nil(t, err)
validateStart(t, c, clientset)