operator configures a file system and starts mds

This commit is contained in:
Travis Nielsen committed 2017-02-08 15:46:47 -08:00
1 parent dc7501e707
commit a414d64b82
8 files changed
+400 -39

No files matched your search

+27 -2
View File
@@ -18,10 +18,19 @@ limitations under the License.
package main
import (
"github.com/rook/rook/pkg/cephmgr/cephd"
"github.com/rook/rook/pkg/cephmgr/mds"
"github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/util/flags"
"github.com/spf13/cobra"
)
var (
mdsID string
mdsKeyring string
)
var mdsCmd = &cobra.Command{
Use: "mds",
Short: "Generates mds config and runs the mds daemon",
@@ -29,16 +38,32 @@ var mdsCmd = &cobra.Command{
}
func init() {
mdsCmd.Flags().StringVar(&mdsID, "mds-id", "", "the mds ID")
mdsCmd.Flags().StringVar(&mdsKeyring, "mds-keyring", "", "the mds keyring")
flags.SetFlagsFromEnv(mdsCmd.Flags(), "ROOKD")
mdsCmd.RunE = startMDS
}
func startMDS(cmd *cobra.Command, args []string) error {
if err := flags.VerifyRequiredFlags(mdsCmd, []string{""}); err != nil {
if err := flags.VerifyRequiredFlags(mdsCmd, []string{"mon-endpoints", "cluster-name", "mon-secret", "admin-secret", "mds-id", "mds-keyring"}); err != nil {
return err
}
setLogLevel()
// mds.Start(config)
clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
config := &mds.Config{
ID: mdsID,
Keyring: mdsKeyring,
ClusterInfo: &clusterInfo,
CephLauncher: cephd.New(),
InProc: true,
}
context := clusterd.NewDaemonContext(cfg.dataDir, cfg.cephConfigOverride, cfg.logLevel)
mds.Run(context, config)
return nil
}
+23 -35
View File
@@ -18,7 +18,6 @@ package mds
import (
"fmt"
"path"
"regexp"
etcd "github.com/coreos/etcd/client"
ctx "golang.org/x/net/context"
@@ -89,7 +88,7 @@ func (a *mdsAgent) ConfigureLocalService(context *clusterd.Context) error {
}
defer conn.Shutdown()
err = createMDSKeyring(conn, mdsID, context.ConfigDir)
err = createKeyringAndSave(conn, mdsID, context.ConfigDir)
if err != nil {
return fmt.Errorf("failed to create mds keyring. %+v", err)
}
@@ -119,11 +118,16 @@ func (a *mdsAgent) DestroyLocalService(context *clusterd.Context) error {
return nil
}
// create a keyring for the mds client with a limited set of privileges
func createMDSKeyring(conn client.Connection, id, configDir string) error {
func getKeyringProperties(id string) (string, []string) {
username := fmt.Sprintf("mds.%s", id)
keyringPath := getMDSKeyringPath(configDir, id)
access := []string{"osd", "allow *", "mds", "allow", "mon", "allow profile mds"}
return username, access
}
// create a keyring for the mds client with a limited set of privileges
func createKeyringAndSave(conn client.Connection, id, configDir string) error {
username, access := getKeyringProperties(id)
keyringPath := getMDSKeyringPath(configDir, id)
keyringEval := func(key string) string {
return fmt.Sprintf(keyringTemplate, id, key)
}
@@ -131,26 +135,22 @@ func createMDSKeyring(conn client.Connection, id, configDir string) error {
return mon.CreateKeyring(conn, username, keyringPath, access, keyringEval)
}
// create a keyring for the mds client with a limited set of privileges
func CreateKeyring(conn client.Connection, id string) (string, error) {
// get-or-create-key for the user account
username, access := getKeyringProperties(id)
keyring, err := client.AuthGetOrCreateKey(conn, username, access)
if err != nil {
return "", fmt.Errorf("failed to get or create auth key for %s. %+v", username, err)
}
return keyring, nil
}
func (a *mdsAgent) startMDS(context *clusterd.Context, cluster *mon.ClusterInfo, id string) error {
// start the monitor daemon in the foreground with the given config
logger.Infof("starting mds %s", id)
confFile := getMDSConfFilePath(context.ConfigDir, id, cluster.Name)
util.WriteFileToLog(logger, confFile)
mdsNameArg := fmt.Sprintf("--name=mds.%s", id)
mdsProc, err := context.ProcMan.Start(
fmt.Sprintf("mds%s", id),
"mds",
regexp.QuoteMeta(mdsNameArg),
proc.ReuseExisting,
"--foreground",
mdsNameArg,
fmt.Sprintf("--cluster=%s", cluster.Name),
fmt.Sprintf("--conf=%s", confFile),
fmt.Sprintf("--keyring=%s", getMDSKeyringPath(context.ConfigDir, id)),
"-i", id)
config := &Config{ID: id, InProc: false, ClusterInfo: cluster}
mdsProc, err := startMDS(clusterd.ToDaemonContext(context), config)
if err != nil {
return fmt.Errorf("failed to start mds %s: %+v", id, err)
}
@@ -197,15 +197,3 @@ func getMDSIDKey(nodeID string, applied bool) string {
func getMDSFileSystemKey(nodeID string, applied bool) string {
return path.Join(getMDSDesiredNodesKey(applied), nodeID, "filesystem")
}
func getMDSConfDir(dir, id string) string {
return path.Join(dir, fmt.Sprintf("mds%s", id))
}
func getMDSConfFilePath(dir, id, clusterName string) string {
return path.Join(getMDSConfDir(dir, id), fmt.Sprintf("%s.config", clusterName))
}
func getMDSKeyringPath(dir, id string) string {
return path.Join(getMDSConfDir(dir, id), "keyring")
}
+111
View File
@@ -0,0 +1,111 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mds
import (
"fmt"
"path"
"regexp"
"github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/util"
"github.com/rook/rook/pkg/util/proc"
)
type Config struct {
ID string
Keyring string
InProc bool
ClusterInfo *mon.ClusterInfo
mon.CephLauncher
}
func Run(context *clusterd.DaemonContext, config *Config) error {
err := generateConfigFiles(context, config)
if err != nil {
return fmt.Errorf("failed to generate mds config files. %+v", err)
}
_, err = startMDS(context, config)
if err != nil {
return fmt.Errorf("failed to run mds. %+v", err)
}
return err
}
func generateConfigFiles(context *clusterd.DaemonContext, config *Config) error {
keyringPath := getMDSKeyringPath(context.ConfigDir, config.ID)
_, err := mon.GenerateConnectionConfigFile(clusterd.ToContext(context), config.ClusterInfo, getMDSConfDir(context.ConfigDir, config.ID),
fmt.Sprintf("mds.%s", config.ID), keyringPath)
if err != nil {
return fmt.Errorf("failed to create mds config file. %+v", err)
}
keyringEval := func(key string) string {
return fmt.Sprintf(keyringTemplate, config.ID, key)
}
err = mon.WriteKeyring(keyringPath, config.Keyring, keyringEval)
if err != nil {
return fmt.Errorf("failed to create mds keyring. %+v", err)
}
return nil
}
func startMDS(context *clusterd.DaemonContext, config *Config) (mdsProc *proc.MonitoredProc, err error) {
// start the mds daemon in the foreground with the given config
logger.Infof("starting mds %s", config.ID)
confFile := getMDSConfFilePath(context.ConfigDir, config.ID, config.ClusterInfo.Name)
util.WriteFileToLog(logger, confFile)
mdsNameArg := fmt.Sprintf("--name=mds.%s", config.ID)
args := []string{
"--foreground",
mdsNameArg,
fmt.Sprintf("--cluster=%s", config.ClusterInfo.Name),
fmt.Sprintf("--conf=%s", confFile),
fmt.Sprintf("--keyring=%s", getMDSKeyringPath(context.ConfigDir, config.ID)),
"-i", config.ID,
}
if config.InProc {
err = config.CephLauncher.Run("mds", args...)
} else {
mdsProc, err = context.ProcMan.Start("mds", "mds", regexp.QuoteMeta(mdsNameArg), proc.ReuseExisting, args...)
}
if err != nil {
err = fmt.Errorf("failed to start mds: %+v", err)
}
return
}
func getMDSConfDir(dir, id string) string {
return path.Join(dir, fmt.Sprintf("mds%s", id))
}
func getMDSConfFilePath(dir, id, clusterName string) string {
return path.Join(getMDSConfDir(dir, id), fmt.Sprintf("%s.config", clusterName))
}
func getMDSKeyringPath(dir, id string) string {
return path.Join(getMDSConfDir(dir, id), "keyring")
}
+1 -1
View File
@@ -28,7 +28,7 @@ import (
const (
Namespace = "rook"
DataDirVolume = "rook-data"
DataDir = "/var/lib/rook/data"
DataDir = "/var/lib/rook"
)
func IsKubernetesResourceAlreadyExistError(err error) bool {
+22
View File
@@ -0,0 +1,22 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mds
import (
"github.com/coreos/pkg/capnslog"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook-operator", "op-mds")
+207
View File
@@ -0,0 +1,207 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mds
import (
"fmt"
"github.com/rook/rook/pkg/cephmgr/client"
cephmds "github.com/rook/rook/pkg/cephmgr/mds"
"github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
k8smon "github.com/rook/rook/pkg/operator/mon"
"k8s.io/client-go/1.5/kubernetes"
"k8s.io/client-go/1.5/pkg/api/v1"
extensions "k8s.io/client-go/1.5/pkg/apis/extensions/v1beta1"
)
const (
appName = "mds"
dataPoolSuffix = "-data"
metadataPoolSuffix = "-metadata"
keyringName = "keyring"
)
type Cluster struct {
Namespace string
Version string
Replicas int32
factory client.ConnectionFactory
}
func New(namespace, version string, factory client.ConnectionFactory) *Cluster {
return &Cluster{
Namespace: namespace,
Version: version,
Replicas: 1,
factory: factory,
}
}
func (c *Cluster) Start(clientset *kubernetes.Clientset, cluster *mon.ClusterInfo) error {
logger.Infof("start running mds")
if cluster == nil || len(cluster.Monitors) == 0 {
return fmt.Errorf("missing mons to start mds")
}
context := &clusterd.DaemonContext{ConfigDir: k8sutil.DataDir}
conn, err := mon.ConnectToClusterAsAdmin(clusterd.ToContext(context), c.factory, cluster)
if err != nil {
return fmt.Errorf("failed to connect to cluster as admin: %+v", err)
}
defer conn.Shutdown()
// creating a named file system will be done with third party resources in the future
err = c.createFilesystem("myfs", "myfspool", cluster, conn)
if err != nil {
return fmt.Errorf("failed to create file system. %+v", err)
}
id := "mds1"
err = c.createKeyring(clientset, context, cluster, conn, id)
if err != nil {
return fmt.Errorf("failed to create mds keyring. %+v", err)
}
// start the deployment
deployment, err := c.makeDeployment(cluster, id)
_, err = clientset.Deployments(c.Namespace).Create(deployment)
if err != nil {
if !k8sutil.IsKubernetesResourceAlreadyExistError(err) {
return fmt.Errorf("failed to create mds deployment. %+v", err)
}
logger.Infof("mds deployment already exists")
} else {
logger.Infof("mds deployment started")
}
return nil
}
func (c *Cluster) createFilesystem(id, poolName string, cluster *mon.ClusterInfo, conn client.Connection) error {
_, err := client.GetFilesystem(conn, id)
if err == nil {
logger.Infof("file system %s already exists", id)
return nil
}
dataPool := poolName + dataPoolSuffix
metadataPool := poolName + metadataPoolSuffix
// Create the metadata and data pools
pool := client.CephStoragePoolDetails{Name: dataPool}
_, err = client.CreatePool(conn, pool)
if err != nil {
return fmt.Errorf("failed to create data pool '%s': %+v", dataPool, err)
}
pool = client.CephStoragePoolDetails{Name: metadataPool}
_, err = client.CreatePool(conn, pool)
if err != nil {
return fmt.Errorf("failed to create metadata pool '%s': %+v", metadataPool, err)
}
// create the file system
if err := client.CreateFilesystem(conn, id, metadataPool, dataPool); err != nil {
return err
}
logger.Infof("created file system %s on data pool %s and metadata pool %s", id, dataPool, metadataPool)
return nil
}
func (c *Cluster) createKeyring(clientset *kubernetes.Clientset, context *clusterd.DaemonContext, cluster *mon.ClusterInfo, conn client.Connection, id string) error {
_, err := clientset.Secrets(c.Namespace).Get(appName)
if err == nil {
logger.Infof("the mds keyring was already generated")
return nil
}
if !k8sutil.IsKubernetesResourceNotFoundError(err) {
return fmt.Errorf("failed to get mds secrets. %+v", err)
}
// get-or-create-key for the user account
keyring, err := cephmds.CreateKeyring(conn, id)
if err != nil {
return fmt.Errorf("failed to create mds keyring. %+v", err)
}
// Store the keyring in a secret
secrets := map[string]string{
keyringName: keyring,
}
_, err = clientset.Secrets(c.Namespace).Create(&v1.Secret{ObjectMeta: v1.ObjectMeta{Name: appName}, StringData: secrets})
if err != nil {
return fmt.Errorf("failed to save mds secrets. %+v", err)
}
return nil
}
func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo, id string) (*extensions.Deployment, error) {
deployment := &extensions.Deployment{}
deployment.Name = appName
deployment.Namespace = c.Namespace
podSpec := v1.PodTemplateSpec{
ObjectMeta: v1.ObjectMeta{
Name: appName,
Labels: getLabels(cluster.Name),
Annotations: map[string]string{},
},
Spec: v1.PodSpec{
Containers: []v1.Container{c.mdsContainer(cluster, id)},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: []v1.Volume{
{Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}},
},
},
}
deployment.Spec = extensions.DeploymentSpec{Template: podSpec, Replicas: &c.Replicas}
return deployment, nil
}
func (c *Cluster) mdsContainer(cluster *mon.ClusterInfo, id string) v1.Container {
command := fmt.Sprintf("/usr/bin/rookd mds --data-dir=%s --mon-endpoints=%s --cluster-name=%s --mds-id=%s ",
k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name, id)
return v1.Container{
// TODO: fix "sleep 5".
// Without waiting some time, there is highly probable flakes in network setup.
Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)},
Name: appName,
Image: k8sutil.MakeRookImage(c.Version),
VolumeMounts: []v1.VolumeMount{
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
},
Env: []v1.EnvVar{
{Name: "ROOKD_MDS_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: appName}, Key: keyringName}}},
k8smon.MonSecretEnvVar(),
k8smon.AdminSecretEnvVar(),
},
}
}
func getLabels(clusterName string) map[string]string {
return map[string]string{
k8sutil.AppAttr: appName,
k8sutil.ClusterAttr: clusterName,
}
}
+8
View File
@@ -23,6 +23,7 @@ import (
"k8s.io/client-go/1.5/kubernetes"
"github.com/rook/rook/pkg/cephmgr/client"
"github.com/rook/rook/pkg/operator/mds"
"github.com/rook/rook/pkg/operator/mon"
"github.com/rook/rook/pkg/operator/osd"
"github.com/rook/rook/pkg/operator/rgw"
@@ -69,6 +70,13 @@ func (o *Operator) Run() error {
return fmt.Errorf("failed to start rgw. %+v", err)
}
// Start the file system
fs := mds.New(o.Namespace, o.containerVersion, o.factory)
err = fs.Start(o.clientset, cluster)
if err != nil {
return fmt.Errorf("failed to start mds. %+v", err)
}
logger.Infof("DONE!")
<-time.After(1000000 * time.Second)
+1 -1
View File
@@ -98,7 +98,7 @@ func (c *Cluster) createKeyring(clientset *kubernetes.Clientset, cluster *mon.Cl
// connect to the ceph cluster
logger.Infof("generating rgw keyring")
context := &clusterd.Context{ConfigDir: "/var/lib/rook"}
context := &clusterd.Context{ConfigDir: k8sutil.DataDir}
conn, err := mon.ConnectToClusterAsAdmin(context, c.factory, cluster)
if err != nil {
return fmt.Errorf("failed to connect to cluster. %+v", err)