diff --git a/cmd/rookd/mds.go b/cmd/rookd/mds.go index 77aa28592..2405bfc2f 100644 --- a/cmd/rookd/mds.go +++ b/cmd/rookd/mds.go @@ -18,10 +18,19 @@ limitations under the License. package main import ( + "github.com/rook/rook/pkg/cephmgr/cephd" + "github.com/rook/rook/pkg/cephmgr/mds" + "github.com/rook/rook/pkg/cephmgr/mon" + "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/util/flags" "github.com/spf13/cobra" ) +var ( + mdsID string + mdsKeyring string +) + var mdsCmd = &cobra.Command{ Use: "mds", Short: "Generates mds config and runs the mds daemon", @@ -29,16 +38,32 @@ var mdsCmd = &cobra.Command{ } func init() { + mdsCmd.Flags().StringVar(&mdsID, "mds-id", "", "the mds ID") + mdsCmd.Flags().StringVar(&mdsKeyring, "mds-keyring", "", "the mds keyring") + + flags.SetFlagsFromEnv(mdsCmd.Flags(), "ROOKD") + mdsCmd.RunE = startMDS } func startMDS(cmd *cobra.Command, args []string) error { - if err := flags.VerifyRequiredFlags(mdsCmd, []string{""}); err != nil { + if err := flags.VerifyRequiredFlags(mdsCmd, []string{"mon-endpoints", "cluster-name", "mon-secret", "admin-secret", "mds-id", "mds-keyring"}); err != nil { return err } setLogLevel() - // mds.Start(config) + clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints) + config := &mds.Config{ + ID: mdsID, + Keyring: mdsKeyring, + ClusterInfo: &clusterInfo, + CephLauncher: cephd.New(), + InProc: true, + } + + context := clusterd.NewDaemonContext(cfg.dataDir, cfg.cephConfigOverride, cfg.logLevel) + mds.Run(context, config) + return nil } diff --git a/pkg/cephmgr/mds/agent.go b/pkg/cephmgr/mds/agent.go index c2a2d3cff..64410f895 100644 --- a/pkg/cephmgr/mds/agent.go +++ b/pkg/cephmgr/mds/agent.go @@ -18,7 +18,6 @@ package mds import ( "fmt" "path" - "regexp" etcd "github.com/coreos/etcd/client" ctx "golang.org/x/net/context" @@ -89,7 +88,7 @@ func (a *mdsAgent) ConfigureLocalService(context *clusterd.Context) error { } defer conn.Shutdown() - err = createMDSKeyring(conn, mdsID, context.ConfigDir) + err = createKeyringAndSave(conn, mdsID, context.ConfigDir) if err != nil { return fmt.Errorf("failed to create mds keyring. %+v", err) } @@ -119,11 +118,16 @@ func (a *mdsAgent) DestroyLocalService(context *clusterd.Context) error { return nil } -// create a keyring for the mds client with a limited set of privileges -func createMDSKeyring(conn client.Connection, id, configDir string) error { +func getKeyringProperties(id string) (string, []string) { username := fmt.Sprintf("mds.%s", id) - keyringPath := getMDSKeyringPath(configDir, id) access := []string{"osd", "allow *", "mds", "allow", "mon", "allow profile mds"} + return username, access +} + +// create a keyring for the mds client with a limited set of privileges +func createKeyringAndSave(conn client.Connection, id, configDir string) error { + username, access := getKeyringProperties(id) + keyringPath := getMDSKeyringPath(configDir, id) keyringEval := func(key string) string { return fmt.Sprintf(keyringTemplate, id, key) } @@ -131,26 +135,22 @@ func createMDSKeyring(conn client.Connection, id, configDir string) error { return mon.CreateKeyring(conn, username, keyringPath, access, keyringEval) } +// create a keyring for the mds client with a limited set of privileges +func CreateKeyring(conn client.Connection, id string) (string, error) { + // get-or-create-key for the user account + username, access := getKeyringProperties(id) + keyring, err := client.AuthGetOrCreateKey(conn, username, access) + if err != nil { + return "", fmt.Errorf("failed to get or create auth key for %s. %+v", username, err) + } + + return keyring, nil +} + func (a *mdsAgent) startMDS(context *clusterd.Context, cluster *mon.ClusterInfo, id string) error { - // start the monitor daemon in the foreground with the given config - logger.Infof("starting mds %s", id) - - confFile := getMDSConfFilePath(context.ConfigDir, id, cluster.Name) - util.WriteFileToLog(logger, confFile) - - mdsNameArg := fmt.Sprintf("--name=mds.%s", id) - mdsProc, err := context.ProcMan.Start( - fmt.Sprintf("mds%s", id), - "mds", - regexp.QuoteMeta(mdsNameArg), - proc.ReuseExisting, - "--foreground", - mdsNameArg, - fmt.Sprintf("--cluster=%s", cluster.Name), - fmt.Sprintf("--conf=%s", confFile), - fmt.Sprintf("--keyring=%s", getMDSKeyringPath(context.ConfigDir, id)), - "-i", id) + config := &Config{ID: id, InProc: false, ClusterInfo: cluster} + mdsProc, err := startMDS(clusterd.ToDaemonContext(context), config) if err != nil { return fmt.Errorf("failed to start mds %s: %+v", id, err) } @@ -197,15 +197,3 @@ func getMDSIDKey(nodeID string, applied bool) string { func getMDSFileSystemKey(nodeID string, applied bool) string { return path.Join(getMDSDesiredNodesKey(applied), nodeID, "filesystem") } - -func getMDSConfDir(dir, id string) string { - return path.Join(dir, fmt.Sprintf("mds%s", id)) -} - -func getMDSConfFilePath(dir, id, clusterName string) string { - return path.Join(getMDSConfDir(dir, id), fmt.Sprintf("%s.config", clusterName)) -} - -func getMDSKeyringPath(dir, id string) string { - return path.Join(getMDSConfDir(dir, id), "keyring") -} diff --git a/pkg/cephmgr/mds/daemon.go b/pkg/cephmgr/mds/daemon.go new file mode 100644 index 000000000..9ad406ca5 --- /dev/null +++ b/pkg/cephmgr/mds/daemon.go @@ -0,0 +1,111 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package mds + +import ( + "fmt" + "path" + "regexp" + + "github.com/rook/rook/pkg/cephmgr/mon" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/util" + "github.com/rook/rook/pkg/util/proc" +) + +type Config struct { + ID string + Keyring string + InProc bool + ClusterInfo *mon.ClusterInfo + mon.CephLauncher +} + +func Run(context *clusterd.DaemonContext, config *Config) error { + + err := generateConfigFiles(context, config) + if err != nil { + return fmt.Errorf("failed to generate mds config files. %+v", err) + } + + _, err = startMDS(context, config) + if err != nil { + return fmt.Errorf("failed to run mds. %+v", err) + } + + return err +} + +func generateConfigFiles(context *clusterd.DaemonContext, config *Config) error { + + keyringPath := getMDSKeyringPath(context.ConfigDir, config.ID) + _, err := mon.GenerateConnectionConfigFile(clusterd.ToContext(context), config.ClusterInfo, getMDSConfDir(context.ConfigDir, config.ID), + fmt.Sprintf("mds.%s", config.ID), keyringPath) + if err != nil { + return fmt.Errorf("failed to create mds config file. %+v", err) + } + keyringEval := func(key string) string { + return fmt.Sprintf(keyringTemplate, config.ID, key) + } + + err = mon.WriteKeyring(keyringPath, config.Keyring, keyringEval) + if err != nil { + return fmt.Errorf("failed to create mds keyring. %+v", err) + } + + return nil +} + +func startMDS(context *clusterd.DaemonContext, config *Config) (mdsProc *proc.MonitoredProc, err error) { + + // start the mds daemon in the foreground with the given config + logger.Infof("starting mds %s", config.ID) + + confFile := getMDSConfFilePath(context.ConfigDir, config.ID, config.ClusterInfo.Name) + util.WriteFileToLog(logger, confFile) + + mdsNameArg := fmt.Sprintf("--name=mds.%s", config.ID) + args := []string{ + "--foreground", + mdsNameArg, + fmt.Sprintf("--cluster=%s", config.ClusterInfo.Name), + fmt.Sprintf("--conf=%s", confFile), + fmt.Sprintf("--keyring=%s", getMDSKeyringPath(context.ConfigDir, config.ID)), + "-i", config.ID, + } + + if config.InProc { + err = config.CephLauncher.Run("mds", args...) + } else { + mdsProc, err = context.ProcMan.Start("mds", "mds", regexp.QuoteMeta(mdsNameArg), proc.ReuseExisting, args...) + } + if err != nil { + err = fmt.Errorf("failed to start mds: %+v", err) + } + return +} + +func getMDSConfDir(dir, id string) string { + return path.Join(dir, fmt.Sprintf("mds%s", id)) +} + +func getMDSConfFilePath(dir, id, clusterName string) string { + return path.Join(getMDSConfDir(dir, id), fmt.Sprintf("%s.config", clusterName)) +} + +func getMDSKeyringPath(dir, id string) string { + return path.Join(getMDSConfDir(dir, id), "keyring") +} diff --git a/pkg/operator/k8sutil/k8sutil.go b/pkg/operator/k8sutil/k8sutil.go index d314d188f..a2b149a41 100644 --- a/pkg/operator/k8sutil/k8sutil.go +++ b/pkg/operator/k8sutil/k8sutil.go @@ -28,7 +28,7 @@ import ( const ( Namespace = "rook" DataDirVolume = "rook-data" - DataDir = "/var/lib/rook/data" + DataDir = "/var/lib/rook" ) func IsKubernetesResourceAlreadyExistError(err error) bool { diff --git a/pkg/operator/mds/log.go b/pkg/operator/mds/log.go new file mode 100644 index 000000000..2d596ef2a --- /dev/null +++ b/pkg/operator/mds/log.go @@ -0,0 +1,22 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package mds + +import ( + "github.com/coreos/pkg/capnslog" +) + +var logger = capnslog.NewPackageLogger("github.com/rook/rook-operator", "op-mds") diff --git a/pkg/operator/mds/mds.go b/pkg/operator/mds/mds.go new file mode 100644 index 000000000..eec783266 --- /dev/null +++ b/pkg/operator/mds/mds.go @@ -0,0 +1,207 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package mds + +import ( + "fmt" + + "github.com/rook/rook/pkg/cephmgr/client" + cephmds "github.com/rook/rook/pkg/cephmgr/mds" + "github.com/rook/rook/pkg/cephmgr/mon" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/k8sutil" + k8smon "github.com/rook/rook/pkg/operator/mon" + "k8s.io/client-go/1.5/kubernetes" + "k8s.io/client-go/1.5/pkg/api/v1" + extensions "k8s.io/client-go/1.5/pkg/apis/extensions/v1beta1" +) + +const ( + appName = "mds" + dataPoolSuffix = "-data" + metadataPoolSuffix = "-metadata" + keyringName = "keyring" +) + +type Cluster struct { + Namespace string + Version string + Replicas int32 + factory client.ConnectionFactory +} + +func New(namespace, version string, factory client.ConnectionFactory) *Cluster { + return &Cluster{ + Namespace: namespace, + Version: version, + Replicas: 1, + factory: factory, + } +} + +func (c *Cluster) Start(clientset *kubernetes.Clientset, cluster *mon.ClusterInfo) error { + logger.Infof("start running mds") + + if cluster == nil || len(cluster.Monitors) == 0 { + return fmt.Errorf("missing mons to start mds") + } + + context := &clusterd.DaemonContext{ConfigDir: k8sutil.DataDir} + conn, err := mon.ConnectToClusterAsAdmin(clusterd.ToContext(context), c.factory, cluster) + if err != nil { + return fmt.Errorf("failed to connect to cluster as admin: %+v", err) + } + defer conn.Shutdown() + + // creating a named file system will be done with third party resources in the future + err = c.createFilesystem("myfs", "myfspool", cluster, conn) + if err != nil { + return fmt.Errorf("failed to create file system. %+v", err) + } + + id := "mds1" + err = c.createKeyring(clientset, context, cluster, conn, id) + if err != nil { + return fmt.Errorf("failed to create mds keyring. %+v", err) + } + + // start the deployment + deployment, err := c.makeDeployment(cluster, id) + _, err = clientset.Deployments(c.Namespace).Create(deployment) + if err != nil { + if !k8sutil.IsKubernetesResourceAlreadyExistError(err) { + return fmt.Errorf("failed to create mds deployment. %+v", err) + } + logger.Infof("mds deployment already exists") + } else { + logger.Infof("mds deployment started") + } + + return nil +} + +func (c *Cluster) createFilesystem(id, poolName string, cluster *mon.ClusterInfo, conn client.Connection) error { + _, err := client.GetFilesystem(conn, id) + if err == nil { + logger.Infof("file system %s already exists", id) + return nil + } + + dataPool := poolName + dataPoolSuffix + metadataPool := poolName + metadataPoolSuffix + + // Create the metadata and data pools + pool := client.CephStoragePoolDetails{Name: dataPool} + _, err = client.CreatePool(conn, pool) + if err != nil { + return fmt.Errorf("failed to create data pool '%s': %+v", dataPool, err) + } + + pool = client.CephStoragePoolDetails{Name: metadataPool} + _, err = client.CreatePool(conn, pool) + if err != nil { + return fmt.Errorf("failed to create metadata pool '%s': %+v", metadataPool, err) + } + + // create the file system + if err := client.CreateFilesystem(conn, id, metadataPool, dataPool); err != nil { + return err + } + + logger.Infof("created file system %s on data pool %s and metadata pool %s", id, dataPool, metadataPool) + return nil +} + +func (c *Cluster) createKeyring(clientset *kubernetes.Clientset, context *clusterd.DaemonContext, cluster *mon.ClusterInfo, conn client.Connection, id string) error { + _, err := clientset.Secrets(c.Namespace).Get(appName) + if err == nil { + logger.Infof("the mds keyring was already generated") + return nil + } + if !k8sutil.IsKubernetesResourceNotFoundError(err) { + return fmt.Errorf("failed to get mds secrets. %+v", err) + } + + // get-or-create-key for the user account + keyring, err := cephmds.CreateKeyring(conn, id) + if err != nil { + return fmt.Errorf("failed to create mds keyring. %+v", err) + } + + // Store the keyring in a secret + secrets := map[string]string{ + keyringName: keyring, + } + _, err = clientset.Secrets(c.Namespace).Create(&v1.Secret{ObjectMeta: v1.ObjectMeta{Name: appName}, StringData: secrets}) + if err != nil { + return fmt.Errorf("failed to save mds secrets. %+v", err) + } + + return nil +} + +func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo, id string) (*extensions.Deployment, error) { + deployment := &extensions.Deployment{} + deployment.Name = appName + deployment.Namespace = c.Namespace + + podSpec := v1.PodTemplateSpec{ + ObjectMeta: v1.ObjectMeta{ + Name: appName, + Labels: getLabels(cluster.Name), + Annotations: map[string]string{}, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{c.mdsContainer(cluster, id)}, + RestartPolicy: v1.RestartPolicyAlways, + Volumes: []v1.Volume{ + {Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}, + }, + }, + } + + deployment.Spec = extensions.DeploymentSpec{Template: podSpec, Replicas: &c.Replicas} + + return deployment, nil +} + +func (c *Cluster) mdsContainer(cluster *mon.ClusterInfo, id string) v1.Container { + + command := fmt.Sprintf("/usr/bin/rookd mds --data-dir=%s --mon-endpoints=%s --cluster-name=%s --mds-id=%s ", + k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name, id) + return v1.Container{ + // TODO: fix "sleep 5". + // Without waiting some time, there is highly probable flakes in network setup. + Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)}, + Name: appName, + Image: k8sutil.MakeRookImage(c.Version), + VolumeMounts: []v1.VolumeMount{ + {Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir}, + }, + Env: []v1.EnvVar{ + {Name: "ROOKD_MDS_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: appName}, Key: keyringName}}}, + k8smon.MonSecretEnvVar(), + k8smon.AdminSecretEnvVar(), + }, + } +} + +func getLabels(clusterName string) map[string]string { + return map[string]string{ + k8sutil.AppAttr: appName, + k8sutil.ClusterAttr: clusterName, + } +} diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 6a84bd84c..b13f255d1 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -23,6 +23,7 @@ import ( "k8s.io/client-go/1.5/kubernetes" "github.com/rook/rook/pkg/cephmgr/client" + "github.com/rook/rook/pkg/operator/mds" "github.com/rook/rook/pkg/operator/mon" "github.com/rook/rook/pkg/operator/osd" "github.com/rook/rook/pkg/operator/rgw" @@ -69,6 +70,13 @@ func (o *Operator) Run() error { return fmt.Errorf("failed to start rgw. %+v", err) } + // Start the file system + fs := mds.New(o.Namespace, o.containerVersion, o.factory) + err = fs.Start(o.clientset, cluster) + if err != nil { + return fmt.Errorf("failed to start mds. %+v", err) + } + logger.Infof("DONE!") <-time.After(1000000 * time.Second) diff --git a/pkg/operator/rgw/rgw.go b/pkg/operator/rgw/rgw.go index eed778207..d2cea6945 100644 --- a/pkg/operator/rgw/rgw.go +++ b/pkg/operator/rgw/rgw.go @@ -98,7 +98,7 @@ func (c *Cluster) createKeyring(clientset *kubernetes.Clientset, cluster *mon.Cl // connect to the ceph cluster logger.Infof("generating rgw keyring") - context := &clusterd.Context{ConfigDir: "/var/lib/rook"} + context := &clusterd.Context{ConfigDir: k8sutil.DataDir} conn, err := mon.ConnectToClusterAsAdmin(context, c.factory, cluster) if err != nil { return fmt.Errorf("failed to connect to cluster. %+v", err)