start the rgw service

This commit is contained in:
Travis Nielsen committed 2017-02-01 09:50:02 -08:00
1 parent ba35f0a788
commit a6b4430d8a
12 files changed
+275 -113

No files matched your search

+1 -1
View File
@@ -47,7 +47,7 @@ func runDaemon(cmd *cobra.Command, args []string) error {
if err := flags.VerifyRequiredFlags(daemonCmd, []string{"type"}); err != nil {
return err
}
if daemonType != "osd" && daemonType != "mds" && daemonType != "rgw-admin" {
if daemonType != "mon" && daemonType != "osd" && daemonType != "rgw" && daemonType != "mds" && daemonType != "rgw-admin" {
return fmt.Errorf("unknown daemon type: %s", daemonType)
}
+18 -30
View File
@@ -19,7 +19,6 @@ package main
import (
"fmt"
"strings"
"github.com/rook/rook/pkg/cephmgr/cephd"
"github.com/rook/rook/pkg/cephmgr/mon"
@@ -37,36 +36,42 @@ var rgwCmd = &cobra.Command{
}
var (
keyring string
host string
port int
rgwCluster mon.ClusterInfo
rgwKeyring string
rgwHost string
rgwPort int
)
func init() {
rgwCmd.Flags().StringVar(&keyring, "keyring", "", "keyring for connecting rgw to the cluster")
rgwCmd.Flags().StringVar(&host, "host", "", "dns host name")
rgwCmd.Flags().IntVar(&port, "port", 0, "rgw port number")
rgwCmd.Flags().StringVar(&rgwKeyring, "rgw-keyring", "", "the rgw keyring")
rgwCmd.Flags().StringVar(&osdCluster.MonitorSecret, "rgw-mon-secret", "", "the monitor secret")
rgwCmd.Flags().StringVar(&osdCluster.AdminSecret, "rgw-admin-secret", "", "the admin secret")
rgwCmd.Flags().StringVar(&rgwHost, "rgw-host", "", "dns host name")
rgwCmd.Flags().IntVar(&rgwPort, "rgw-port", 0, "rgw port number")
rgwCmd.RunE = startRGW
}
func startRGW(cmd *cobra.Command, args []string) error {
if err := flags.VerifyRequiredFlags(rgwCmd, []string{"mons", "keyring", "host"}); err != nil {
if err := flags.VerifyRequiredFlags(rgwCmd, []string{"mon-endpoints", "cluster-name", "rgw-host", "rgw-mon-secret", "rgw-admin-secret", "rgw-keyring"}); err != nil {
return err
}
if port == 0 {
if rgwPort == 0 {
return fmt.Errorf("port is required")
}
setLogLevel()
rgwCluster.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
rgwCluster.Name = cfg.clusterName
config := &rgw.Config{
ClusterInfo: &mon.ClusterInfo{Name: cfg.clusterName, Monitors: parseMonitors(cfg.monEndpoints)},
ClusterInfo: &rgwCluster,
CephLauncher: cephd.New(),
Keyring: keyring,
Host: host,
Port: port,
Keyring: rgwKeyring,
Host: rgwHost,
Port: rgwPort,
InProc: true,
}
executor := &exec.CommandExecutor{}
@@ -79,20 +84,3 @@ func startRGW(cmd *cobra.Command, args []string) error {
return rgw.Run(context, config)
}
func parseMonitors(mons string) map[string]*mon.CephMonitorConfig {
result := map[string]*mon.CephMonitorConfig{}
endpoints := strings.Split(mons, ",")
for _, endpoint := range endpoints {
parts := strings.Split(endpoint, "-")
if len(parts) != 2 {
logger.Warningf("invalid monitor format: %s", endpoint)
continue
}
name := parts[0]
endpoint := parts[1]
result[name] = &mon.CephMonitorConfig{Name: name, Endpoint: endpoint}
}
return result
}
+7 -14
View File
@@ -18,7 +18,6 @@ package rgw
import (
"fmt"
"path"
"regexp"
etcd "github.com/coreos/etcd/client"
ctx "golang.org/x/net/context"
@@ -110,20 +109,14 @@ func (a *rgwAgent) startRGW(context *clusterd.Context, cluster *mon.ClusterInfo)
}
keyring := val.Node.Value
// start the monitor daemon in the foreground with the given config
logger.Infof("starting rgw")
keyringArg := fmt.Sprintf("--keyring=%s", keyring)
config := &Config{Keyring: keyring, ClusterInfo: cluster, Host: DNSName, Port: RGWPort}
dcontext := toDaemonContext(context)
err = generateConfigFiles(dcontext, config)
if err != nil {
return fmt.Errorf("failed to generate rgw config files. %+v", err)
}
rgwProc, err := context.ProcMan.StartDirect(
"rgw",
regexp.QuoteMeta(keyringArg),
proc.ReuseExisting,
"rgw",
fmt.Sprintf("--cluster-name=%s", cluster.Name),
fmt.Sprintf("--mons=%s", cluster.MonEndpoints()),
fmt.Sprintf("--host=%s", DNSName),
fmt.Sprintf("--port=%d", RGWPort),
keyringArg)
rgwProc, err := startRGW(dcontext, config)
if err != nil {
return fmt.Errorf("failed to start rgw daemon: %+v", err)
}
+27 -21
View File
@@ -20,19 +20,21 @@ import (
"io/ioutil"
"os"
"path"
"regexp"
"strconv"
"github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/util"
"github.com/rook/rook/pkg/util/proc"
)
type Config struct {
DataDir string
Host string
Port int
Keyring string
InProc bool
ClusterInfo *mon.ClusterInfo
mon.CephLauncher
}
@@ -44,7 +46,7 @@ func Run(context *clusterd.DaemonContext, config *Config) error {
return fmt.Errorf("failed to generate rgw config files. %+v", err)
}
err = startRGW(context, config)
_, err = startRGW(context, config)
if err != nil {
return fmt.Errorf("failed to run rgw. %+v", err)
}
@@ -54,7 +56,7 @@ func Run(context *clusterd.DaemonContext, config *Config) error {
func generateConfigFiles(context *clusterd.DaemonContext, config *Config) error {
// create the rgw data directory
dataDir := path.Join(getRGWConfDir(config.DataDir), "data")
dataDir := path.Join(getRGWConfDir(context.ConfigDir), "data")
if err := os.MkdirAll(dataDir, 0744); err != nil {
logger.Warningf("failed to create data directory %s: %+v", dataDir, err)
}
@@ -68,8 +70,8 @@ func generateConfigFiles(context *clusterd.DaemonContext, config *Config) error
"rgw intent log object name utc": "true",
"rgw enable usage log": "true",
}
_, err := mon.GenerateConfigFile(toContext(context), config.ClusterInfo, getRGWConfDir(config.DataDir),
"client.radosgw.gateway", getRGWKeyringPath(config.DataDir), false, nil, settings)
_, err := mon.GenerateConfigFile(mon.ToContext(context), config.ClusterInfo, getRGWConfDir(context.ConfigDir),
"client.radosgw.gateway", getRGWKeyringPath(context.ConfigDir), false, nil, settings)
if err != nil {
return fmt.Errorf("failed to create config file. %+v", err)
}
@@ -79,13 +81,13 @@ func generateConfigFiles(context *clusterd.DaemonContext, config *Config) error
}
// create rgw config
err = mon.WriteKeyring(getRGWKeyringPath(config.DataDir), config.Keyring, keyringEval)
err = mon.WriteKeyring(getRGWKeyringPath(context.ConfigDir), config.Keyring, keyringEval)
if err != nil {
return fmt.Errorf("failed to save keyring. %+v", err)
}
// write the mime types config
mimeTypesPath := getMimeTypesPath(config.DataDir)
mimeTypesPath := getMimeTypesPath(context.ConfigDir)
logger.Debugf("Writing mime types to: %s", mimeTypesPath)
if err := ioutil.WriteFile(mimeTypesPath, []byte(mimeTypes), 0644); err != nil {
return fmt.Errorf("failed to write mime types to %s: %+v", mimeTypesPath, err)
@@ -94,33 +96,37 @@ func generateConfigFiles(context *clusterd.DaemonContext, config *Config) error
return nil
}
// TEMP: Convert a context to a daemon context. This should go away after all daemons convert
func toContext(context *clusterd.DaemonContext) *clusterd.Context {
return &clusterd.Context{Executor: context.Executor, ConfigDir: context.ConfigDir, LogLevel: context.LogLevel, ConfigFileOverride: context.ConfigFileOverride}
func toDaemonContext(context *clusterd.Context) *clusterd.DaemonContext {
return &clusterd.DaemonContext{ProcMan: context.ProcMan, Executor: context.Executor, ConfigDir: context.ConfigDir, LogLevel: context.LogLevel, ConfigFileOverride: context.ConfigFileOverride}
}
func startRGW(context *clusterd.DaemonContext, config *Config) error {
func startRGW(context *clusterd.DaemonContext, config *Config) (rgwProc *proc.MonitoredProc, err error) {
// start the monitor daemon in the foreground with the given config
logger.Infof("starting rgw")
confFile := getRGWConfFilePath(config.DataDir, config.ClusterInfo.Name)
confFile := getRGWConfFilePath(context.ConfigDir, config.ClusterInfo.Name)
util.WriteFileToLog(logger, confFile)
err := config.CephLauncher.Run(
"rgw",
rgwNameArg := "--name=client.radosgw.gateway"
args := []string{
"--foreground",
"--name=client.radosgw.gateway",
rgwNameArg,
fmt.Sprintf("--cluster=%s", config.ClusterInfo.Name),
fmt.Sprintf("--conf=%s", confFile),
fmt.Sprintf("--keyring=%s", getRGWKeyringPath(config.DataDir)),
fmt.Sprintf("--keyring=%s", getRGWKeyringPath(context.ConfigDir)),
fmt.Sprintf("--rgw-frontends=civetweb port=%d", config.Port),
fmt.Sprintf("--rgw-mime-types-file=%s", getMimeTypesPath(config.DataDir)))
if err != nil {
return fmt.Errorf("failed to start rgw: %+v", err)
fmt.Sprintf("--rgw-mime-types-file=%s", getMimeTypesPath(context.ConfigDir)),
}
return nil
if config.InProc {
err = config.CephLauncher.Run("rgw", args...)
} else {
rgwProc, err = context.ProcMan.Start("rgw", "rgw", regexp.QuoteMeta(rgwNameArg), proc.ReuseExisting, args...)
}
if err != nil {
err = fmt.Errorf("failed to start rgw: %+v", err)
}
return
}
func getRGWConfFilePath(configDir, clusterName string) string {
+1 -1
View File
@@ -251,7 +251,7 @@ func createKeyringOnInit(context *clusterd.Context, conn client.Connection) erro
}
// create the keyring
keyring, err := createKeyring(conn)
keyring, err := CreateKeyring(conn)
if err != nil {
return fmt.Errorf("failed to create keyring. %+v", err)
}
+1 -1
View File
@@ -22,7 +22,7 @@ import (
)
// create a keyring for the rgw client with a limited set of privileges
func createKeyring(conn client.Connection) (string, error) {
func CreateKeyring(conn client.Connection) (string, error) {
username := "client.radosgw.gateway"
access := []string{"osd", "allow rwx", "mon", "allow rw"}
+1 -24
View File
@@ -36,7 +36,6 @@ func getLabels(clusterName string) map[string]string {
func (c *Cluster) makeMonPod(config *MonConfig, clusterInfo *mon.ClusterInfo, antiAffinity bool) *v1.Pod {
container := c.monContainer(config, clusterInfo)
// TODO: container.LivenessProbe = config.livenessProbe()
pod := &v1.Pod{
ObjectMeta: v1.ObjectMeta{
@@ -87,21 +86,6 @@ func (c *Cluster) monContainer(config *MonConfig, clusterInfo *mon.ClusterInfo)
}
}
func (m *MonConfig) livenessProbe() *v1.Probe {
// simple query of the REST api locally to see if the pod is alive
return &v1.Probe{
Handler: v1.Handler{
Exec: &v1.ExecAction{
Command: []string{"/bin/sh", "-c", "curl localhost:8124"},
},
},
InitialDelaySeconds: 10,
TimeoutSeconds: 10,
PeriodSeconds: 60,
FailureThreshold: 3,
}
}
func (c *Cluster) pollPods(clientset *kubernetes.Clientset, clusterName string) ([]*v1.Pod, []*v1.Pod, error) {
podList, err := clientset.Core().Pods(c.Namespace).List(listOptions(clusterName))
if err != nil {
@@ -112,14 +96,7 @@ func (c *Cluster) pollPods(clientset *kubernetes.Clientset, clusterName string)
var pending []*v1.Pod
for i := range podList.Items {
pod := &podList.Items[i]
/*if len(pod.OwnerReferences) < 1 {
logger.Warningf("pollPods: ignore pod %v: no owner", pod.Name)
continue
}
if pod.OwnerReferences[0].UID != c.cluster.UID {
logger.Warningf("pollPods: ignore pod %v: owner (%v) is not %v", pod.Name, pod.OwnerReferences[0].UID, c.cluster.UID)
continue
}*/
switch pod.Status.Phase {
case v1.PodRunning:
running = append(running, pod)
+11 -15
View File
@@ -25,6 +25,7 @@ import (
"github.com/rook/rook/pkg/cephmgr/client"
"github.com/rook/rook/pkg/operator/mon"
"github.com/rook/rook/pkg/operator/osd"
"github.com/rook/rook/pkg/operator/rgw"
)
type Operator struct {
@@ -47,18 +48,6 @@ func New(namespace string, factory client.ConnectionFactory, clientset *kubernet
func (o *Operator) Run() error {
// Create the namespace
logger.Infof("Creating namespace %s", o.Namespace)
ns := &v1.Namespace{}
ns.Name = o.Namespace
_, err := o.clientset.Namespaces().Create(ns)
if err != nil {
if !k8sutil.IsKubernetesResourceAlreadyExistError(err) {
return fmt.Errorf("failed to create namespace %s. %+v", o.Namespace, err)
}
logger.Infof("namespace %s already exists", o.Namespace)
}
// Start the mon pods
m := mon.New(o.Namespace, o.factory, o.containerVersion)
cluster, err := m.Start(o.clientset)
@@ -67,14 +56,21 @@ func (o *Operator) Run() error {
}
// Start the OSDs
oset := osd.New(o.Namespace, o.containerVersion)
err = oset.Start(o.clientset, cluster)
osds := osd.New(o.Namespace, o.containerVersion)
err = osds.Start(o.clientset, cluster)
if err != nil {
return fmt.Errorf("failed to start the osds. %+v", err)
}
// Start the object store
r := rgw.New(o.Namespace, o.containerVersion, o.factory)
err = r.Start(o.clientset, cluster)
if err != nil {
return fmt.Errorf("failed to start rgw. %+v", err)
}
logger.Infof("DONE!")
<-time.After(9999 * time.Second)
<-time.After(1000000 * time.Second)
return nil
}
+1 -1
View File
@@ -18,5 +18,5 @@ package operator
import "testing"
func TestOperator(t *testing.T) {
New("foo", nil, nil)
New("foo", nil, nil, "version")
}
+22
View File
@@ -0,0 +1,22 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package rgw
import (
"github.com/coreos/pkg/capnslog"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook-operator", "rgw")
+183
View File
@@ -0,0 +1,183 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package rgw
import (
"fmt"
"github.com/rook/rook/pkg/cephmgr/client"
"github.com/rook/rook/pkg/cephmgr/mon"
cephrgw "github.com/rook/rook/pkg/cephmgr/rgw"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/client-go/1.5/kubernetes"
"k8s.io/client-go/1.5/pkg/api/v1"
api "k8s.io/client-go/1.5/pkg/api/v1"
extensions "k8s.io/client-go/1.5/pkg/apis/extensions/v1beta1"
"k8s.io/client-go/1.5/pkg/util/intstr"
)
const (
rgwApp = "cephrgw"
deploymentName = "rgw"
)
type Cluster struct {
Namespace string
Version string
Replicas int32
factory client.ConnectionFactory
}
func New(namespace, version string, factory client.ConnectionFactory) *Cluster {
return &Cluster{
Namespace: namespace,
Version: version,
Replicas: 2,
factory: factory,
}
}
func (c *Cluster) Start(clientset *kubernetes.Clientset, cluster *mon.ClusterInfo) error {
logger.Infof("start running rgw")
if cluster == nil || len(cluster.Monitors) == 0 {
return fmt.Errorf("missing mons to start rgw")
}
keyring, err := c.createKeyring(cluster)
if err != nil {
return fmt.Errorf("failed to create rgw keyring. %+v", err)
}
// start the service
err = c.startService(clientset, cluster)
if err != nil {
return fmt.Errorf("failed to start rgw service. %+v", err)
}
// start the deployment
deployment, err := c.makeDeployment(cluster, keyring)
_, err = clientset.Deployments(c.Namespace).Create(deployment)
if err != nil {
if !k8sutil.IsKubernetesResourceAlreadyExistError(err) {
return fmt.Errorf("failed to create rgw deployment. %+v", err)
}
logger.Infof("rgw deployment already exists")
} else {
logger.Infof("rgw deployment started")
}
return nil
}
func (c *Cluster) createKeyring(cluster *mon.ClusterInfo) (string, error) {
context := &clusterd.Context{ConfigDir: "/var/lib/rook"}
// connect to the ceph cluster
conn, err := mon.ConnectToClusterAsAdmin(context, c.factory, cluster)
if err != nil {
return "", fmt.Errorf("failed to connect to cluster. %+v", err)
}
defer conn.Shutdown()
// create the keyring
keyring, err := cephrgw.CreateKeyring(conn)
if err != nil {
return "", fmt.Errorf("failed to create keyring. %+v", err)
}
return keyring, nil
}
func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo, keyring string) (*extensions.Deployment, error) {
deployment := &extensions.Deployment{}
deployment.Name = deploymentName
deployment.Namespace = c.Namespace
podSpec := api.PodTemplateSpec{
ObjectMeta: api.ObjectMeta{
Name: "rook-rgw",
Labels: getLabels(cluster.Name),
Annotations: map[string]string{},
},
Spec: api.PodSpec{
Containers: []api.Container{c.rgwContainer(cluster, keyring)},
RestartPolicy: api.RestartPolicyAlways,
Volumes: []api.Volume{
{Name: k8sutil.DataDirVolume, VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}},
},
},
}
deployment.Spec = extensions.DeploymentSpec{Template: podSpec, Replicas: &c.Replicas}
return deployment, nil
}
func (c *Cluster) rgwContainer(cluster *mon.ClusterInfo, keyring string) api.Container {
command := fmt.Sprintf("/usr/bin/rookd rgw --data-dir=%s --mon-endpoints=%s --cluster-name=%s --rgw-mon-secret=%s --rgw-admin-secret=%s --rgw-port=%d --rgw-host=%s --rgw-keyring=%s",
k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name, cluster.MonitorSecret, cluster.AdminSecret, cephrgw.RGWPort, cephrgw.DNSName, keyring)
return api.Container{
// TODO: fix "sleep 5".
// Without waiting some time, there is highly probable flakes in network setup.
Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)},
Name: rgwApp,
Image: k8sutil.MakeRookImage(c.Version),
VolumeMounts: []api.VolumeMount{
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
},
}
}
func (c *Cluster) startService(clientset *kubernetes.Clientset, clusterInfo *mon.ClusterInfo) error {
labels := getLabels(clusterInfo.Name)
s := &v1.Service{
ObjectMeta: v1.ObjectMeta{
Name: "ceph-rgw",
Labels: labels,
},
Spec: v1.ServiceSpec{
Ports: []v1.ServicePort{
{
Name: "ceph-rgw",
Port: cephrgw.RGWPort,
TargetPort: intstr.FromInt(int(cephrgw.RGWPort)),
Protocol: v1.ProtocolTCP,
},
},
Selector: labels,
},
}
s, err := clientset.Services(k8sutil.Namespace).Create(s)
if err != nil {
if !k8sutil.IsKubernetesResourceAlreadyExistError(err) {
return fmt.Errorf("failed to create mon service. %+v", err)
}
}
logger.Infof("RGW service running at %s:%d", s.Spec.ClusterIP, cephrgw.RGWPort)
return nil
}
func getLabels(clusterName string) map[string]string {
return map[string]string{
k8sutil.AppAttr: rgwApp,
k8sutil.ClusterAttr: clusterName,
}
}
+2 -5
View File
@@ -72,15 +72,11 @@ func (p *ProcManager) Run(logName, tool string, args ...string) error {
return nil
}
func (p *ProcManager) Start(logName, daemon, procSearchPattern string, policy ProcStartPolicy, args ...string) (*MonitoredProc, error) {
return p.StartDirect(logName, procSearchPattern, policy, createDaemonArgs(daemon, args...)...)
}
// Start the given daemon and provided arguments. Handling of any matching existing process will be in accordance
// with the given ProcStartPolicy. The search pattern will be used to search through the cmdline args of existing
// processes to find any matching existing process. Therefore, it should be a regex pattern that can uniquely
// identify the process (e.g., --id=1)
func (p *ProcManager) StartDirect(name, procSearchPattern string, policy ProcStartPolicy, args ...string) (*MonitoredProc, error) {
func (p *ProcManager) Start(name, daemon, procSearchPattern string, policy ProcStartPolicy, args ...string) (*MonitoredProc, error) {
// look for an existing process first
shouldStart, err := p.checkProcessExists(os.Args[0], procSearchPattern, policy)
if err != nil {
@@ -92,6 +88,7 @@ func (p *ProcManager) StartDirect(name, procSearchPattern string, policy ProcSta
return nil, nil
}
args = createDaemonArgs(daemon, args...)
logger.Infof("Starting process %s with args: %v", name, args)
cmd, err := p.executor.StartExecuteCommand(name, os.Args[0], args...)
if err != nil {