rgw: add support for ssl cert

This commit is contained in:
Travis Nielsen committed 2017-09-06 16:55:23 -07:00
1 parent 15809f4842
commit ac073dd448
13 files changed
+153 -51

No files matched your search

+9 -6
View File
@@ -35,6 +35,7 @@ var (
rgwName string
rgwKeyring string
rgwHost string
rgwCert string
rgwPort int
)
@@ -42,6 +43,7 @@ func init() {
rgwCmd.Flags().StringVar(&rgwName, "rgw-name", "", "name of the object store")
rgwCmd.Flags().StringVar(&rgwKeyring, "rgw-keyring", "", "the rgw keyring")
rgwCmd.Flags().StringVar(&rgwHost, "rgw-host", "", "dns host name")
rgwCmd.Flags().StringVar(&rgwCert, "rgw-cert", "", "path to the ssl certificate in pem format")
rgwCmd.Flags().IntVar(&rgwPort, "rgw-port", 0, "rgw port number")
addCephFlags(rgwCmd)
@@ -66,12 +68,13 @@ func startRGW(cmd *cobra.Command, args []string) error {
clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
config := &rgw.Config{
ClusterInfo: &clusterInfo,
Name: rgwName,
Keyring: rgwKeyring,
Host: rgwHost,
Port: rgwPort,
InProc: true,
ClusterInfo: &clusterInfo,
Name: rgwName,
Keyring: rgwKeyring,
Host: rgwHost,
Port: rgwPort,
CertificatePath: rgwCert,
InProc: true,
}
err := rgw.Run(createContext(), config)
+19 -8
View File
@@ -17,6 +17,7 @@ package object
import (
"fmt"
"io/ioutil"
"os"
"time"
@@ -33,10 +34,11 @@ const (
)
var (
store model.ObjectStore
dataConfig pool.Config
metadataConfig pool.Config
createCmd = &cobra.Command{
store model.ObjectStore
dataConfig pool.Config
metadataConfig pool.Config
certificateFile string
createCmd = &cobra.Command{
Use: "create",
Short: "Creates a new object storage instance in the cluster",
}
@@ -46,7 +48,7 @@ func init() {
createCmd.Flags().StringVarP(&store.Name, "name", "n", "default", "The name of the object store instance")
createCmd.Flags().Int32VarP(&store.Port, "port", "p", model.RGWPort, "The port on which to expose the object store")
createCmd.Flags().Int32Var(&store.RGWReplicas, "rgw-replicas", 1, "The number of RGW services for load balancing")
createCmd.Flags().StringVar(&store.CertificateRef, "certificate-ref", "", "The name of the secret containing the cert")
createCmd.Flags().StringVar(&certificateFile, "certificate", "", "Path to the ssl cert file (pem format)")
pool.AddPoolFlags(createCmd, "data-", &dataConfig)
pool.AddPoolFlags(createCmd, "metadata-", &metadataConfig)
@@ -55,19 +57,28 @@ func init() {
func createObjectStoreEntry(cmd *cobra.Command, args []string) error {
rook.SetupLogging()
if err := flags.VerifyRequiredFlags(cmd, []string{"name", "type"}); err != nil {
if err := flags.VerifyRequiredFlags(cmd, []string{"name", "data-type", "metadata-type"}); err != nil {
return err
}
if certificateFile != "" {
cert, err := ioutil.ReadFile(certificateFile)
if err != nil {
fmt.Fprintln(os.Stderr, "failed to read certificate", err)
os.Exit(1)
}
store.Certificate = string(cert)
}
dataPool, err := pool.ConfigToModel(dataConfig)
if err != nil {
fmt.Fprintln(os.Stderr, err)
fmt.Fprintln(os.Stderr, "failed to read data settings", err)
os.Exit(1)
}
store.DataConfig = *dataPool
metadataPool, err := pool.ConfigToModel(metadataConfig)
if err != nil {
fmt.Fprintln(os.Stderr, err)
fmt.Fprintln(os.Stderr, "failed to read metadata settings", err)
os.Exit(1)
}
store.MetadataConfig = *metadataPool
+4 -5
View File
@@ -21,25 +21,24 @@ import (
"github.com/stretchr/testify/assert"
"github.com/rook/rook/pkg/model"
"github.com/rook/rook/pkg/rook/test"
)
func TestCreateObjectStore(t *testing.T) {
c := &test.MockRookRestClient{}
out, err := createObjectStore(c)
err := createObjectStore(c)
assert.Nil(t, err)
assert.Equal(t, "succeeded starting creation of object store", out)
}
func TestCreateObjectStoreError(t *testing.T) {
c := &test.MockRookRestClient{
MockCreateObjectStore: func() (string, error) {
MockCreateObjectStore: func(store model.ObjectStore) (string, error) {
return "", fmt.Errorf("mock create object store failed")
},
}
out, err := createObjectStore(c)
err := createObjectStore(c)
assert.NotNil(t, err)
assert.Equal(t, "", out)
}
+1 -1
View File
@@ -61,7 +61,7 @@ func AddPoolFlags(cmd *cobra.Command, prefix string, config *Config) {
shorthand = "t"
}
cmd.Flags().StringVarP(&config.PoolType, prefix+"type", shorthand, PoolTypeReplicated,
fmt.Sprintf("Type of storage pool, '%s' or '%s' (required)", PoolTypeReplicated, PoolTypeErasureCoded))
fmt.Sprintf("Type of storage pool, '%s' or '%s'", PoolTypeReplicated, PoolTypeErasureCoded))
shorthand = ""
if prefix == "" {
+18 -13
View File
@@ -29,27 +29,24 @@ const (
)
func TestCreatePoolValidTypeRequired(t *testing.T) {
c := &test.MockRookRestClient{}
out, err := createPool("pool1", "foo", 1, 0, 0, c)
config := Config{PoolType: "foo"}
_, err := ConfigToModel(config)
assert.NotNil(t, err)
assert.Equal(t, "invalid pool type 'foo', allowed pool types are 'replicated' and 'erasure-coded'", err.Error())
assert.Equal(t, "", out)
}
func TestCreatePoolErasureCodedParamsRequired(t *testing.T) {
c := &test.MockRookRestClient{}
out, err := createPool("pool1", PoolTypeErasureCoded, 0, 0, 0, c)
config := Config{PoolType: PoolTypeErasureCoded}
_, err := ConfigToModel(config)
assert.NotNil(t, err)
assert.Equal(t, "both data chunks and coding chunks must be greater than zero for pool type 'erasure-coded'", err.Error())
assert.Equal(t, "", out)
}
func TestCreatePoolReplicatedErasureCodedParamsNotAllowed(t *testing.T) {
c := &test.MockRookRestClient{}
out, err := createPool("pool1", PoolTypeReplicated, 0, 2, 1, c)
config := Config{PoolType: PoolTypeReplicated, DataChunks: 2, CodingChunks: 1}
_, err := ConfigToModel(config)
assert.NotNil(t, err)
assert.Equal(t, "both data chunks and coding chunks must be zero for pool type 'replicated'", err.Error())
assert.Equal(t, "", out)
}
func TestCreatePoolReplicatedNoParams(t *testing.T) {
@@ -66,7 +63,8 @@ func TestCreatePoolReplicatedNoParams(t *testing.T) {
}
// replicated pool replica count of 0 is OK, it will get the ceph default
out, err := createPool("pool1", PoolTypeReplicated, 0, 0, 0, c)
pool := model.Pool{Name: "pool1", Type: model.Replicated}
out, err := createPool(pool, c)
assert.Nil(t, err)
assert.Equal(t, SuccessPoolCreatedMessage, out)
}
@@ -87,7 +85,9 @@ func TestCreatePoolReplicated(t *testing.T) {
},
}
out, err := createPool("pool1", PoolTypeReplicated, 3, 0, 0, c)
pool := model.Pool{Name: "pool1", Type: model.Replicated}
pool.ReplicationConfig.Size = 3
out, err := createPool(pool, c)
assert.Nil(t, err)
assert.Equal(t, SuccessPoolCreatedMessage, out)
}
@@ -109,7 +109,11 @@ func TestCreatePoolErasureCoded(t *testing.T) {
},
}
out, err := createPool("pool1", PoolTypeErasureCoded, 0, 2, 1, c)
pool := model.Pool{Name: "pool1", Type: model.ErasureCoded}
pool.ErasureCodedConfig.DataChunkCount = 2
pool.ErasureCodedConfig.CodingChunkCount = 1
out, err := createPool(pool, c)
assert.Nil(t, err)
assert.Equal(t, SuccessPoolCreatedMessage, out)
}
@@ -121,7 +125,8 @@ func TestCreatePoolFailure(t *testing.T) {
},
}
out, err := createPool("pool1", PoolTypeReplicated, 0, 0, 0, c)
pool := model.Pool{Name: "pool1", Type: model.Replicated}
out, err := createPool(pool, c)
assert.NotNil(t, err)
assert.Equal(t, "failed to create new pool 'pool1': mock error", err.Error())
assert.Equal(t, "", out)
+21
View File
@@ -16,6 +16,7 @@ limitations under the License.
package k8s
import (
"encoding/base64"
"fmt"
"github.com/rook/rook/pkg/ceph/mon"
@@ -25,6 +26,8 @@ import (
"github.com/rook/rook/pkg/operator/k8sutil"
k8smds "github.com/rook/rook/pkg/operator/mds"
k8srgw "github.com/rook/rook/pkg/operator/rgw"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -45,6 +48,24 @@ func (s *clusterHandler) GetClusterInfo() (*mon.ClusterInfo, error) {
func (s *clusterHandler) EnableObjectStore(config model.ObjectStore) error {
logger.Infof("Starting the Object store")
// save the certificate in a secret if we weren't given a reference to a secret
if config.Certificate != "" && config.CertificateRef == "" {
certName := fmt.Sprintf("rook-rgw-%s-cert", config.Name)
config.CertificateRef = certName
encodedCert := base64.StdEncoding.EncodeToString([]byte(config.Certificate))
data := map[string][]byte{"cert": []byte(encodedCert)}
certSecret := &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: certName, Namespace: s.namespace}, Data: data}
_, err := s.context.Clientset.Core().Secrets(s.namespace).Create(certSecret)
if err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create cert secret. %+v", err)
}
}
}
// Passing an empty Placement{} as the api doesn't know about placement
// information. This should be resolved with the transition to CRD (TPR).
r := k8srgw.New(s.context, config, s.namespace, s.versionTag, k8sutil.Placement{})
+2 -1
View File
@@ -25,6 +25,7 @@ import (
"os"
"path"
"path/filepath"
"strings"
"testing"
cephtest "github.com/rook/rook/pkg/ceph/test"
@@ -41,7 +42,7 @@ func TestCreateObjectStoreHandler(t *testing.T) {
etcdClient := util.NewMockEtcdClient()
context := &clusterd.Context{DirectContext: clusterd.DirectContext{EtcdClient: etcdClient}}
req, err := http.NewRequest("POST", "http://10.0.0.100/objectstore", nil)
req, err := http.NewRequest("POST", "http://10.0.0.100/objectstore", strings.NewReader(`{"name": "default"}`))
if err != nil {
logger.Fatal(err)
}
+15 -8
View File
@@ -31,12 +31,13 @@ import (
)
type Config struct {
Name string
Host string
Port int
Keyring string
InProc bool
ClusterInfo *mon.ClusterInfo
Name string
Host string
Port int
Keyring string
CertificatePath string
InProc bool
ClusterInfo *mon.ClusterInfo
}
func Run(context *clusterd.Context, config *Config) error {
@@ -66,6 +67,13 @@ func generateConfigFiles(context *clusterd.Context, config *Config) error {
logger.Warningf("failed to create data directory %s: %+v", dataDir, err)
}
sslSuffix := ""
if config.CertificatePath != "" {
// the suffix is intended to be appended to the end of the rgw_frontends arg, immediately after the port.
// with ssl enabled, the port number must end with the letter s.
sslSuffix = fmt.Sprintf("s ssl_certificate=%s", config.CertificatePath)
}
settings := map[string]string{
"host": config.Host,
"rgw port": strconv.Itoa(config.Port),
@@ -74,6 +82,7 @@ func generateConfigFiles(context *clusterd.Context, config *Config) error {
"rgw log nonexistent bucket": "true",
"rgw intent log object name utc": "true",
"rgw enable usage log": "true",
"rgw_frontends": fmt.Sprintf("civetweb port=%d%s", config.Port, sslSuffix),
"rgw_zone": config.Name,
"rgw_zonegroup": config.Name,
}
@@ -104,7 +113,6 @@ func generateConfigFiles(context *clusterd.Context, config *Config) error {
}
func startRGW(context *clusterd.Context, config *Config) (rgwProc *proc.MonitoredProc, err error) {
// start the monitor daemon in the foreground with the given config
logger.Infof("starting rgw")
@@ -118,7 +126,6 @@ func startRGW(context *clusterd.Context, config *Config) (rgwProc *proc.Monitore
fmt.Sprintf("--cluster=%s", config.ClusterInfo.Name),
fmt.Sprintf("--conf=%s", confFile),
fmt.Sprintf("--keyring=%s", getRGWKeyringPath(context.ConfigDir)),
fmt.Sprintf("--rgw-frontends=civetweb port=%d", config.Port),
fmt.Sprintf("--rgw-mime-types-file=%s", getMimeTypesPath(context.ConfigDir)),
}
if config.InProc {
+1
View File
@@ -27,6 +27,7 @@ type ObjectStore struct {
Name string `json:"name"`
Port int32 `json:"port"`
RGWReplicas int32 `json:"rgwReplicas"`
Certificate string `json:"certificate"`
CertificateRef string `json:"certificateRef"`
DataConfig Pool `json:"dataConfig"`
MetadataConfig Pool `json:"metadataConfig"`
+1 -1
View File
@@ -48,7 +48,7 @@ const (
ConfigOverrideVal = "config"
repoPrefixEnvVar = "ROOK_REPO_PREFIX"
defaultVersion = "latest"
configMountDir = "/etc/rook"
configMountDir = "/etc/rook/config"
overrideFilename = "override.conf"
)
+39 -6
View File
@@ -20,6 +20,8 @@ package rgw
import (
"encoding/json"
"fmt"
"path"
"strings"
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/pkg/ceph/client"
@@ -39,8 +41,12 @@ import (
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-rgw")
const (
appName = "rook-ceph-rgw"
keyringName = "keyring"
appName = "rook-ceph-rgw"
keyringName = "keyring"
certVolumeName = "rook-rgw-cert"
certMountPath = "/etc/rook/private"
certKeyName = "cert"
certFilename = "rgw-cert.pem"
)
// Cluster for rgw management
@@ -142,7 +148,7 @@ func (c *Cluster) createRealm(serviceIP string) error {
zoneGroupID, err := decodeID(output)
if err != nil {
return fmt.Errorf("failed to parse realm id. %+v", err)
return fmt.Errorf("failed to parse zone group id. %+v", err)
}
// create the zone if it doesn't exist yet
@@ -160,6 +166,7 @@ func (c *Cluster) createRealm(serviceIP string) error {
}
if updatePeriod {
// the period will help notify other zones of changes if there are multi-zones
_, err = c.runRGWCommand("period", "update", "--commit")
if err != nil {
return fmt.Errorf("failed to update period. %+v", err)
@@ -201,7 +208,11 @@ func (c *Cluster) createSimilarPools(pools []string, poolConfig model.Pool) erro
for _, pool := range pools {
// create the pool if it doesn't exist yet
name := fmt.Sprintf("%s.%s", c.Config.Name, pool)
name := pool
if !strings.HasPrefix(pool, ".") {
// the name of the pool is <instance>.<name>, except for the pool ".rgw.root" that spans object stores
name = fmt.Sprintf("%s.%s", c.Config.Name, pool)
}
if _, err := ceph.GetPoolDetails(c.context, c.Namespace, name); err != nil {
cephConfig.Name = name
err := ceph.CreatePool(c.context, c.Namespace, cephConfig)
@@ -289,11 +300,21 @@ func (c *Cluster) makeDeployment() *extensions.Deployment {
k8sutil.ConfigOverrideVolume(),
},
}
// Set the ssl cert if specified
if c.Config.CertificateRef != "" {
certVol := v1.Volume{Name: certVolumeName, VolumeSource: v1.VolumeSource{Secret: &v1.SecretVolumeSource{
SecretName: c.Config.CertificateRef,
Items: []v1.KeyToPath{{Key: certKeyName, Path: certFilename}},
}}}
podSpec.Volumes = append(podSpec.Volumes, certVol)
}
c.placement.ApplyToPodSpec(&podSpec)
podTemplateSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: "rook-ceph-rgw",
Name: c.instanceName(),
Labels: c.getLabels(),
Annotations: map[string]string{},
},
@@ -307,7 +328,7 @@ func (c *Cluster) makeDeployment() *extensions.Deployment {
func (c *Cluster) rgwContainer() v1.Container {
return v1.Container{
container := v1.Container{
Args: []string{
"rgw",
fmt.Sprintf("--config-dir=%s", k8sutil.DataDir),
@@ -332,6 +353,18 @@ func (c *Cluster) rgwContainer() v1.Container {
k8sutil.ConfigOverrideEnvVar(),
},
}
if c.Config.CertificateRef != "" {
// Add a volume mount for the ssl certificate
mount := v1.VolumeMount{Name: certVolumeName, MountPath: certMountPath, ReadOnly: true}
container.VolumeMounts = append(container.VolumeMounts, mount)
// Pass the flag for using the ssl cert
path := path.Join(certMountPath, certFilename)
container.Args = append(container.Args, fmt.Sprintf("--rgw-cert=%s", path))
}
return container
}
func (c *Cluster) startService() (string, error) {
+20
View File
@@ -101,9 +101,29 @@ func TestPodSpecs(t *testing.T) {
assert.Equal(t, "rook/rook:myversion", cont.Image)
assert.Equal(t, 2, len(cont.VolumeMounts))
assert.Equal(t, 5, len(cont.Args))
assert.Equal(t, "rgw", cont.Args[0])
assert.Equal(t, "--config-dir=/var/lib/rook", cont.Args[1])
assert.Equal(t, fmt.Sprintf("--rgw-name=%s", "default"), cont.Args[2])
assert.Equal(t, fmt.Sprintf("--rgw-port=%d", 123), cont.Args[3])
assert.Equal(t, fmt.Sprintf("--rgw-host=%s", c.instanceName()), cont.Args[4])
}
func TestSSLPodSpec(t *testing.T) {
config := model.ObjectStore{Name: "default", Port: 123, CertificateRef: "mycert"}
c := New(nil, config, "ns", "myversion", k8sutil.Placement{})
d := c.makeDeployment()
assert.NotNil(t, d)
assert.Equal(t, c.instanceName(), d.Name)
assert.Equal(t, 3, len(d.Spec.Template.Spec.Volumes))
assert.Equal(t, certVolumeName, d.Spec.Template.Spec.Volumes[2].Name)
cont := d.Spec.Template.Spec.Containers[0]
assert.Equal(t, 3, len(cont.VolumeMounts))
assert.Equal(t, certVolumeName, cont.VolumeMounts[2].Name)
assert.Equal(t, certMountPath, cont.VolumeMounts[2].MountPath)
assert.Equal(t, 6, len(cont.Args))
assert.Equal(t, fmt.Sprintf("--rgw-cert=%s/%s", certMountPath, certFilename), cont.Args[5])
}
+3 -2
View File
@@ -253,7 +253,8 @@ func TestCreateObjectStore(t *testing.T) {
mockHttpClient := NewMockHttpClient(mockServer.URL)
client := NewRookNetworkRestClient(mockServer.URL, mockHttpClient)
resp, err := client.CreateObjectStore()
store := model.ObjectStore{Name: "default", Port: 123}
resp, err := client.CreateObjectStore(store)
assert.NotNil(t, err)
assert.True(t, IsHttpAccepted(err))
assert.Equal(t, "", resp)
@@ -343,7 +344,7 @@ func TestDeleteFilesystemFailure(t *testing.T) {
func TestCreateObjectStoreFailure(t *testing.T) {
clientFunc := func(client RookRestClient) (interface{}, error) {
return client.CreateObjectStore()
return client.CreateObjectStore(model.ObjectStore{Name: "name"})
}
verifyFunc := getStringVerifyFunc(t)
ClientFailureHelperWithVerification(t, clientFunc, verifyFunc)