diff --git a/cmd/rook/rgw.go b/cmd/rook/rgw.go index 1b2a14a99..b9c5ad2f9 100644 --- a/cmd/rook/rgw.go +++ b/cmd/rook/rgw.go @@ -35,6 +35,7 @@ var ( rgwName string rgwKeyring string rgwHost string + rgwCert string rgwPort int ) @@ -42,6 +43,7 @@ func init() { rgwCmd.Flags().StringVar(&rgwName, "rgw-name", "", "name of the object store") rgwCmd.Flags().StringVar(&rgwKeyring, "rgw-keyring", "", "the rgw keyring") rgwCmd.Flags().StringVar(&rgwHost, "rgw-host", "", "dns host name") + rgwCmd.Flags().StringVar(&rgwCert, "rgw-cert", "", "path to the ssl certificate in pem format") rgwCmd.Flags().IntVar(&rgwPort, "rgw-port", 0, "rgw port number") addCephFlags(rgwCmd) @@ -66,12 +68,13 @@ func startRGW(cmd *cobra.Command, args []string) error { clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints) config := &rgw.Config{ - ClusterInfo: &clusterInfo, - Name: rgwName, - Keyring: rgwKeyring, - Host: rgwHost, - Port: rgwPort, - InProc: true, + ClusterInfo: &clusterInfo, + Name: rgwName, + Keyring: rgwKeyring, + Host: rgwHost, + Port: rgwPort, + CertificatePath: rgwCert, + InProc: true, } err := rgw.Run(createContext(), config) diff --git a/cmd/rookctl/object/create.go b/cmd/rookctl/object/create.go index 44596e2eb..5e5b4485f 100644 --- a/cmd/rookctl/object/create.go +++ b/cmd/rookctl/object/create.go @@ -17,6 +17,7 @@ package object import ( "fmt" + "io/ioutil" "os" "time" @@ -33,10 +34,11 @@ const ( ) var ( - store model.ObjectStore - dataConfig pool.Config - metadataConfig pool.Config - createCmd = &cobra.Command{ + store model.ObjectStore + dataConfig pool.Config + metadataConfig pool.Config + certificateFile string + createCmd = &cobra.Command{ Use: "create", Short: "Creates a new object storage instance in the cluster", } @@ -46,7 +48,7 @@ func init() { createCmd.Flags().StringVarP(&store.Name, "name", "n", "default", "The name of the object store instance") createCmd.Flags().Int32VarP(&store.Port, "port", "p", model.RGWPort, "The port on which to expose the object store") createCmd.Flags().Int32Var(&store.RGWReplicas, "rgw-replicas", 1, "The number of RGW services for load balancing") - createCmd.Flags().StringVar(&store.CertificateRef, "certificate-ref", "", "The name of the secret containing the cert") + createCmd.Flags().StringVar(&certificateFile, "certificate", "", "Path to the ssl cert file (pem format)") pool.AddPoolFlags(createCmd, "data-", &dataConfig) pool.AddPoolFlags(createCmd, "metadata-", &metadataConfig) @@ -55,19 +57,28 @@ func init() { func createObjectStoreEntry(cmd *cobra.Command, args []string) error { rook.SetupLogging() - if err := flags.VerifyRequiredFlags(cmd, []string{"name", "type"}); err != nil { + if err := flags.VerifyRequiredFlags(cmd, []string{"name", "data-type", "metadata-type"}); err != nil { return err } + if certificateFile != "" { + cert, err := ioutil.ReadFile(certificateFile) + if err != nil { + fmt.Fprintln(os.Stderr, "failed to read certificate", err) + os.Exit(1) + } + store.Certificate = string(cert) + } + dataPool, err := pool.ConfigToModel(dataConfig) if err != nil { - fmt.Fprintln(os.Stderr, err) + fmt.Fprintln(os.Stderr, "failed to read data settings", err) os.Exit(1) } store.DataConfig = *dataPool metadataPool, err := pool.ConfigToModel(metadataConfig) if err != nil { - fmt.Fprintln(os.Stderr, err) + fmt.Fprintln(os.Stderr, "failed to read metadata settings", err) os.Exit(1) } store.MetadataConfig = *metadataPool diff --git a/cmd/rookctl/object/create_test.go b/cmd/rookctl/object/create_test.go index 1bb69f731..c97659d55 100644 --- a/cmd/rookctl/object/create_test.go +++ b/cmd/rookctl/object/create_test.go @@ -21,25 +21,24 @@ import ( "github.com/stretchr/testify/assert" + "github.com/rook/rook/pkg/model" "github.com/rook/rook/pkg/rook/test" ) func TestCreateObjectStore(t *testing.T) { c := &test.MockRookRestClient{} - out, err := createObjectStore(c) + err := createObjectStore(c) assert.Nil(t, err) - assert.Equal(t, "succeeded starting creation of object store", out) } func TestCreateObjectStoreError(t *testing.T) { c := &test.MockRookRestClient{ - MockCreateObjectStore: func() (string, error) { + MockCreateObjectStore: func(store model.ObjectStore) (string, error) { return "", fmt.Errorf("mock create object store failed") }, } - out, err := createObjectStore(c) + err := createObjectStore(c) assert.NotNil(t, err) - assert.Equal(t, "", out) } diff --git a/cmd/rookctl/pool/create.go b/cmd/rookctl/pool/create.go index c856864c8..8a9894eb8 100644 --- a/cmd/rookctl/pool/create.go +++ b/cmd/rookctl/pool/create.go @@ -61,7 +61,7 @@ func AddPoolFlags(cmd *cobra.Command, prefix string, config *Config) { shorthand = "t" } cmd.Flags().StringVarP(&config.PoolType, prefix+"type", shorthand, PoolTypeReplicated, - fmt.Sprintf("Type of storage pool, '%s' or '%s' (required)", PoolTypeReplicated, PoolTypeErasureCoded)) + fmt.Sprintf("Type of storage pool, '%s' or '%s'", PoolTypeReplicated, PoolTypeErasureCoded)) shorthand = "" if prefix == "" { diff --git a/cmd/rookctl/pool/create_test.go b/cmd/rookctl/pool/create_test.go index 7ba5c2011..163bf83d3 100644 --- a/cmd/rookctl/pool/create_test.go +++ b/cmd/rookctl/pool/create_test.go @@ -29,27 +29,24 @@ const ( ) func TestCreatePoolValidTypeRequired(t *testing.T) { - c := &test.MockRookRestClient{} - out, err := createPool("pool1", "foo", 1, 0, 0, c) + config := Config{PoolType: "foo"} + _, err := ConfigToModel(config) assert.NotNil(t, err) assert.Equal(t, "invalid pool type 'foo', allowed pool types are 'replicated' and 'erasure-coded'", err.Error()) - assert.Equal(t, "", out) } func TestCreatePoolErasureCodedParamsRequired(t *testing.T) { - c := &test.MockRookRestClient{} - out, err := createPool("pool1", PoolTypeErasureCoded, 0, 0, 0, c) + config := Config{PoolType: PoolTypeErasureCoded} + _, err := ConfigToModel(config) assert.NotNil(t, err) assert.Equal(t, "both data chunks and coding chunks must be greater than zero for pool type 'erasure-coded'", err.Error()) - assert.Equal(t, "", out) } func TestCreatePoolReplicatedErasureCodedParamsNotAllowed(t *testing.T) { - c := &test.MockRookRestClient{} - out, err := createPool("pool1", PoolTypeReplicated, 0, 2, 1, c) + config := Config{PoolType: PoolTypeReplicated, DataChunks: 2, CodingChunks: 1} + _, err := ConfigToModel(config) assert.NotNil(t, err) assert.Equal(t, "both data chunks and coding chunks must be zero for pool type 'replicated'", err.Error()) - assert.Equal(t, "", out) } func TestCreatePoolReplicatedNoParams(t *testing.T) { @@ -66,7 +63,8 @@ func TestCreatePoolReplicatedNoParams(t *testing.T) { } // replicated pool replica count of 0 is OK, it will get the ceph default - out, err := createPool("pool1", PoolTypeReplicated, 0, 0, 0, c) + pool := model.Pool{Name: "pool1", Type: model.Replicated} + out, err := createPool(pool, c) assert.Nil(t, err) assert.Equal(t, SuccessPoolCreatedMessage, out) } @@ -87,7 +85,9 @@ func TestCreatePoolReplicated(t *testing.T) { }, } - out, err := createPool("pool1", PoolTypeReplicated, 3, 0, 0, c) + pool := model.Pool{Name: "pool1", Type: model.Replicated} + pool.ReplicationConfig.Size = 3 + out, err := createPool(pool, c) assert.Nil(t, err) assert.Equal(t, SuccessPoolCreatedMessage, out) } @@ -109,7 +109,11 @@ func TestCreatePoolErasureCoded(t *testing.T) { }, } - out, err := createPool("pool1", PoolTypeErasureCoded, 0, 2, 1, c) + pool := model.Pool{Name: "pool1", Type: model.ErasureCoded} + pool.ErasureCodedConfig.DataChunkCount = 2 + pool.ErasureCodedConfig.CodingChunkCount = 1 + + out, err := createPool(pool, c) assert.Nil(t, err) assert.Equal(t, SuccessPoolCreatedMessage, out) } @@ -121,7 +125,8 @@ func TestCreatePoolFailure(t *testing.T) { }, } - out, err := createPool("pool1", PoolTypeReplicated, 0, 0, 0, c) + pool := model.Pool{Name: "pool1", Type: model.Replicated} + out, err := createPool(pool, c) assert.NotNil(t, err) assert.Equal(t, "failed to create new pool 'pool1': mock error", err.Error()) assert.Equal(t, "", out) diff --git a/pkg/api/k8s/cluster_handler.go b/pkg/api/k8s/cluster_handler.go index 7543c54aa..685473963 100644 --- a/pkg/api/k8s/cluster_handler.go +++ b/pkg/api/k8s/cluster_handler.go @@ -16,6 +16,7 @@ limitations under the License. package k8s import ( + "encoding/base64" "fmt" "github.com/rook/rook/pkg/ceph/mon" @@ -25,6 +26,8 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" k8smds "github.com/rook/rook/pkg/operator/mds" k8srgw "github.com/rook/rook/pkg/operator/rgw" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -45,6 +48,24 @@ func (s *clusterHandler) GetClusterInfo() (*mon.ClusterInfo, error) { func (s *clusterHandler) EnableObjectStore(config model.ObjectStore) error { logger.Infof("Starting the Object store") + + // save the certificate in a secret if we weren't given a reference to a secret + if config.Certificate != "" && config.CertificateRef == "" { + certName := fmt.Sprintf("rook-rgw-%s-cert", config.Name) + config.CertificateRef = certName + + encodedCert := base64.StdEncoding.EncodeToString([]byte(config.Certificate)) + data := map[string][]byte{"cert": []byte(encodedCert)} + certSecret := &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: certName, Namespace: s.namespace}, Data: data} + + _, err := s.context.Clientset.Core().Secrets(s.namespace).Create(certSecret) + if err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create cert secret. %+v", err) + } + } + } + // Passing an empty Placement{} as the api doesn't know about placement // information. This should be resolved with the transition to CRD (TPR). r := k8srgw.New(s.context, config, s.namespace, s.versionTag, k8sutil.Placement{}) diff --git a/pkg/api/object_test.go b/pkg/api/object_test.go index 60a56afc0..2abb19d7b 100644 --- a/pkg/api/object_test.go +++ b/pkg/api/object_test.go @@ -25,6 +25,7 @@ import ( "os" "path" "path/filepath" + "strings" "testing" cephtest "github.com/rook/rook/pkg/ceph/test" @@ -41,7 +42,7 @@ func TestCreateObjectStoreHandler(t *testing.T) { etcdClient := util.NewMockEtcdClient() context := &clusterd.Context{DirectContext: clusterd.DirectContext{EtcdClient: etcdClient}} - req, err := http.NewRequest("POST", "http://10.0.0.100/objectstore", nil) + req, err := http.NewRequest("POST", "http://10.0.0.100/objectstore", strings.NewReader(`{"name": "default"}`)) if err != nil { logger.Fatal(err) } diff --git a/pkg/ceph/rgw/daemon.go b/pkg/ceph/rgw/daemon.go index 24d9d9e11..adad88ac9 100644 --- a/pkg/ceph/rgw/daemon.go +++ b/pkg/ceph/rgw/daemon.go @@ -31,12 +31,13 @@ import ( ) type Config struct { - Name string - Host string - Port int - Keyring string - InProc bool - ClusterInfo *mon.ClusterInfo + Name string + Host string + Port int + Keyring string + CertificatePath string + InProc bool + ClusterInfo *mon.ClusterInfo } func Run(context *clusterd.Context, config *Config) error { @@ -66,6 +67,13 @@ func generateConfigFiles(context *clusterd.Context, config *Config) error { logger.Warningf("failed to create data directory %s: %+v", dataDir, err) } + sslSuffix := "" + if config.CertificatePath != "" { + // the suffix is intended to be appended to the end of the rgw_frontends arg, immediately after the port. + // with ssl enabled, the port number must end with the letter s. + sslSuffix = fmt.Sprintf("s ssl_certificate=%s", config.CertificatePath) + } + settings := map[string]string{ "host": config.Host, "rgw port": strconv.Itoa(config.Port), @@ -74,6 +82,7 @@ func generateConfigFiles(context *clusterd.Context, config *Config) error { "rgw log nonexistent bucket": "true", "rgw intent log object name utc": "true", "rgw enable usage log": "true", + "rgw_frontends": fmt.Sprintf("civetweb port=%d%s", config.Port, sslSuffix), "rgw_zone": config.Name, "rgw_zonegroup": config.Name, } @@ -104,7 +113,6 @@ func generateConfigFiles(context *clusterd.Context, config *Config) error { } func startRGW(context *clusterd.Context, config *Config) (rgwProc *proc.MonitoredProc, err error) { - // start the monitor daemon in the foreground with the given config logger.Infof("starting rgw") @@ -118,7 +126,6 @@ func startRGW(context *clusterd.Context, config *Config) (rgwProc *proc.Monitore fmt.Sprintf("--cluster=%s", config.ClusterInfo.Name), fmt.Sprintf("--conf=%s", confFile), fmt.Sprintf("--keyring=%s", getRGWKeyringPath(context.ConfigDir)), - fmt.Sprintf("--rgw-frontends=civetweb port=%d", config.Port), fmt.Sprintf("--rgw-mime-types-file=%s", getMimeTypesPath(context.ConfigDir)), } if config.InProc { diff --git a/pkg/model/pool.go b/pkg/model/pool.go index fd7907044..fc14304d4 100644 --- a/pkg/model/pool.go +++ b/pkg/model/pool.go @@ -27,6 +27,7 @@ type ObjectStore struct { Name string `json:"name"` Port int32 `json:"port"` RGWReplicas int32 `json:"rgwReplicas"` + Certificate string `json:"certificate"` CertificateRef string `json:"certificateRef"` DataConfig Pool `json:"dataConfig"` MetadataConfig Pool `json:"metadataConfig"` diff --git a/pkg/operator/k8sutil/pod.go b/pkg/operator/k8sutil/pod.go index 8e6301547..db40161d6 100644 --- a/pkg/operator/k8sutil/pod.go +++ b/pkg/operator/k8sutil/pod.go @@ -48,7 +48,7 @@ const ( ConfigOverrideVal = "config" repoPrefixEnvVar = "ROOK_REPO_PREFIX" defaultVersion = "latest" - configMountDir = "/etc/rook" + configMountDir = "/etc/rook/config" overrideFilename = "override.conf" ) diff --git a/pkg/operator/rgw/rgw.go b/pkg/operator/rgw/rgw.go index 8aa4d3535..159c8fa0b 100644 --- a/pkg/operator/rgw/rgw.go +++ b/pkg/operator/rgw/rgw.go @@ -20,6 +20,8 @@ package rgw import ( "encoding/json" "fmt" + "path" + "strings" "github.com/coreos/pkg/capnslog" "github.com/rook/rook/pkg/ceph/client" @@ -39,8 +41,12 @@ import ( var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-rgw") const ( - appName = "rook-ceph-rgw" - keyringName = "keyring" + appName = "rook-ceph-rgw" + keyringName = "keyring" + certVolumeName = "rook-rgw-cert" + certMountPath = "/etc/rook/private" + certKeyName = "cert" + certFilename = "rgw-cert.pem" ) // Cluster for rgw management @@ -142,7 +148,7 @@ func (c *Cluster) createRealm(serviceIP string) error { zoneGroupID, err := decodeID(output) if err != nil { - return fmt.Errorf("failed to parse realm id. %+v", err) + return fmt.Errorf("failed to parse zone group id. %+v", err) } // create the zone if it doesn't exist yet @@ -160,6 +166,7 @@ func (c *Cluster) createRealm(serviceIP string) error { } if updatePeriod { + // the period will help notify other zones of changes if there are multi-zones _, err = c.runRGWCommand("period", "update", "--commit") if err != nil { return fmt.Errorf("failed to update period. %+v", err) @@ -201,7 +208,11 @@ func (c *Cluster) createSimilarPools(pools []string, poolConfig model.Pool) erro for _, pool := range pools { // create the pool if it doesn't exist yet - name := fmt.Sprintf("%s.%s", c.Config.Name, pool) + name := pool + if !strings.HasPrefix(pool, ".") { + // the name of the pool is ., except for the pool ".rgw.root" that spans object stores + name = fmt.Sprintf("%s.%s", c.Config.Name, pool) + } if _, err := ceph.GetPoolDetails(c.context, c.Namespace, name); err != nil { cephConfig.Name = name err := ceph.CreatePool(c.context, c.Namespace, cephConfig) @@ -289,11 +300,21 @@ func (c *Cluster) makeDeployment() *extensions.Deployment { k8sutil.ConfigOverrideVolume(), }, } + + // Set the ssl cert if specified + if c.Config.CertificateRef != "" { + certVol := v1.Volume{Name: certVolumeName, VolumeSource: v1.VolumeSource{Secret: &v1.SecretVolumeSource{ + SecretName: c.Config.CertificateRef, + Items: []v1.KeyToPath{{Key: certKeyName, Path: certFilename}}, + }}} + podSpec.Volumes = append(podSpec.Volumes, certVol) + } + c.placement.ApplyToPodSpec(&podSpec) podTemplateSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Name: "rook-ceph-rgw", + Name: c.instanceName(), Labels: c.getLabels(), Annotations: map[string]string{}, }, @@ -307,7 +328,7 @@ func (c *Cluster) makeDeployment() *extensions.Deployment { func (c *Cluster) rgwContainer() v1.Container { - return v1.Container{ + container := v1.Container{ Args: []string{ "rgw", fmt.Sprintf("--config-dir=%s", k8sutil.DataDir), @@ -332,6 +353,18 @@ func (c *Cluster) rgwContainer() v1.Container { k8sutil.ConfigOverrideEnvVar(), }, } + + if c.Config.CertificateRef != "" { + // Add a volume mount for the ssl certificate + mount := v1.VolumeMount{Name: certVolumeName, MountPath: certMountPath, ReadOnly: true} + container.VolumeMounts = append(container.VolumeMounts, mount) + + // Pass the flag for using the ssl cert + path := path.Join(certMountPath, certFilename) + container.Args = append(container.Args, fmt.Sprintf("--rgw-cert=%s", path)) + } + + return container } func (c *Cluster) startService() (string, error) { diff --git a/pkg/operator/rgw/rgw_test.go b/pkg/operator/rgw/rgw_test.go index fa18d4cdb..b8251d0ec 100644 --- a/pkg/operator/rgw/rgw_test.go +++ b/pkg/operator/rgw/rgw_test.go @@ -101,9 +101,29 @@ func TestPodSpecs(t *testing.T) { assert.Equal(t, "rook/rook:myversion", cont.Image) assert.Equal(t, 2, len(cont.VolumeMounts)) + assert.Equal(t, 5, len(cont.Args)) assert.Equal(t, "rgw", cont.Args[0]) assert.Equal(t, "--config-dir=/var/lib/rook", cont.Args[1]) assert.Equal(t, fmt.Sprintf("--rgw-name=%s", "default"), cont.Args[2]) assert.Equal(t, fmt.Sprintf("--rgw-port=%d", 123), cont.Args[3]) assert.Equal(t, fmt.Sprintf("--rgw-host=%s", c.instanceName()), cont.Args[4]) } + +func TestSSLPodSpec(t *testing.T) { + config := model.ObjectStore{Name: "default", Port: 123, CertificateRef: "mycert"} + c := New(nil, config, "ns", "myversion", k8sutil.Placement{}) + + d := c.makeDeployment() + assert.NotNil(t, d) + assert.Equal(t, c.instanceName(), d.Name) + assert.Equal(t, 3, len(d.Spec.Template.Spec.Volumes)) + assert.Equal(t, certVolumeName, d.Spec.Template.Spec.Volumes[2].Name) + + cont := d.Spec.Template.Spec.Containers[0] + assert.Equal(t, 3, len(cont.VolumeMounts)) + assert.Equal(t, certVolumeName, cont.VolumeMounts[2].Name) + assert.Equal(t, certMountPath, cont.VolumeMounts[2].MountPath) + + assert.Equal(t, 6, len(cont.Args)) + assert.Equal(t, fmt.Sprintf("--rgw-cert=%s/%s", certMountPath, certFilename), cont.Args[5]) +} diff --git a/pkg/rook/client/client_test.go b/pkg/rook/client/client_test.go index f127a3223..39651e81f 100644 --- a/pkg/rook/client/client_test.go +++ b/pkg/rook/client/client_test.go @@ -253,7 +253,8 @@ func TestCreateObjectStore(t *testing.T) { mockHttpClient := NewMockHttpClient(mockServer.URL) client := NewRookNetworkRestClient(mockServer.URL, mockHttpClient) - resp, err := client.CreateObjectStore() + store := model.ObjectStore{Name: "default", Port: 123} + resp, err := client.CreateObjectStore(store) assert.NotNil(t, err) assert.True(t, IsHttpAccepted(err)) assert.Equal(t, "", resp) @@ -343,7 +344,7 @@ func TestDeleteFilesystemFailure(t *testing.T) { func TestCreateObjectStoreFailure(t *testing.T) { clientFunc := func(client RookRestClient) (interface{}, error) { - return client.CreateObjectStore() + return client.CreateObjectStore(model.ObjectStore{Name: "name"}) } verifyFunc := getStringVerifyFunc(t) ClientFailureHelperWithVerification(t, clientFunc, verifyFunc)