Merge pull request #2738 from SUSE/rgw-all-in-operator

Ceph rgw: configure completely from operator
This commit is contained in:
Travis Nielsen authored and GitHub committed 2019-03-07 14:35:28 -07:00
commit dcd24d3cca
26 files changed
+492 -592

No files matched your search

+15
View File
@@ -86,3 +86,18 @@ The gateway settings correspond to the RGW daemon settings.
- `allNodes`: Whether RGW pods should be started on all nodes. If true, a daemonset is created. If false, `instances` must be set.
- `placement`: The Kubernetes placement settings to determine where the RGW pods should be started in the cluster.
- `resources`: Set resource requests/limits for the Gateway Pod(s), see [Resource Requirements/Limits](ceph-cluster-crd.md#resource-requirementslimits).
## Runtime settings
### MIME types
Rook provides a default `mime.types` file for each Ceph object store. This file is stored in a
Kubernetes ConfigMap with the name `rook-ceph-rgw-<STORE-NAME>-mime-types`. For most users, the
default file should suffice, however, the option is available to users to edit the `mime.types`
file in the ConfigMap as they desire. Users may have their own special file types, and particularly
security conscious users may wish to pare down the file to reduce the possibility of a file type
execution attack.
Rook will not overwrite an existing `mime.types` ConfigMap so that user modifications will not be
destroyed. If the object store is destroyed and recreated, the ConfigMap will also be destroyed and
created anew.
+4 -3
View File
@@ -29,9 +29,10 @@
- Rook will no longer create a directory-based osd in the `dataDirHostPath` if no directories or
devices are specified or if there are no disks on the host.
- Containers in `mon`, `mgr`, and `mds` pods have been removed and/or changed names.
- Config paths in `mon`, `mgr`, and `mds` containers are now always the Ceph default paths
(`/etc/ceph`, `/var/lib/ceph/...`) regardless of the `dataDirHostPath` setting.
- Containers in `mon`, `mgr`, `mds`, and `rgw` pods have been removed and/or changed names.
- Config paths in `mon`, `mgr`, `mds` and `rgw` containers are now always under
`/etc/ceph` or `/var/lib/ceph` and as close to Ceph's default path as possible regardless of the
`dataDirHostPath` setting.
## Known Issues
-1
View File
@@ -58,7 +58,6 @@ func init() {
Cmd.AddCommand(operatorCmd,
agentCmd,
osdCmd,
rgwCmd,
configCmd,
nfsCmd)
}
-93
View File
@@ -1,93 +0,0 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ceph
import (
"fmt"
"os"
"github.com/rook/rook/cmd/rook/rook"
rgwdaemon "github.com/rook/rook/pkg/daemon/ceph/rgw"
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
"github.com/rook/rook/pkg/util/flags"
"github.com/spf13/cobra"
)
var rgwCmd = &cobra.Command{
Use: "rgw",
Short: "Generates rgw config and runs the rgw daemon",
}
var (
rgwName string
rgwKeyring string
rgwHost string
rgwCert string
rgwPort int
rgwSecurePort int
)
func init() {
rgwCmd.Flags().StringVar(&rgwName, "rgw-name", "", "name of the object store")
rgwCmd.Flags().StringVar(&rgwKeyring, "rgw-keyring", "", "the rgw keyring")
rgwCmd.Flags().StringVar(&rgwHost, "rgw-host", os.Getenv("HOSTNAME"), "RGW host name. Becomes the only accepted hostname if the rgw dns name property is unset. Defaults to the pod hostname")
rgwCmd.Flags().StringVar(&rgwCert, "rgw-cert", "", "path to the ssl certificate in pem format")
rgwCmd.Flags().IntVar(&rgwPort, "rgw-port", 0, "rgw port (http)")
rgwCmd.Flags().IntVar(&rgwSecurePort, "rgw-secure-port", 0, "rgw secure port number (https)")
addCephFlags(rgwCmd)
flags.SetFlagsFromEnv(rgwCmd.Flags(), rook.RookEnvVarPrefix)
rgwCmd.RunE = initRGW
}
func initRGW(cmd *cobra.Command, args []string) error {
required := []string{"mon-endpoints", "cluster-name", "rgw-name", "rgw-keyring"}
if err := flags.VerifyRequiredFlags(rgwCmd, required); err != nil {
return err
}
if err := verifyRenamedFlags(rgwCmd); err != nil {
return err
}
if rgwPort == 0 && rgwSecurePort == 0 {
return fmt.Errorf("port or secure port are required")
}
rook.SetLogLevel()
rook.LogStartupInfo(rgwCmd.Flags())
clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
config := &rgwdaemon.Config{
ClusterInfo: &clusterInfo,
Name: rgwName,
Keyring: rgwKeyring,
Host: rgwHost,
Port: rgwPort,
SecurePort: rgwSecurePort,
CertificatePath: rgwCert,
}
err := rgwdaemon.Initialize(createContext(), config)
if err != nil {
rook.TerminateFatal(err)
}
return nil
}
-147
View File
@@ -1,147 +0,0 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package rgw
import (
"fmt"
"io/ioutil"
"os"
"path"
"strconv"
"github.com/coreos/pkg/capnslog"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
"github.com/rook/rook/pkg/util"
)
var (
logger = capnslog.NewPackageLogger("github.com/rook/rook", "cephrgw")
rgwFrontendName = "civetweb"
)
const (
keyringTemplate = `[client.radosgw.gateway]
key = %s
caps mon = "allow rw"
caps osd = "allow *"
`
)
type Config struct {
Name string
Host string
Port int
SecurePort int
Keyring string
CertificatePath string
ClusterInfo *cephconfig.ClusterInfo
}
func Initialize(context *clusterd.Context, config *Config) error {
err := generateConfigFiles(context, config)
if err != nil {
return fmt.Errorf("failed to generate rgw config files. %+v", err)
}
return err
}
func portString(config *Config) string {
var portString string
if config.Port != 0 {
portString = strconv.Itoa(config.Port)
}
if config.SecurePort != 0 && config.CertificatePath != "" {
var separator string
if config.Port != 0 {
separator = "+"
}
// the suffix is intended to be appended to the end of the rgw_frontends arg, immediately after the port.
// with ssl enabled, the port number must end with the letter s.
portString = fmt.Sprintf("%s%s%ds ssl_certificate=%s", portString, separator, config.SecurePort, config.CertificatePath)
}
return portString
}
func rgwFrontend(config *Config) string {
if cephv1.VersionAtLeast(config.ClusterInfo.CephVersionName, cephv1.Nautilus) {
rgwFrontendName = "beast"
}
return rgwFrontendName
}
func generateConfigFiles(context *clusterd.Context, config *Config) error {
// create the rgw data directory
dataDir := path.Join(getRGWConfDir(context.ConfigDir), "data")
if err := os.MkdirAll(dataDir, 0744); err != nil {
logger.Warningf("failed to create data directory %s: %+v", dataDir, err)
}
settings := map[string]string{
"host": config.Host,
"rgw data": dataDir,
"rgw log nonexistent bucket": "true",
"rgw intent log object name utc": "true",
"rgw enable usage log": "true",
"rgw_frontends": fmt.Sprintf("%s port=%s", rgwFrontend(config), portString(config)),
"rgw_zone": config.Name,
"rgw_zonegroup": config.Name,
}
configFile, err := cephconfig.GenerateConfigFile(context, config.ClusterInfo, getRGWConfDir(context.ConfigDir),
"client.radosgw.gateway", getRGWKeyringPath(context.ConfigDir), nil, settings)
if err != nil {
return fmt.Errorf("failed to create config file. %+v", err)
}
util.WriteFileToLog(logger, configFile)
keyringEval := func(key string) string {
return fmt.Sprintf(keyringTemplate, key)
}
// create rgw config
err = cephconfig.WriteKeyring(getRGWKeyringPath(context.ConfigDir), config.Keyring, keyringEval)
if err != nil {
return fmt.Errorf("failed to save keyring. %+v", err)
}
// write the mime types config
mimeTypesPath := GetMimeTypesPath(context.ConfigDir)
logger.Infof("Writing mime types to: %s", mimeTypesPath)
if err := ioutil.WriteFile(mimeTypesPath, []byte(mimeTypes), 0644); err != nil {
return fmt.Errorf("failed to write mime types to %s: %+v", mimeTypesPath, err)
}
return nil
}
func getRGWConfDir(configDir string) string {
return path.Join(configDir, "rgw")
}
func getRGWKeyringPath(configDir string) string {
return path.Join(getRGWConfDir(configDir), "keyring")
}
func GetMimeTypesPath(configDir string) string {
return path.Join(getRGWConfDir(configDir), "mime.types")
}
-67
View File
@@ -1,67 +0,0 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package rgw
import (
"testing"
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
"github.com/stretchr/testify/assert"
)
func TestPortString(t *testing.T) {
// No port or secure port
cfg := &Config{}
result := portString(cfg)
assert.Equal(t, "", result)
// Insecure port
cfg = &Config{Port: 80}
result = portString(cfg)
assert.Equal(t, "80", result)
// Secure port
cfg = &Config{SecurePort: 443, CertificatePath: "/etc/rgw/cert.pem"}
result = portString(cfg)
assert.Equal(t, "443s ssl_certificate=/etc/rgw/cert.pem", result)
// Both ports
cfg = &Config{Port: 80, SecurePort: 443, CertificatePath: "/etc/rgw/cert.pem"}
result = portString(cfg)
assert.Equal(t, "80+443s ssl_certificate=/etc/rgw/cert.pem", result)
// Secure port requires the cert
cfg = &Config{SecurePort: 443}
result = portString(cfg)
assert.Equal(t, "", result)
}
func TestFrontend(t *testing.T) {
cfg := &Config{}
cfg.ClusterInfo = &cephconfig.ClusterInfo{CephVersionName: "mimic"}
result := rgwFrontend(cfg)
assert.Equal(t, "civetweb", result)
cfg.ClusterInfo.CephVersionName = "nautilus"
result = rgwFrontend(cfg)
assert.Equal(t, "beast", result)
cfg.ClusterInfo.CephVersionName = "octopus"
result = rgwFrontend(cfg)
assert.Equal(t, "beast", result)
}
+1 -1
View File
@@ -229,7 +229,7 @@ func (c *ClusterController) onAdd(obj interface{}) {
poolController.StartWatch(cluster.Namespace, cluster.stopCh)
// Start object store CRD watcher
objectStoreController := object.NewObjectStoreController(c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef)
objectStoreController := object.NewObjectStoreController(cluster.Info, c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef)
objectStoreController.StartWatch(cluster.Namespace, cluster.stopCh)
// Start object store user CRD watcher
+1 -1
View File
@@ -161,7 +161,7 @@ func (s *Store) createOrUpdateMonHostSecrets(clusterInfo *cephconfig.ClusterInfo
monIP := cephutil.GetIPFromEndpoint(m.Endpoint)
// This tries to detect the current port if the mon already exists
// This basically handles the transtion between monitors running on 6790 to msgr2
// This basically handles the transition between monitors running on 6790 to msgr2
// So whatever the previous monitor port was we keep it
currentMonPort := cephutil.GetPortFromEndpoint(m.Endpoint)
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package rgw
package object
import (
"fmt"
@@ -23,12 +23,14 @@ import (
"github.com/rook/rook/pkg/daemon/ceph/client"
)
// Context holds the context for the object store.
type Context struct {
context *clusterd.Context
Name string
ClusterName string
}
// NewContext creates a new object store context.
func NewContext(context *clusterd.Context, name, clusterName string) *Context {
return &Context{context: context, Name: name, ClusterName: clusterName}
}
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package rgw
package object
import (
"encoding/json"
+100
View File
@@ -0,0 +1,100 @@
/*
Copyright 2019 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package object
import (
"fmt"
"path"
"strconv"
cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
keyringTemplate = `
[client.radosgw.gateway]
key = %s
caps mon = "allow rw"
caps osd = "allow rwx"
`
certVolumeName = "rook-ceph-rgw-cert"
certDir = "/etc/ceph/private"
certKeyName = "cert"
certFilename = "rgw-cert.pem"
)
// TODO: these should be set in the mon's central kv store for mimic+
func (c *clusterConfig) defaultSettings() *cephconfig.Config {
s := cephconfig.NewConfig()
s.Section("global").
Set("rgw log nonexistent bucket", "true").
Set("rgw intent log object name utc", "true").
Set("rgw enable usage log", "true").
Set("rgw frontends", fmt.Sprintf("civetweb port=%s", c.portString())).
Set("rgw zone", c.store.Name).
Set("rgw zonegroup", c.store.Name)
return s
}
func (c *clusterConfig) portString() string {
var portString string
port := c.store.Spec.Gateway.Port
if port != 0 {
portString = strconv.Itoa(int(port))
}
if c.store.Spec.Gateway.SecurePort != 0 && c.store.Spec.Gateway.SSLCertificateRef != "" {
var separator string
if port != 0 {
separator = "+"
}
certPath := path.Join(certDir, certFilename)
// with ssl enabled, the port number must end with the letter s.
// e.g., "443s ssl_certificate=/etc/ceph/private/keyandcert.pem"
portString = fmt.Sprintf("%s%s%ds ssl_certificate=%s",
portString, separator, c.store.Spec.Gateway.SecurePort, certPath)
}
return portString
}
func (c *clusterConfig) generateKeyring(replicationControllerOwnerRef *metav1.OwnerReference) error {
user := "client.radosgw.gateway"
/* TODO: this says `osd allow rwx` while template says `osd allow *`; which is correct? */
access := []string{"osd", "allow rwx", "mon", "allow rw"}
s := keyring.GetSecretStore(c.context, c.store.Namespace, replicationControllerOwnerRef)
key, err := s.GenerateKey(c.instanceName(), user, access)
if err != nil {
return err
}
// Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x
err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Delete(c.instanceName(), &metav1.DeleteOptions{})
if err != nil {
if errors.IsNotFound(err) {
logger.Debugf("legacy rgw key %s is already removed", c.instanceName())
} else {
logger.Warningf("legacy rgw key %s could not be removed. %+v", c.instanceName(), err)
}
}
keyring := fmt.Sprintf(keyringTemplate, key)
return s.CreateOrUpdate(c.instanceName(), keyring)
}
+66
View File
@@ -0,0 +1,66 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package object
import (
"testing"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/stretchr/testify/assert"
)
func newConfig() *clusterConfig {
return &clusterConfig{
store: cephv1.CephObjectStore{
Spec: cephv1.ObjectStoreSpec{
Gateway: cephv1.GatewaySpec{},
}}}
}
func TestPortString(t *testing.T) {
// No port or secure port
cfg := newConfig()
result := cfg.portString()
assert.Equal(t, "", result)
// Insecure port
cfg = newConfig()
cfg.store.Spec.Gateway.Port = 80
result = cfg.portString()
assert.Equal(t, "80", result)
// Secure port
cfg = newConfig()
cfg.store.Spec.Gateway.SecurePort = 443
cfg.store.Spec.Gateway.SSLCertificateRef = "some-k8s-key-secret"
result = cfg.portString()
assert.Equal(t, "443s ssl_certificate=/etc/ceph/private/rgw-cert.pem", result)
// Both ports
cfg = newConfig()
cfg.store.Spec.Gateway.Port = 80
cfg.store.Spec.Gateway.SecurePort = 443
cfg.store.Spec.Gateway.SSLCertificateRef = "some-k8s-key-secret"
result = cfg.portString()
assert.Equal(t, "80+443s ssl_certificate=/etc/ceph/private/rgw-cert.pem", result)
// Secure port requires the cert
cfg = newConfig()
cfg.store.Spec.Gateway.SecurePort = 443
result = cfg.portString()
assert.Equal(t, "", result)
}
+33 -5
View File
@@ -14,7 +14,6 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
// Package rgw to manage a rook object store.
package object
import (
@@ -26,6 +25,8 @@ import (
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
cephbeta "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
"github.com/rook/rook/pkg/clusterd"
daemonconfig "github.com/rook/rook/pkg/daemon/ceph/config"
cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/pool"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
@@ -56,6 +57,7 @@ var ObjectStoreResourceRookLegacy = opkit.CustomResource{
// ObjectStoreController represents a controller object for object store custom resources
type ObjectStoreController struct {
clusterInfo *daemonconfig.ClusterInfo
context *clusterd.Context
rookImage string
cephVersion cephv1.CephVersionSpec
@@ -64,8 +66,16 @@ type ObjectStoreController struct {
}
// NewObjectStoreController create controller for watching object store custom resources created
func NewObjectStoreController(context *clusterd.Context, rookImage string, cephVersion cephv1.CephVersionSpec, hostNetwork bool, ownerRef metav1.OwnerReference) *ObjectStoreController {
func NewObjectStoreController(
clusterInfo *daemonconfig.ClusterInfo,
context *clusterd.Context,
rookImage string,
cephVersion cephv1.CephVersionSpec,
hostNetwork bool,
ownerRef metav1.OwnerReference,
) *ObjectStoreController {
return &ObjectStoreController{
clusterInfo: clusterInfo,
context: context,
rookImage: rookImage,
cephVersion: cephVersion,
@@ -107,7 +117,16 @@ func (c *ObjectStoreController) onAdd(obj interface{}) {
return
}
cfg := config{c.context, *objectstore, c.rookImage, c.cephVersion, c.hostNetwork, c.storeOwners(objectstore)}
cfg := clusterConfig{
clusterInfo: c.clusterInfo,
context: c.context,
store: *objectstore,
rookVersion: c.rookImage,
cephVersion: c.cephVersion,
hostNetwork: c.hostNetwork,
ownerRefs: c.storeOwners(objectstore),
DataPathMap: cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, objectstore.Name),
}
if err = cfg.createStore(); err != nil {
logger.Errorf("failed to create object store %s. %+v", objectstore.Name, err)
}
@@ -139,7 +158,16 @@ func (c *ObjectStoreController) onUpdate(oldObj, newObj interface{}) {
}
logger.Infof("applying object store %s changes", newStore.Name)
cfg := config{c.context, *newStore, c.rookImage, c.cephVersion, c.hostNetwork, c.storeOwners(newStore)}
cfg := clusterConfig{
c.clusterInfo,
c.context,
*newStore,
c.rookImage,
c.cephVersion,
c.hostNetwork,
c.storeOwners(newStore),
cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, newStore.Name),
}
if err = cfg.updateStore(); err != nil {
logger.Errorf("failed to create (modify) object store %s. %+v", newStore.Name, err)
}
@@ -157,7 +185,7 @@ func (c *ObjectStoreController) onDelete(obj interface{}) {
return
}
cfg := config{context: c.context, store: *objectstore}
cfg := clusterConfig{context: c.context, store: *objectstore}
if err = cfg.deleteStore(); err != nil {
logger.Errorf("failed to delete object store %s. %+v", objectstore.Name, err)
}
+2 -1
View File
@@ -91,7 +91,8 @@ func TestMigrateObjectStoreObject(t *testing.T) {
Clientset: clientset,
RookClientset: rookfake.NewSimpleClientset(legacyObjectStore),
}
controller := NewObjectStoreController(context, "", cephv1.CephVersionSpec{}, false, metav1.OwnerReference{})
info := testop.CreateConfigDir(1)
controller := NewObjectStoreController(info, context, "", cephv1.CephVersionSpec{}, false, metav1.OwnerReference{})
// convert the legacy objectstore object in memory and assert that a migration is needed
convertedObjectStore, migrationNeeded, err := getObjectStoreObject(legacyObjectStore)
@@ -14,7 +14,67 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package rgw
package object
import (
"fmt"
"path"
"github.com/rook/rook/pkg/operator/k8sutil"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
// these control the location where the mime.types file is mounted in containers.
// Ceph's default is /etc/mime.types, but we can't mount a configmap to `/etc` without breaking
// the container. /etc/ceph/rgw won't overwrite ceph.conf (if it exists), and it will be easy to
// find manually by inspection of the `/etc/ceph` dir.
mimeTypesMountDir = "/etc/ceph/rgw"
mimeTypesFileName = "mime.types"
)
func (c *clusterConfig) mimeTypesConfigMapName() string {
return fmt.Sprintf("%s-mime-types", c.instanceName())
}
func mimeTypesMountPath() string {
return path.Join(mimeTypesMountDir, mimeTypesFileName)
}
// store mime.types file in a config map
func (c *clusterConfig) generateMimeTypes(ownerRef *metav1.OwnerReference) error {
k := k8sutil.NewConfigMapKVStore(c.store.Namespace, c.context.Clientset, *ownerRef)
if _, err := k.GetValue(c.mimeTypesConfigMapName(), mimeTypesFileName); err == nil || !errors.IsNotFound(err) {
logger.Infof("config map for object pool %s already exists, not overwriting", c.store.Name)
return nil
}
// is not found
if err := k.SetValue(c.mimeTypesConfigMapName(), mimeTypesFileName, mimeTypes); err != nil {
return fmt.Errorf("failed to create config map for object pool %s. %+v", c.store.Name, err)
}
return nil
}
func (c *clusterConfig) mimeTypesVolume() v1.Volume {
return v1.Volume{
Name: c.mimeTypesConfigMapName(),
VolumeSource: v1.VolumeSource{
ConfigMap: &v1.ConfigMapVolumeSource{LocalObjectReference: v1.LocalObjectReference{
Name: c.mimeTypesConfigMapName(),
}}}}
}
func (c *clusterConfig) mimeTypesVolumeMount() v1.VolumeMount {
return v1.VolumeMount{
Name: c.mimeTypesConfigMapName(),
MountPath: mimeTypesMountDir,
ReadOnly: true,
// read-only so malicious actors can't edit, which could allow them to add file type
// execution vulnerabilities that an admin has taken care to eliminate
}
}
const mimeTypes = `
application/activemessage
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package rgw
package object
import (
"encoding/json"
@@ -27,7 +27,9 @@ import (
const (
rootPool = ".rgw.root"
appName = "rgw"
// AppName is the name Rook uses for the object store's application
AppName = "rook-ceph-rgw"
)
var (
@@ -51,7 +53,7 @@ type realmType struct {
Realms []string `json:"realms"`
}
func CreateObjectStore(context *Context, metadataSpec, dataSpec model.Pool, serviceIP string, port int32) error {
func createObjectStore(context *Context, metadataSpec, dataSpec model.Pool, serviceIP string, port int32) error {
err := createPools(context, metadataSpec, dataSpec)
if err != nil {
return fmt.Errorf("failed to create object pools. %+v", err)
@@ -64,8 +66,8 @@ func CreateObjectStore(context *Context, metadataSpec, dataSpec model.Pool, serv
return nil
}
func DeleteObjectStore(context *Context) error {
stores, err := GetObjectStores(context)
func deleteRealmAndPools(context *Context) error {
stores, err := getObjectStores(context)
if err != nil {
return fmt.Errorf("failed to detect object stores during deletion. %+v", err)
}
@@ -95,7 +97,7 @@ func createRealm(context *Context, serviceIP string, port int32) error {
// The first realm must be marked as the default
defaultArg := ""
stores, err := GetObjectStores(context)
stores, err := getObjectStores(context)
if err != nil {
return fmt.Errorf("failed to get object stores. %+v", err)
}
@@ -189,7 +191,7 @@ func decodeID(data string) (string, error) {
return id.ID, err
}
func GetObjectStores(context *Context) ([]string, error) {
func getObjectStores(context *Context) ([]string, error) {
output, err := runAdminCommandNoRealm(context, "realm", "list")
if err != nil {
if strings.Index(err.Error(), "exit status 2") != 0 {
@@ -274,9 +276,9 @@ func createSimilarPools(context *Context, pools []string, poolSpec model.Pool) e
if isECPool {
// An EC pool backing an object store does not need to enable EC overwrites, so the pool is
// created with that property disabled to avoid unnecessary performance impact.
err = ceph.CreateECPoolForApp(context.context, context.ClusterName, cephConfig, appName, false /* enableECOverwrite */, poolSpec.ErasureCodedConfig)
err = ceph.CreateECPoolForApp(context.context, context.ClusterName, cephConfig, AppName, false /* enableECOverwrite */, poolSpec.ErasureCodedConfig)
} else {
err = ceph.CreateReplicatedPoolForApp(context.context, context.ClusterName, cephConfig, appName)
err = ceph.CreateReplicatedPoolForApp(context.context, context.ClusterName, cephConfig, AppName)
}
if err != nil {
return fmt.Errorf("failed to create pool %s for object store %s", name, context.Name)
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package rgw
package object
import (
"fmt"
@@ -43,9 +43,8 @@ func TestCreateRealm(t *testing.T) {
} else if args[0] == "realm" && args[1] == "list" {
if defaultStore {
return "", fmt.Errorf("failed to run radosgw-admin: Failed to complete : exit status 2")
} else {
return `{"realms": ["myobj"]}`, nil
}
return `{"realms": ["myobj"]}`, nil
}
return idResponse, nil
}
@@ -146,7 +145,7 @@ func deleteStore(t *testing.T, name string, existingStores string, expectedDelet
context := &Context{context: &clusterd.Context{Executor: executor}, Name: "myobj", ClusterName: "ns"}
// Delete an object store
err := DeleteObjectStore(context)
err := deleteRealmAndPools(context)
assert.Nil(t, err)
expectedPoolsDeleted := 5
if expectedDeleteRootPool {
+55 -84
View File
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
// Package rgw for the Ceph object store.
// Package object for the Ceph object store.
package object
import (
@@ -22,43 +22,36 @@ import (
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
rgwdaemon "github.com/rook/rook/pkg/daemon/ceph/rgw"
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/pool"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
const (
appName = "rook-ceph-rgw"
keyringName = "keyring"
certVolumeName = "rook-rgw-cert"
certMountPath = "/etc/rook/private"
certKeyName = "cert"
certFilename = "rgw-cert.pem"
)
type config struct {
type clusterConfig struct {
clusterInfo *cephconfig.ClusterInfo
context *clusterd.Context
store cephv1.CephObjectStore
rookVersion string
cephVersion cephv1.CephVersionSpec
hostNetwork bool
ownerRefs []metav1.OwnerReference
DataPathMap *config.DataPathMap
}
// Start the rgw manager
func (c *config) createStore() error {
func (c *clusterConfig) createStore() error {
return c.createOrUpdate(false)
}
func (c *config) updateStore() error {
func (c *clusterConfig) updateStore() error {
return c.createOrUpdate(true)
}
func (c *config) createOrUpdate(update bool) error {
func (c *clusterConfig) createOrUpdate(update bool) error {
// validate the object store settings
if err := validateStore(c.context, c.store); err != nil {
return fmt.Errorf("invalid object store %s arguments. %+v", c.store.Name, err)
@@ -75,10 +68,6 @@ func (c *config) createOrUpdate(update bool) error {
}
logger.Infof("creating object store %s in namespace %s", c.store.Name, c.store.Namespace)
err = c.createKeyring()
if err != nil {
return fmt.Errorf("failed to create rgw keyring. %+v", err)
}
// start the service
serviceIP, err := c.startService()
@@ -87,8 +76,8 @@ func (c *config) createOrUpdate(update bool) error {
}
// create the ceph artifacts for the object store
objContext := rgwdaemon.NewContext(c.context, c.store.Name, c.store.Namespace)
err = rgwdaemon.CreateObjectStore(objContext, *c.store.Spec.MetadataPool.ToModel(""), *c.store.Spec.DataPool.ToModel(""), serviceIP, c.store.Spec.Gateway.Port)
objContext := NewContext(c.context, c.store.Name, c.store.Namespace)
err = createObjectStore(objContext, *c.store.Spec.MetadataPool.ToModel(""), *c.store.Spec.DataPool.ToModel(""), serviceIP, c.store.Spec.Gateway.Port)
if err != nil {
return fmt.Errorf("failed to create pools. %+v", err)
}
@@ -101,7 +90,7 @@ func (c *config) createOrUpdate(update bool) error {
return nil
}
func (c *config) startRGWPods(update bool) error {
func (c *clusterConfig) startRGWPods(update bool) error {
// if intended to update, remove the old pods so they can be created with the new spec settings
if update {
@@ -116,15 +105,49 @@ func (c *config) startRGWPods(update bool) error {
}
// start the deployment or daemonset
var uid types.UID
var controllerType string
if c.store.Spec.Gateway.AllNodes {
return c.startDaemonset()
daemonSet, err := c.startDaemonset()
if err != nil {
return err
}
uid = daemonSet.UID
controllerType = "daemonset"
} else {
deployment, err := c.startDeployment()
if err != nil {
return err
}
uid = deployment.UID
controllerType = "deployment"
}
return c.startDeployment()
resourceControllerOwnerRef := &metav1.OwnerReference{
UID: uid,
APIVersion: "v1",
Kind: controllerType,
Name: c.instanceName(),
}
// Generate the keyring after starting the replication controller so that the keyring may use
// the controller as its owner reference; the keyring is deleted with the controller
err := c.generateKeyring(resourceControllerOwnerRef)
if err != nil {
return fmt.Errorf("failed to create rgw keyring. %+v", err)
}
// Generate the mime.types file after the rep. controller as well for the same reason as keyring
if err := c.generateMimeTypes(resourceControllerOwnerRef); err != nil {
return fmt.Errorf("failed to generate the rgw mime.types config. %+v", err)
}
return nil
}
// Delete the object store.
// WARNING: This is a very destructive action that deletes all metadata and data pools.
func (c *config) deleteStore() error {
func (c *clusterConfig) deleteStore() error {
// check if the object store exists
exists, err := c.storeExists()
if err != nil {
@@ -164,8 +187,8 @@ func (c *config) deleteStore() error {
}
// Delete the realm and pools
objContext := rgwdaemon.NewContext(c.context, c.store.Name, c.store.Namespace)
err = rgwdaemon.DeleteObjectStore(objContext)
objContext := NewContext(c.context, c.store.Name, c.store.Namespace)
err = deleteRealmAndPools(objContext)
if err != nil {
return fmt.Errorf("failed to delete the realm and pools. %+v", err)
}
@@ -175,7 +198,7 @@ func (c *config) deleteStore() error {
}
// Check if the object store exists depending on either the deployment or the daemonset
func (c *config) storeExists() (bool, error) {
func (c *clusterConfig) storeExists() (bool, error) {
_, err := c.context.Clientset.Apps().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
if err == nil {
// the deployment was found
@@ -198,60 +221,8 @@ func (c *config) storeExists() (bool, error) {
return false, nil
}
func (c *config) createKeyring() error {
_, err := c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
if err == nil {
logger.Infof("the rgw keyring was already generated")
return nil
}
if !errors.IsNotFound(err) {
return fmt.Errorf("failed to get rgw secrets. %+v", err)
}
// create the keyring
logger.Infof("generating rgw keyring")
keyring, err := c.createRGWKeyring()
if err != nil {
return fmt.Errorf("failed to create keyring. %+v", err)
}
// store the secrets
secrets := map[string]string{
keyringName: keyring,
}
secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: c.instanceName(),
Namespace: c.store.Namespace,
},
StringData: secrets,
Type: k8sutil.RookType,
}
k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &secret.ObjectMeta, c.ownerRefs)
_, err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Create(secret)
if err != nil {
return fmt.Errorf("failed to save rgw secrets. %+v", err)
}
return nil
}
func (c *config) instanceName() string {
return fmt.Sprintf("%s-%s", appName, c.store.Name)
}
// create a keyring for the rgw client with a limited set of privileges
func (c *config) createRGWKeyring() (string, error) {
username := "client.radosgw.gateway"
access := []string{"osd", "allow rwx", "mon", "allow rw"}
// get-or-create-key for the user account
key, err := client.AuthGetOrCreateKey(c.context, c.store.Namespace, username, access)
if err != nil {
return "", fmt.Errorf("failed to get or create auth key for %store. %+v", username, err)
}
return key, err
func (c *clusterConfig) instanceName() string {
return fmt.Sprintf("%s-%s", AppName, c.store.Name)
}
// Validate the object store arguments
+8 -8
View File
@@ -23,6 +23,7 @@ import (
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
testop "github.com/rook/rook/pkg/operator/test"
exectest "github.com/rook/rook/pkg/util/exec/test"
"github.com/stretchr/testify/assert"
@@ -44,12 +45,14 @@ func TestStartRGW(t *testing.T) {
configDir, _ := ioutil.TempDir("", "")
defer os.RemoveAll(configDir)
info := testop.CreateConfigDir(1)
context := &clusterd.Context{Clientset: clientset, Executor: executor, ConfigDir: configDir}
store := simpleStore()
version := "v1.1.0"
data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "my-fs")
// start a basic cluster
c := &config{context, store, version, cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}}
c := &clusterConfig{info, context, store, version, cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}, data}
err := c.createStore()
assert.Nil(t, err)
@@ -63,7 +66,7 @@ func TestStartRGW(t *testing.T) {
validateStart(t, c, clientset, true)
}
func validateStart(t *testing.T, c *config, clientset *fake.Clientset, allNodes bool) {
func validateStart(t *testing.T, c *clusterConfig, clientset *fake.Clientset, allNodes bool) {
if !allNodes {
r, err := clientset.Apps().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
assert.Nil(t, err)
@@ -83,11 +86,6 @@ func validateStart(t *testing.T, c *config, clientset *fake.Clientset, allNodes
s, err := clientset.CoreV1().Services(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, c.instanceName(), s.Name)
secret, err := clientset.CoreV1().Secrets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, c.instanceName(), secret.Name)
assert.Equal(t, 1, len(secret.StringData))
}
func TestCreateObjectStore(t *testing.T) {
@@ -114,9 +112,11 @@ func TestCreateObjectStore(t *testing.T) {
store := simpleStore()
clientset := testop.New(3)
context := &clusterd.Context{Executor: executor, Clientset: clientset}
info := testop.CreateConfigDir(1)
data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "my-fs")
// create the pools
c := &config{context, store, "1.2.3.4", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}}
c := &clusterConfig{info, context, store, "1.2.3.4", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}, data}
err := c.createStore()
assert.Nil(t, err)
}
+55 -79
View File
@@ -14,15 +14,12 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
// Package object for the Ceph object store.
package object
import (
"fmt"
"path"
rgwdaemon "github.com/rook/rook/pkg/daemon/ceph/rgw"
opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon"
cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
opspec "github.com/rook/rook/pkg/operator/ceph/spec"
"github.com/rook/rook/pkg/operator/k8sutil"
apps "k8s.io/api/apps/v1"
@@ -32,11 +29,12 @@ import (
"k8s.io/apimachinery/pkg/util/intstr"
)
func (c *config) startDeployment() error {
func (c *clusterConfig) startDeployment() (*apps.Deployment, error) {
d := &apps.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: c.instanceName(),
Namespace: c.store.Namespace,
Labels: c.getLabels(),
},
Spec: apps.DeploymentSpec{
Selector: &metav1.LabelSelector{
@@ -51,29 +49,29 @@ func (c *config) startDeployment() error {
}
k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &d.ObjectMeta, c.ownerRefs)
logger.Debugf("starting mds deployment: %+v", d)
_, err := c.context.Clientset.Apps().Deployments(c.store.Namespace).Create(d)
logger.Debugf("starting rgw deployment: %+v", d)
deployment, err := c.context.Clientset.Apps().Deployments(c.store.Namespace).Create(d)
if err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create rgw deployment %s: %+v", c.instanceName(), err)
return nil, fmt.Errorf("failed to create rgw deployment %s: %+v", c.instanceName(), err)
}
logger.Infof("deployment for rgw %s already exists. updating if needed", c.instanceName())
// There may be a *lot* of rgws, and they are stateless, so don't bother waiting until the
// entire deployment is updated to move on.
_, err := c.context.Clientset.Apps().Deployments(c.store.Namespace).Update(d)
deployment, err = c.context.Clientset.Apps().Deployments(c.store.Namespace).Update(d)
if err != nil {
return fmt.Errorf("failed to update rgw deployment %s. %+v", c.instanceName(), err)
return nil, fmt.Errorf("failed to update rgw deployment %s. %+v", c.instanceName(), err)
}
}
return nil
return deployment, nil
}
func (c *config) startDaemonset() error {
func (c *clusterConfig) startDaemonset() (*apps.DaemonSet, error) {
d := &apps.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Name: c.instanceName(),
Namespace: c.store.Namespace,
Labels: c.getLabels(),
},
Spec: apps.DaemonSetSpec{
Selector: &metav1.LabelSelector{
@@ -88,36 +86,36 @@ func (c *config) startDaemonset() error {
k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &d.ObjectMeta, c.ownerRefs)
logger.Debugf("starting rgw daemonset: %+v", d)
_, err := c.context.Clientset.Apps().DaemonSets(c.store.Namespace).Create(d)
daemonSet, err := c.context.Clientset.Apps().DaemonSets(c.store.Namespace).Create(d)
if err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create rgw daemonset %s: %+v", c.instanceName(), err)
return nil, fmt.Errorf("failed to create rgw daemonset %s: %+v", c.instanceName(), err)
}
logger.Infof("daemonset for rgw %s already exists. updating if needed", c.instanceName())
// There may be a *lot* of rgws, and they are stateless, so don't bother waiting until the
// entire daemonset is updated to move on.
// TODO: is the above statement safe to assume?
// TODO: Are there any steps for RGW that need to happen before the daemons upgrade?
_, err := c.context.Clientset.Apps().DaemonSets(c.store.Namespace).Update(d)
daemonSet, err = c.context.Clientset.Apps().DaemonSets(c.store.Namespace).Update(d)
if err != nil {
return fmt.Errorf("failed to update rgw daemonset %s. %+v", c.instanceName(), err)
return nil, fmt.Errorf("failed to update rgw daemonset %s. %+v", c.instanceName(), err)
}
}
return nil
return daemonSet, nil
}
func (c *config) makeRGWPodSpec() v1.PodTemplateSpec {
func (c *clusterConfig) makeRGWPodSpec() v1.PodTemplateSpec {
podSpec := v1.PodSpec{
InitContainers: []v1.Container{
c.makeConfigInitContainer(),
},
InitContainers: []v1.Container{},
Containers: []v1.Container{
c.makeDaemonContainer(),
},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: opspec.PodVolumes(""),
HostNetwork: c.hostNetwork,
Volumes: append(
opspec.DaemonVolumes(c.DataPathMap, c.instanceName()),
c.mimeTypesVolume(),
),
HostNetwork: c.hostNetwork,
}
if c.hostNetwork {
podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
@@ -125,10 +123,16 @@ func (c *config) makeRGWPodSpec() v1.PodTemplateSpec {
// Set the ssl cert if specified
if c.store.Spec.Gateway.SSLCertificateRef != "" {
certVol := v1.Volume{Name: certVolumeName, VolumeSource: v1.VolumeSource{Secret: &v1.SecretVolumeSource{
SecretName: c.store.Spec.Gateway.SSLCertificateRef,
Items: []v1.KeyToPath{{Key: certKeyName, Path: certFilename}},
}}}
// Keep the SSL secret as secure as possible in the container. Give only user read perms.
userReadOnly := int32(0400)
certVol := v1.Volume{
Name: certVolumeName,
VolumeSource: v1.VolumeSource{
Secret: &v1.SecretVolumeSource{
SecretName: c.store.Spec.Gateway.SSLCertificateRef,
Items: []v1.KeyToPath{
{Key: certKeyName, Path: certFilename, Mode: &userReadOnly},
}}}}
podSpec.Volumes = append(podSpec.Volumes, certVol)
}
@@ -144,40 +148,7 @@ func (c *config) makeRGWPodSpec() v1.PodTemplateSpec {
}
}
func (c *config) makeConfigInitContainer() v1.Container {
container := v1.Container{
Name: opspec.ConfigInitContainerName,
Image: k8sutil.MakeRookImage(c.rookVersion),
Args: []string{
"ceph",
"rgw",
fmt.Sprintf("--config-dir=%s", k8sutil.DataDir),
fmt.Sprintf("--rgw-name=%s", c.store.Name),
fmt.Sprintf("--rgw-port=%d", c.store.Spec.Gateway.Port),
fmt.Sprintf("--rgw-secure-port=%d", c.store.Spec.Gateway.SecurePort),
},
VolumeMounts: opspec.RookVolumeMounts(),
Env: []v1.EnvVar{
{Name: "ROOK_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: c.instanceName()}, Key: keyringName}}},
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
opmon.ClusterNameEnvVar(c.store.Namespace),
opmon.EndpointEnvVar(),
opmon.SecretEnvVar(),
k8sutil.ConfigOverrideEnvVar(),
},
Resources: c.store.Spec.Gateway.Resources,
}
if c.store.Spec.Gateway.SSLCertificateRef != "" {
path := path.Join(certMountPath, certFilename)
container.Args = append(container.Args, fmt.Sprintf("--rgw-cert=%s", path))
}
return container
}
func (c *config) makeDaemonContainer() v1.Container {
func (c *clusterConfig) makeDaemonContainer() v1.Container {
// start the rgw daemon in the foreground
container := v1.Container{
@@ -186,26 +157,33 @@ func (c *config) makeDaemonContainer() v1.Container {
Command: []string{
"radosgw",
},
Args: []string{
"--foreground",
"--name=client.radosgw.gateway",
fmt.Sprintf("--rgw-mime-types-file=%s", rgwdaemon.GetMimeTypesPath(k8sutil.DataDir)),
},
VolumeMounts: opspec.CephVolumeMounts(),
Env: k8sutil.ClusterDaemonEnvVars(c.cephVersion.Image),
Resources: c.store.Spec.Gateway.Resources,
Args: append(
append(
opspec.DaemonFlags(c.clusterInfo, cephconfig.RgwType, c.store.Name),
"--foreground",
"--name=client.radosgw.gateway",
cephconfig.NewFlag("host", opspec.ContainerEnvVarReference("POD_NAME")),
cephconfig.NewFlag("rgw-mime-types-file", mimeTypesMountPath()),
), c.defaultSettings().GlobalFlags()..., // use default settings as flags until mon kv store supported
),
VolumeMounts: append(
opspec.DaemonVolumeMounts(c.DataPathMap, c.instanceName()),
c.mimeTypesVolumeMount(),
),
Env: opspec.DaemonEnvVars(c.cephVersion.Image),
Resources: c.store.Spec.Gateway.Resources,
}
if c.store.Spec.Gateway.SSLCertificateRef != "" {
// Add a volume mount for the ssl certificate
mount := v1.VolumeMount{Name: certVolumeName, MountPath: certMountPath, ReadOnly: true}
mount := v1.VolumeMount{Name: certVolumeName, MountPath: certDir, ReadOnly: true}
container.VolumeMounts = append(container.VolumeMounts, mount)
}
return container
}
func (c *config) startService() (string, error) {
func (c *clusterConfig) startService() (string, error) {
labels := c.getLabels()
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
@@ -253,10 +231,8 @@ func addPort(service *v1.Service, name string, port int32) {
})
}
func (c *config) getLabels() map[string]string {
return map[string]string{
k8sutil.AppAttr: appName,
k8sutil.ClusterAttr: c.store.Namespace,
"rook_object_store": c.store.Name,
}
func (c *clusterConfig) getLabels() map[string]string {
labels := opspec.PodLabels(AppName, c.store.Namespace, "rgw", c.store.Name)
labels["rook_object_store"] = c.store.Name
return labels
}
+41 -62
View File
@@ -17,13 +17,13 @@ limitations under the License.
package object
import (
"fmt"
"testing"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
"github.com/rook/rook/pkg/operator/k8sutil"
cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
cephtest "github.com/rook/rook/pkg/operator/ceph/test"
testop "github.com/rook/rook/pkg/operator/test"
exectest "github.com/rook/rook/pkg/util/exec/test"
"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
@@ -40,80 +40,59 @@ func TestPodSpecs(t *testing.T) {
v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI),
},
}
info := testop.CreateConfigDir(1)
data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "default")
c := &clusterConfig{
clusterInfo: info,
store: store,
rookVersion: "rook/rook:myversion",
cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.1", Name: "mimic"},
hostNetwork: true,
DataPathMap: data,
}
c := &config{store: store, rookVersion: "rook/rook:myversion", cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.1", Name: "mimic"}, hostNetwork: true}
s := c.makeRGWPodSpec()
assert.NotNil(t, s)
//assert.Equal(t, instanceName(store), s.Name)
assert.Equal(t, v1.RestartPolicyAlways, s.Spec.RestartPolicy)
assert.Equal(t, 3, len(s.Spec.Volumes))
assert.Equal(t, "rook-data", s.Spec.Volumes[0].Name)
assert.Equal(t, cephconfig.DefaultConfigMountName, s.Spec.Volumes[1].Name)
assert.Equal(t, c.instanceName(), s.ObjectMeta.Name)
assert.Equal(t, appName, s.ObjectMeta.Labels["app"])
assert.Equal(t, store.Namespace, s.ObjectMeta.Labels["rook_cluster"])
assert.Equal(t, store.Name, s.ObjectMeta.Labels["rook_object_store"])
assert.Equal(t, 0, len(s.ObjectMeta.Annotations))
cont := s.Spec.InitContainers[0]
assert.Equal(t, 1, len(s.Spec.InitContainers))
assert.Equal(t, "rook/rook:myversion", cont.Image)
assert.Equal(t, 3, len(cont.VolumeMounts))
assert.Equal(t, 0, len(cont.Command))
assert.Equal(t, 6, len(cont.Args))
assert.Equal(t, "ceph", cont.Args[0])
assert.Equal(t, "rgw", cont.Args[1])
assert.Equal(t, "--config-dir=/var/lib/rook", cont.Args[2])
assert.Equal(t, "--rgw-name=default", cont.Args[3])
assert.Equal(t, "--rgw-port=123", cont.Args[4])
assert.Equal(t, "--rgw-secure-port=0", cont.Args[5])
cont = s.Spec.Containers[0]
assert.Equal(t, "ceph/ceph:v13.2.1", cont.Image)
assert.Equal(t, 2, len(cont.VolumeMounts))
assert.Equal(t, 1, len(cont.Command))
assert.Equal(t, "radosgw", cont.Command[0])
assert.Equal(t, 3, len(cont.Args))
assert.Equal(t, "--foreground", cont.Args[0])
assert.Equal(t, "--name=client.radosgw.gateway", cont.Args[1])
assert.Equal(t, "--rgw-mime-types-file=/var/lib/rook/rgw/mime.types", cont.Args[2])
assert.Equal(t, len(k8sutil.ClusterDaemonEnvVars("ceph/ceph:v14")), len(cont.Env))
assert.Equal(t, "100", cont.Resources.Limits.Cpu().String())
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
podTemplate := cephtest.NewPodTemplateSpecTester(t, &s)
podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion",
"100", "1337" /* resources */)
}
func TestSSLPodSpec(t *testing.T) {
store := simpleStore()
store.Spec.Gateway.Resources = v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: *resource.NewQuantity(100.0, resource.BinarySI),
},
Requests: v1.ResourceList{
v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI),
},
}
info := testop.CreateConfigDir(1)
data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "default")
store.Spec.Gateway.SSLCertificateRef = "mycert"
store.Spec.Gateway.SecurePort = 443
c := &config{store: store, rookVersion: "v1.0", hostNetwork: true}
c := &clusterConfig{
clusterInfo: info,
store: store,
rookVersion: "rook/rook:myversion",
cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.1", Name: "mimic"},
hostNetwork: true,
DataPathMap: data,
}
c.hostNetwork = true
s := c.makeRGWPodSpec()
assert.NotNil(t, s)
assert.Equal(t, c.instanceName(), s.Name)
assert.Equal(t, 4, len(s.Spec.Volumes))
assert.Equal(t, certVolumeName, s.Spec.Volumes[3].Name)
podTemplate := cephtest.NewPodTemplateSpecTester(t, &s)
podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion",
"100", "1337" /* resources */)
assert.True(t, s.Spec.HostNetwork)
assert.Equal(t, v1.DNSClusterFirstWithHostNet, s.Spec.DNSPolicy)
initCont := s.Spec.InitContainers[0]
assert.Equal(t, 3, len(initCont.VolumeMounts))
assert.Equal(t, 7, len(initCont.Args))
assert.Equal(t, fmt.Sprintf("--rgw-secure-port=%d", 443), initCont.Args[5])
assert.Equal(t, fmt.Sprintf("--rgw-cert=%s/%s", certMountPath, certFilename), initCont.Args[6])
rgwCont := s.Spec.Containers[0]
assert.Equal(t, 3, len(rgwCont.VolumeMounts))
assert.Equal(t, certVolumeName, rgwCont.VolumeMounts[2].Name)
assert.Equal(t, certMountPath, rgwCont.VolumeMounts[2].MountPath)
assert.Equal(t, 3, len(rgwCont.Args))
}
func TestValidateSpec(t *testing.T) {
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package rgw
package object
import (
"encoding/json"
@@ -24,13 +24,14 @@ import (
)
const (
RGWErrorNone = iota
RGWErrorUnknown = iota
RGWErrorNotFound = iota
RGWErrorBadData = iota
RGWErrorParse = iota
RGWErrorNone = iota
RGWErrorUnknown
RGWErrorNotFound
RGWErrorBadData
RGWErrorParse
)
// An ObjectUser defines the details of an object store user.
type ObjectUser struct {
UserID string `json:"userId"`
DisplayName *string `json:"displayName"`
@@ -39,6 +40,7 @@ type ObjectUser struct {
SecretKey *string `json:"secretKey"`
}
// ListUsers lists the object pool users.
func ListUsers(c *Context) ([]string, int, error) {
result, err := runAdminCommand(c, "user", "list")
if err != nil {
@@ -80,6 +82,7 @@ func decodeUser(data string) (*ObjectUser, int, error) {
return &rookUser, RGWErrorNone, nil
}
// GetUser returns the user with the given ID.
func GetUser(c *Context, id string) (*ObjectUser, int, error) {
logger.Infof("Getting user: %s", id)
@@ -93,6 +96,7 @@ func GetUser(c *Context, id string) (*ObjectUser, int, error) {
return decodeUser(result)
}
// CreateUser creates a new user with the information given.
func CreateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) {
logger.Infof("Creating user: %s", user.UserID)
@@ -131,6 +135,7 @@ func CreateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) {
return decodeUser(result)
}
// UpdateUser updates the user whose ID matches the user.
func UpdateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) {
logger.Infof("Updating user: %s", user.UserID)
@@ -155,6 +160,7 @@ func UpdateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) {
return decodeUser(body)
}
// DeleteUser deletes the user with the given ID.
func DeleteUser(c *Context, id string) (string, int, error) {
logger.Infof("Deleting user: %s", id)
result, err := runAdminCommand(c, "user", "rm", "--uid", id)
+10 -10
View File
@@ -25,7 +25,7 @@ import (
opkit "github.com/rook/operator-kit"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
cephrgw "github.com/rook/rook/pkg/daemon/ceph/rgw"
"github.com/rook/rook/pkg/operator/ceph/object"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/api/core/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
@@ -34,12 +34,12 @@ import (
)
const (
AppName = "rook-ceph-rgw"
appName = object.AppName
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-object")
// ObjectStoreResource represents the object store user custom resource
// ObjectStoreUserResource represents the object store user custom resource
var ObjectStoreUserResource = opkit.CustomResource{
Name: "cephobjectstoreuser",
Plural: "cephobjectstoreusers",
@@ -139,13 +139,13 @@ func (c *ObjectStoreUserController) createUser(context *clusterd.Context, u *cep
// create the user
logger.Infof("creating user %s in namespace %s", u.Name, u.Namespace)
userConfig := cephrgw.ObjectUser{
userConfig := object.ObjectUser{
UserID: u.Name,
DisplayName: &displayName,
}
objContext := cephrgw.NewContext(context, u.Spec.Store, u.Namespace)
objContext := object.NewContext(context, u.Spec.Store, u.Namespace)
user, rgwerr, err := cephrgw.CreateUser(objContext, userConfig)
user, rgwerr, err := object.CreateUser(objContext, userConfig)
if err != nil {
return fmt.Errorf("failed to create user %s. RadosGW returned error %d: %+v", u.Name, rgwerr, err)
}
@@ -160,7 +160,7 @@ func (c *ObjectStoreUserController) createUser(context *clusterd.Context, u *cep
Name: fmt.Sprintf("rook-ceph-object-user-%s-%s", u.Spec.Store, u.Name),
Namespace: u.Namespace,
Labels: map[string]string{
"app": AppName,
"app": appName,
"user": u.Name,
"rook_cluster": u.Namespace,
"rook_object_store": u.Spec.Store,
@@ -181,8 +181,8 @@ func (c *ObjectStoreUserController) createUser(context *clusterd.Context, u *cep
// Delete the user
func deleteUser(context *clusterd.Context, u *cephv1.CephObjectStoreUser) error {
objContext := cephrgw.NewContext(context, u.Spec.Store, u.Namespace)
_, rgwerr, err := cephrgw.DeleteUser(objContext, u.Name)
objContext := object.NewContext(context, u.Spec.Store, u.Namespace)
_, rgwerr, err := object.DeleteUser(objContext, u.Name)
if err != nil {
if rgwerr == 3 {
logger.Infof("user %s does not exist in store %s", u.Name, u.Spec.Store)
@@ -200,7 +200,7 @@ func deleteUser(context *clusterd.Context, u *cephv1.CephObjectStoreUser) error
return nil
}
// Validate the user arguments
// ValidateUser validates the user arguments
func ValidateUser(context *clusterd.Context, u *cephv1.CephObjectStoreUser) error {
if u.Name == "" {
return fmt.Errorf("missing name")
+2 -2
View File
@@ -95,9 +95,9 @@ func DaemonVolumeMounts(dataPaths *config.DataPathMap, keyringResourceName strin
func DaemonFlags(cluster *cephconfig.ClusterInfo, daemonType config.DaemonType, daemonID string) []string {
return append(
config.DefaultFlags(cluster.FSID, keyring.VolumeMount().KeyringFilePath()),
// all daemons are named in the format <type>.<id>
config.NewFlag("name", fmt.Sprintf("%s.%s", string(daemonType), daemonID)),
config.NewFlag("id", daemonID),
)
}
// AdminFlags returns the command line flags used for Ceph commands requiring admin authentication.
+2 -1
View File
@@ -18,7 +18,8 @@ package clients
import (
"fmt"
"github.com/rook/rook/pkg/daemon/ceph/rgw"
rgw "github.com/rook/rook/pkg/operator/ceph/object"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
)
+6 -5
View File
@@ -19,13 +19,14 @@ package integration
import (
"errors"
rgwdaemon "github.com/rook/rook/pkg/daemon/ceph/rgw"
"time"
rgw "github.com/rook/rook/pkg/operator/ceph/object"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"time"
)
var (
@@ -167,17 +168,17 @@ func objectTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, na
helper.ObjectClient.DeleteUser(storeName, userid)*/
}
func getBucket(bucketname string, bucketList []rgwdaemon.ObjectBucket) (rgwdaemon.ObjectBucket, error) {
func getBucket(bucketname string, bucketList []rgw.ObjectBucket) (rgw.ObjectBucket, error) {
for _, bucket := range bucketList {
if bucket.Name == bucketname {
return bucket, nil
}
}
return rgwdaemon.ObjectBucket{}, errors.New("Bucket not found")
return rgw.ObjectBucket{}, errors.New("Bucket not found")
}
func getBucketSizeAndObjects(bucketname string, bucketList []rgwdaemon.ObjectBucket) (uint64, uint64, error) {
func getBucketSizeAndObjects(bucketname string, bucketList []rgw.ObjectBucket) (uint64, uint64, error) {
bkt, err := getBucket(bucketname, bucketList)
if err != nil {
return 0, 0, errors.New("Bucket not found")