From 086fa8231c78c7bdeca8a1440557d72061b9fe14 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Thu, 14 Feb 2019 09:16:21 -0700 Subject: [PATCH] rgw: configure entirely in operator Configure the Ceph rgw daemon completely from the operator a la the recent changes to the Ceph mon, mgr, and mds operators. Create the rgw deployment or daemonset first, and then create the keyring secret for the object store with its owner reference as the corresponding deployment or daemonset. When the replication controller is deleted, the secret is also deleted. The RGW's mime.types file is now stored in a configmap with a different file created for each object store. This is primarily just a means to get the mime.types file into the rgw pod, but the added benefit is that the administrator can modify the configmap, which could reduce susceptibility to file type execution vulnerabilities (worst case). Signed-off-by: Blaine Gardner --- Documentation/ceph-object-store-crd.md | 15 ++ PendingReleaseNotes.md | 7 +- cmd/rook/ceph/ceph.go | 1 - cmd/rook/ceph/rgw.go | 93 ----------- pkg/daemon/ceph/rgw/init.go | 147 ------------------ pkg/daemon/ceph/rgw/init_test.go | 67 -------- pkg/operator/ceph/cluster/controller.go | 2 +- pkg/operator/ceph/config/store.go | 2 +- .../rgw => operator/ceph/object}/admin.go | 4 +- .../rgw => operator/ceph/object}/bucket.go | 2 +- pkg/operator/ceph/object/config.go | 100 ++++++++++++ pkg/operator/ceph/object/config_test.go | 66 ++++++++ pkg/operator/ceph/object/controller.go | 38 ++++- pkg/operator/ceph/object/controller_test.go | 3 +- .../ceph/rgw => operator/ceph/object}/mime.go | 62 +++++++- .../ceph/object}/objectstore.go | 20 +-- .../ceph/object}/objectstore_test.go | 7 +- pkg/operator/ceph/object/rgw.go | 139 +++++++---------- pkg/operator/ceph/object/rgw_test.go | 16 +- pkg/operator/ceph/object/spec.go | 134 +++++++--------- pkg/operator/ceph/object/spec_test.go | 103 +++++------- .../ceph/rgw => operator/ceph/object}/user.go | 18 ++- pkg/operator/ceph/object/user/controller.go | 20 +-- pkg/operator/ceph/spec/spec.go | 4 +- tests/framework/clients/object_user.go | 3 +- tests/integration/base_object_test.go | 11 +- 26 files changed, 492 insertions(+), 592 deletions(-) delete mode 100644 cmd/rook/ceph/rgw.go delete mode 100644 pkg/daemon/ceph/rgw/init.go delete mode 100644 pkg/daemon/ceph/rgw/init_test.go rename pkg/{daemon/ceph/rgw => operator/ceph/object}/admin.go (93%) rename pkg/{daemon/ceph/rgw => operator/ceph/object}/bucket.go (99%) create mode 100644 pkg/operator/ceph/object/config.go create mode 100644 pkg/operator/ceph/object/config_test.go rename pkg/{daemon/ceph/rgw => operator/ceph/object}/mime.go (91%) rename pkg/{daemon/ceph/rgw => operator/ceph/object}/objectstore.go (94%) rename pkg/{daemon/ceph/rgw => operator/ceph/object}/objectstore_test.go (98%) rename pkg/{daemon/ceph/rgw => operator/ceph/object}/user.go (91%) diff --git a/Documentation/ceph-object-store-crd.md b/Documentation/ceph-object-store-crd.md index aac5239f9..4f26ae6b0 100644 --- a/Documentation/ceph-object-store-crd.md +++ b/Documentation/ceph-object-store-crd.md @@ -86,3 +86,18 @@ The gateway settings correspond to the RGW daemon settings. - `allNodes`: Whether RGW pods should be started on all nodes. If true, a daemonset is created. If false, `instances` must be set. - `placement`: The Kubernetes placement settings to determine where the RGW pods should be started in the cluster. - `resources`: Set resource requests/limits for the Gateway Pod(s), see [Resource Requirements/Limits](ceph-cluster-crd.md#resource-requirementslimits). + +## Runtime settings + +### MIME types + +Rook provides a default `mime.types` file for each Ceph object store. This file is stored in a +Kubernetes ConfigMap with the name `rook-ceph-rgw--mime-types`. For most users, the +default file should suffice, however, the option is available to users to edit the `mime.types` +file in the ConfigMap as they desire. Users may have their own special file types, and particularly +security conscious users may wish to pare down the file to reduce the possibility of a file type +execution attack. + +Rook will not overwrite an existing `mime.types` ConfigMap so that user modifications will not be +destroyed. If the object store is destroyed and recreated, the ConfigMap will also be destroyed and +created anew. diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index bb4fdc703..a46420db1 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -29,9 +29,10 @@ - Rook will no longer create a directory-based osd in the `dataDirHostPath` if no directories or devices are specified or if there are no disks on the host. -- Containers in `mon`, `mgr`, and `mds` pods have been removed and/or changed names. -- Config paths in `mon`, `mgr`, and `mds` containers are now always the Ceph default paths - (`/etc/ceph`, `/var/lib/ceph/...`) regardless of the `dataDirHostPath` setting. +- Containers in `mon`, `mgr`, `mds`, and `rgw` pods have been removed and/or changed names. +- Config paths in `mon`, `mgr`, `mds` and `rgw` containers are now always under + `/etc/ceph` or `/var/lib/ceph` and as close to Ceph's default path as possible regardless of the + `dataDirHostPath` setting. ## Known Issues diff --git a/cmd/rook/ceph/ceph.go b/cmd/rook/ceph/ceph.go index 70e0f7df3..1e71da6ad 100644 --- a/cmd/rook/ceph/ceph.go +++ b/cmd/rook/ceph/ceph.go @@ -58,7 +58,6 @@ func init() { Cmd.AddCommand(operatorCmd, agentCmd, osdCmd, - rgwCmd, configCmd, nfsCmd) } diff --git a/cmd/rook/ceph/rgw.go b/cmd/rook/ceph/rgw.go deleted file mode 100644 index 871a73578..000000000 --- a/cmd/rook/ceph/rgw.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ceph - -import ( - "fmt" - "os" - - "github.com/rook/rook/cmd/rook/rook" - rgwdaemon "github.com/rook/rook/pkg/daemon/ceph/rgw" - "github.com/rook/rook/pkg/operator/ceph/cluster/mon" - "github.com/rook/rook/pkg/util/flags" - "github.com/spf13/cobra" -) - -var rgwCmd = &cobra.Command{ - Use: "rgw", - Short: "Generates rgw config and runs the rgw daemon", -} - -var ( - rgwName string - rgwKeyring string - rgwHost string - rgwCert string - rgwPort int - rgwSecurePort int -) - -func init() { - rgwCmd.Flags().StringVar(&rgwName, "rgw-name", "", "name of the object store") - rgwCmd.Flags().StringVar(&rgwKeyring, "rgw-keyring", "", "the rgw keyring") - rgwCmd.Flags().StringVar(&rgwHost, "rgw-host", os.Getenv("HOSTNAME"), "RGW host name. Becomes the only accepted hostname if the rgw dns name property is unset. Defaults to the pod hostname") - rgwCmd.Flags().StringVar(&rgwCert, "rgw-cert", "", "path to the ssl certificate in pem format") - rgwCmd.Flags().IntVar(&rgwPort, "rgw-port", 0, "rgw port (http)") - rgwCmd.Flags().IntVar(&rgwSecurePort, "rgw-secure-port", 0, "rgw secure port number (https)") - addCephFlags(rgwCmd) - - flags.SetFlagsFromEnv(rgwCmd.Flags(), rook.RookEnvVarPrefix) - - rgwCmd.RunE = initRGW -} - -func initRGW(cmd *cobra.Command, args []string) error { - required := []string{"mon-endpoints", "cluster-name", "rgw-name", "rgw-keyring"} - if err := flags.VerifyRequiredFlags(rgwCmd, required); err != nil { - return err - } - - if err := verifyRenamedFlags(rgwCmd); err != nil { - return err - } - - if rgwPort == 0 && rgwSecurePort == 0 { - return fmt.Errorf("port or secure port are required") - } - - rook.SetLogLevel() - - rook.LogStartupInfo(rgwCmd.Flags()) - - clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints) - config := &rgwdaemon.Config{ - ClusterInfo: &clusterInfo, - Name: rgwName, - Keyring: rgwKeyring, - Host: rgwHost, - Port: rgwPort, - SecurePort: rgwSecurePort, - CertificatePath: rgwCert, - } - - err := rgwdaemon.Initialize(createContext(), config) - if err != nil { - rook.TerminateFatal(err) - } - - return nil -} diff --git a/pkg/daemon/ceph/rgw/init.go b/pkg/daemon/ceph/rgw/init.go deleted file mode 100644 index 64bd943af..000000000 --- a/pkg/daemon/ceph/rgw/init.go +++ /dev/null @@ -1,147 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rgw - -import ( - "fmt" - "io/ioutil" - "os" - "path" - "strconv" - - "github.com/coreos/pkg/capnslog" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - "github.com/rook/rook/pkg/clusterd" - cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" - "github.com/rook/rook/pkg/util" -) - -var ( - logger = capnslog.NewPackageLogger("github.com/rook/rook", "cephrgw") - rgwFrontendName = "civetweb" -) - -const ( - keyringTemplate = `[client.radosgw.gateway] -key = %s -caps mon = "allow rw" -caps osd = "allow *" -` -) - -type Config struct { - Name string - Host string - Port int - SecurePort int - Keyring string - CertificatePath string - ClusterInfo *cephconfig.ClusterInfo -} - -func Initialize(context *clusterd.Context, config *Config) error { - - err := generateConfigFiles(context, config) - if err != nil { - return fmt.Errorf("failed to generate rgw config files. %+v", err) - } - - return err -} - -func portString(config *Config) string { - - var portString string - if config.Port != 0 { - portString = strconv.Itoa(config.Port) - } - if config.SecurePort != 0 && config.CertificatePath != "" { - var separator string - if config.Port != 0 { - separator = "+" - } - // the suffix is intended to be appended to the end of the rgw_frontends arg, immediately after the port. - // with ssl enabled, the port number must end with the letter s. - portString = fmt.Sprintf("%s%s%ds ssl_certificate=%s", portString, separator, config.SecurePort, config.CertificatePath) - } - - return portString -} - -func rgwFrontend(config *Config) string { - if cephv1.VersionAtLeast(config.ClusterInfo.CephVersionName, cephv1.Nautilus) { - rgwFrontendName = "beast" - } - return rgwFrontendName -} - -func generateConfigFiles(context *clusterd.Context, config *Config) error { - - // create the rgw data directory - dataDir := path.Join(getRGWConfDir(context.ConfigDir), "data") - if err := os.MkdirAll(dataDir, 0744); err != nil { - logger.Warningf("failed to create data directory %s: %+v", dataDir, err) - } - - settings := map[string]string{ - "host": config.Host, - "rgw data": dataDir, - "rgw log nonexistent bucket": "true", - "rgw intent log object name utc": "true", - "rgw enable usage log": "true", - "rgw_frontends": fmt.Sprintf("%s port=%s", rgwFrontend(config), portString(config)), - "rgw_zone": config.Name, - "rgw_zonegroup": config.Name, - } - configFile, err := cephconfig.GenerateConfigFile(context, config.ClusterInfo, getRGWConfDir(context.ConfigDir), - "client.radosgw.gateway", getRGWKeyringPath(context.ConfigDir), nil, settings) - if err != nil { - return fmt.Errorf("failed to create config file. %+v", err) - } - util.WriteFileToLog(logger, configFile) - - keyringEval := func(key string) string { - return fmt.Sprintf(keyringTemplate, key) - } - - // create rgw config - err = cephconfig.WriteKeyring(getRGWKeyringPath(context.ConfigDir), config.Keyring, keyringEval) - if err != nil { - return fmt.Errorf("failed to save keyring. %+v", err) - } - - // write the mime types config - mimeTypesPath := GetMimeTypesPath(context.ConfigDir) - logger.Infof("Writing mime types to: %s", mimeTypesPath) - if err := ioutil.WriteFile(mimeTypesPath, []byte(mimeTypes), 0644); err != nil { - return fmt.Errorf("failed to write mime types to %s: %+v", mimeTypesPath, err) - } - - return nil -} - -func getRGWConfDir(configDir string) string { - return path.Join(configDir, "rgw") -} - -func getRGWKeyringPath(configDir string) string { - return path.Join(getRGWConfDir(configDir), "keyring") -} - -func GetMimeTypesPath(configDir string) string { - return path.Join(getRGWConfDir(configDir), "mime.types") -} diff --git a/pkg/daemon/ceph/rgw/init_test.go b/pkg/daemon/ceph/rgw/init_test.go deleted file mode 100644 index 5135f2279..000000000 --- a/pkg/daemon/ceph/rgw/init_test.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rgw - -import ( - "testing" - - cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" - "github.com/stretchr/testify/assert" -) - -func TestPortString(t *testing.T) { - // No port or secure port - cfg := &Config{} - result := portString(cfg) - assert.Equal(t, "", result) - - // Insecure port - cfg = &Config{Port: 80} - result = portString(cfg) - assert.Equal(t, "80", result) - - // Secure port - cfg = &Config{SecurePort: 443, CertificatePath: "/etc/rgw/cert.pem"} - result = portString(cfg) - assert.Equal(t, "443s ssl_certificate=/etc/rgw/cert.pem", result) - - // Both ports - cfg = &Config{Port: 80, SecurePort: 443, CertificatePath: "/etc/rgw/cert.pem"} - result = portString(cfg) - assert.Equal(t, "80+443s ssl_certificate=/etc/rgw/cert.pem", result) - - // Secure port requires the cert - cfg = &Config{SecurePort: 443} - result = portString(cfg) - assert.Equal(t, "", result) -} - -func TestFrontend(t *testing.T) { - cfg := &Config{} - cfg.ClusterInfo = &cephconfig.ClusterInfo{CephVersionName: "mimic"} - - result := rgwFrontend(cfg) - assert.Equal(t, "civetweb", result) - - cfg.ClusterInfo.CephVersionName = "nautilus" - result = rgwFrontend(cfg) - assert.Equal(t, "beast", result) - - cfg.ClusterInfo.CephVersionName = "octopus" - result = rgwFrontend(cfg) - assert.Equal(t, "beast", result) -} diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 40271f06e..2f13bae93 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -229,7 +229,7 @@ func (c *ClusterController) onAdd(obj interface{}) { poolController.StartWatch(cluster.Namespace, cluster.stopCh) // Start object store CRD watcher - objectStoreController := object.NewObjectStoreController(c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef) + objectStoreController := object.NewObjectStoreController(cluster.Info, c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef) objectStoreController.StartWatch(cluster.Namespace, cluster.stopCh) // Start object store user CRD watcher diff --git a/pkg/operator/ceph/config/store.go b/pkg/operator/ceph/config/store.go index d78a63462..47651f51a 100644 --- a/pkg/operator/ceph/config/store.go +++ b/pkg/operator/ceph/config/store.go @@ -161,7 +161,7 @@ func (s *Store) createOrUpdateMonHostSecrets(clusterInfo *cephconfig.ClusterInfo monIP := cephutil.GetIPFromEndpoint(m.Endpoint) // This tries to detect the current port if the mon already exists - // This basically handles the transtion between monitors running on 6790 to msgr2 + // This basically handles the transition between monitors running on 6790 to msgr2 // So whatever the previous monitor port was we keep it currentMonPort := cephutil.GetPortFromEndpoint(m.Endpoint) diff --git a/pkg/daemon/ceph/rgw/admin.go b/pkg/operator/ceph/object/admin.go similarity index 93% rename from pkg/daemon/ceph/rgw/admin.go rename to pkg/operator/ceph/object/admin.go index 5028b4490..03ab593c3 100644 --- a/pkg/daemon/ceph/rgw/admin.go +++ b/pkg/operator/ceph/object/admin.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package rgw +package object import ( "fmt" @@ -23,12 +23,14 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" ) +// Context holds the context for the object store. type Context struct { context *clusterd.Context Name string ClusterName string } +// NewContext creates a new object store context. func NewContext(context *clusterd.Context, name, clusterName string) *Context { return &Context{context: context, Name: name, ClusterName: clusterName} } diff --git a/pkg/daemon/ceph/rgw/bucket.go b/pkg/operator/ceph/object/bucket.go similarity index 99% rename from pkg/daemon/ceph/rgw/bucket.go rename to pkg/operator/ceph/object/bucket.go index 7cea9b83a..3151f284b 100644 --- a/pkg/daemon/ceph/rgw/bucket.go +++ b/pkg/operator/ceph/object/bucket.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package rgw +package object import ( "encoding/json" diff --git a/pkg/operator/ceph/object/config.go b/pkg/operator/ceph/object/config.go new file mode 100644 index 000000000..de5b8b1f0 --- /dev/null +++ b/pkg/operator/ceph/object/config.go @@ -0,0 +1,100 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package object + +import ( + "fmt" + "path" + "strconv" + + cephconfig "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/config/keyring" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + keyringTemplate = ` +[client.radosgw.gateway] +key = %s +caps mon = "allow rw" +caps osd = "allow rwx" +` + + certVolumeName = "rook-ceph-rgw-cert" + certDir = "/etc/ceph/private" + certKeyName = "cert" + certFilename = "rgw-cert.pem" +) + +// TODO: these should be set in the mon's central kv store for mimic+ +func (c *clusterConfig) defaultSettings() *cephconfig.Config { + s := cephconfig.NewConfig() + s.Section("global"). + Set("rgw log nonexistent bucket", "true"). + Set("rgw intent log object name utc", "true"). + Set("rgw enable usage log", "true"). + Set("rgw frontends", fmt.Sprintf("civetweb port=%s", c.portString())). + Set("rgw zone", c.store.Name). + Set("rgw zonegroup", c.store.Name) + return s +} + +func (c *clusterConfig) portString() string { + var portString string + port := c.store.Spec.Gateway.Port + if port != 0 { + portString = strconv.Itoa(int(port)) + } + if c.store.Spec.Gateway.SecurePort != 0 && c.store.Spec.Gateway.SSLCertificateRef != "" { + var separator string + if port != 0 { + separator = "+" + } + certPath := path.Join(certDir, certFilename) + // with ssl enabled, the port number must end with the letter s. + // e.g., "443s ssl_certificate=/etc/ceph/private/keyandcert.pem" + portString = fmt.Sprintf("%s%s%ds ssl_certificate=%s", + portString, separator, c.store.Spec.Gateway.SecurePort, certPath) + } + return portString +} + +func (c *clusterConfig) generateKeyring(replicationControllerOwnerRef *metav1.OwnerReference) error { + user := "client.radosgw.gateway" + /* TODO: this says `osd allow rwx` while template says `osd allow *`; which is correct? */ + access := []string{"osd", "allow rwx", "mon", "allow rw"} + s := keyring.GetSecretStore(c.context, c.store.Namespace, replicationControllerOwnerRef) + + key, err := s.GenerateKey(c.instanceName(), user, access) + if err != nil { + return err + } + + // Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x + err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Delete(c.instanceName(), &metav1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + logger.Debugf("legacy rgw key %s is already removed", c.instanceName()) + } else { + logger.Warningf("legacy rgw key %s could not be removed. %+v", c.instanceName(), err) + } + } + + keyring := fmt.Sprintf(keyringTemplate, key) + return s.CreateOrUpdate(c.instanceName(), keyring) +} diff --git a/pkg/operator/ceph/object/config_test.go b/pkg/operator/ceph/object/config_test.go new file mode 100644 index 000000000..086566bcb --- /dev/null +++ b/pkg/operator/ceph/object/config_test.go @@ -0,0 +1,66 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package object + +import ( + "testing" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/stretchr/testify/assert" +) + +func newConfig() *clusterConfig { + return &clusterConfig{ + store: cephv1.CephObjectStore{ + Spec: cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{}, + }}} +} + +func TestPortString(t *testing.T) { + // No port or secure port + cfg := newConfig() + result := cfg.portString() + assert.Equal(t, "", result) + + // Insecure port + cfg = newConfig() + cfg.store.Spec.Gateway.Port = 80 + result = cfg.portString() + assert.Equal(t, "80", result) + + // Secure port + cfg = newConfig() + cfg.store.Spec.Gateway.SecurePort = 443 + cfg.store.Spec.Gateway.SSLCertificateRef = "some-k8s-key-secret" + result = cfg.portString() + assert.Equal(t, "443s ssl_certificate=/etc/ceph/private/rgw-cert.pem", result) + + // Both ports + cfg = newConfig() + cfg.store.Spec.Gateway.Port = 80 + cfg.store.Spec.Gateway.SecurePort = 443 + cfg.store.Spec.Gateway.SSLCertificateRef = "some-k8s-key-secret" + result = cfg.portString() + assert.Equal(t, "80+443s ssl_certificate=/etc/ceph/private/rgw-cert.pem", result) + + // Secure port requires the cert + cfg = newConfig() + cfg.store.Spec.Gateway.SecurePort = 443 + result = cfg.portString() + assert.Equal(t, "", result) +} diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index 22aed6c1c..4ba616709 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -14,7 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package rgw to manage a rook object store. package object import ( @@ -26,6 +25,8 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" cephbeta "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" "github.com/rook/rook/pkg/clusterd" + daemonconfig "github.com/rook/rook/pkg/daemon/ceph/config" + cephconfig "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/pool" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/api/errors" @@ -56,6 +57,7 @@ var ObjectStoreResourceRookLegacy = opkit.CustomResource{ // ObjectStoreController represents a controller object for object store custom resources type ObjectStoreController struct { + clusterInfo *daemonconfig.ClusterInfo context *clusterd.Context rookImage string cephVersion cephv1.CephVersionSpec @@ -64,8 +66,16 @@ type ObjectStoreController struct { } // NewObjectStoreController create controller for watching object store custom resources created -func NewObjectStoreController(context *clusterd.Context, rookImage string, cephVersion cephv1.CephVersionSpec, hostNetwork bool, ownerRef metav1.OwnerReference) *ObjectStoreController { +func NewObjectStoreController( + clusterInfo *daemonconfig.ClusterInfo, + context *clusterd.Context, + rookImage string, + cephVersion cephv1.CephVersionSpec, + hostNetwork bool, + ownerRef metav1.OwnerReference, +) *ObjectStoreController { return &ObjectStoreController{ + clusterInfo: clusterInfo, context: context, rookImage: rookImage, cephVersion: cephVersion, @@ -107,7 +117,16 @@ func (c *ObjectStoreController) onAdd(obj interface{}) { return } - cfg := config{c.context, *objectstore, c.rookImage, c.cephVersion, c.hostNetwork, c.storeOwners(objectstore)} + cfg := clusterConfig{ + clusterInfo: c.clusterInfo, + context: c.context, + store: *objectstore, + rookVersion: c.rookImage, + cephVersion: c.cephVersion, + hostNetwork: c.hostNetwork, + ownerRefs: c.storeOwners(objectstore), + DataPathMap: cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, objectstore.Name), + } if err = cfg.createStore(); err != nil { logger.Errorf("failed to create object store %s. %+v", objectstore.Name, err) } @@ -139,7 +158,16 @@ func (c *ObjectStoreController) onUpdate(oldObj, newObj interface{}) { } logger.Infof("applying object store %s changes", newStore.Name) - cfg := config{c.context, *newStore, c.rookImage, c.cephVersion, c.hostNetwork, c.storeOwners(newStore)} + cfg := clusterConfig{ + c.clusterInfo, + c.context, + *newStore, + c.rookImage, + c.cephVersion, + c.hostNetwork, + c.storeOwners(newStore), + cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, newStore.Name), + } if err = cfg.updateStore(); err != nil { logger.Errorf("failed to create (modify) object store %s. %+v", newStore.Name, err) } @@ -157,7 +185,7 @@ func (c *ObjectStoreController) onDelete(obj interface{}) { return } - cfg := config{context: c.context, store: *objectstore} + cfg := clusterConfig{context: c.context, store: *objectstore} if err = cfg.deleteStore(); err != nil { logger.Errorf("failed to delete object store %s. %+v", objectstore.Name, err) } diff --git a/pkg/operator/ceph/object/controller_test.go b/pkg/operator/ceph/object/controller_test.go index 235eed4dd..bdd0ba2fd 100644 --- a/pkg/operator/ceph/object/controller_test.go +++ b/pkg/operator/ceph/object/controller_test.go @@ -91,7 +91,8 @@ func TestMigrateObjectStoreObject(t *testing.T) { Clientset: clientset, RookClientset: rookfake.NewSimpleClientset(legacyObjectStore), } - controller := NewObjectStoreController(context, "", cephv1.CephVersionSpec{}, false, metav1.OwnerReference{}) + info := testop.CreateConfigDir(1) + controller := NewObjectStoreController(info, context, "", cephv1.CephVersionSpec{}, false, metav1.OwnerReference{}) // convert the legacy objectstore object in memory and assert that a migration is needed convertedObjectStore, migrationNeeded, err := getObjectStoreObject(legacyObjectStore) diff --git a/pkg/daemon/ceph/rgw/mime.go b/pkg/operator/ceph/object/mime.go similarity index 91% rename from pkg/daemon/ceph/rgw/mime.go rename to pkg/operator/ceph/object/mime.go index 28ce4120d..858c27a0d 100644 --- a/pkg/daemon/ceph/rgw/mime.go +++ b/pkg/operator/ceph/object/mime.go @@ -14,7 +14,67 @@ See the License for the specific language governing permissions and limitations under the License. */ -package rgw +package object + +import ( + "fmt" + "path" + + "github.com/rook/rook/pkg/operator/k8sutil" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // these control the location where the mime.types file is mounted in containers. + // Ceph's default is /etc/mime.types, but we can't mount a configmap to `/etc` without breaking + // the container. /etc/ceph/rgw won't overwrite ceph.conf (if it exists), and it will be easy to + // find manually by inspection of the `/etc/ceph` dir. + mimeTypesMountDir = "/etc/ceph/rgw" + mimeTypesFileName = "mime.types" +) + +func (c *clusterConfig) mimeTypesConfigMapName() string { + return fmt.Sprintf("%s-mime-types", c.instanceName()) +} + +func mimeTypesMountPath() string { + return path.Join(mimeTypesMountDir, mimeTypesFileName) +} + +// store mime.types file in a config map +func (c *clusterConfig) generateMimeTypes(ownerRef *metav1.OwnerReference) error { + k := k8sutil.NewConfigMapKVStore(c.store.Namespace, c.context.Clientset, *ownerRef) + if _, err := k.GetValue(c.mimeTypesConfigMapName(), mimeTypesFileName); err == nil || !errors.IsNotFound(err) { + logger.Infof("config map for object pool %s already exists, not overwriting", c.store.Name) + return nil + } + // is not found + if err := k.SetValue(c.mimeTypesConfigMapName(), mimeTypesFileName, mimeTypes); err != nil { + return fmt.Errorf("failed to create config map for object pool %s. %+v", c.store.Name, err) + } + return nil +} + +func (c *clusterConfig) mimeTypesVolume() v1.Volume { + return v1.Volume{ + Name: c.mimeTypesConfigMapName(), + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{LocalObjectReference: v1.LocalObjectReference{ + Name: c.mimeTypesConfigMapName(), + }}}} +} + +func (c *clusterConfig) mimeTypesVolumeMount() v1.VolumeMount { + return v1.VolumeMount{ + Name: c.mimeTypesConfigMapName(), + MountPath: mimeTypesMountDir, + ReadOnly: true, + // read-only so malicious actors can't edit, which could allow them to add file type + // execution vulnerabilities that an admin has taken care to eliminate + } +} const mimeTypes = ` application/activemessage diff --git a/pkg/daemon/ceph/rgw/objectstore.go b/pkg/operator/ceph/object/objectstore.go similarity index 94% rename from pkg/daemon/ceph/rgw/objectstore.go rename to pkg/operator/ceph/object/objectstore.go index 121da1b0a..ea047305f 100644 --- a/pkg/daemon/ceph/rgw/objectstore.go +++ b/pkg/operator/ceph/object/objectstore.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package rgw +package object import ( "encoding/json" @@ -27,7 +27,9 @@ import ( const ( rootPool = ".rgw.root" - appName = "rgw" + + // AppName is the name Rook uses for the object store's application + AppName = "rook-ceph-rgw" ) var ( @@ -51,7 +53,7 @@ type realmType struct { Realms []string `json:"realms"` } -func CreateObjectStore(context *Context, metadataSpec, dataSpec model.Pool, serviceIP string, port int32) error { +func createObjectStore(context *Context, metadataSpec, dataSpec model.Pool, serviceIP string, port int32) error { err := createPools(context, metadataSpec, dataSpec) if err != nil { return fmt.Errorf("failed to create object pools. %+v", err) @@ -64,8 +66,8 @@ func CreateObjectStore(context *Context, metadataSpec, dataSpec model.Pool, serv return nil } -func DeleteObjectStore(context *Context) error { - stores, err := GetObjectStores(context) +func deleteRealmAndPools(context *Context) error { + stores, err := getObjectStores(context) if err != nil { return fmt.Errorf("failed to detect object stores during deletion. %+v", err) } @@ -95,7 +97,7 @@ func createRealm(context *Context, serviceIP string, port int32) error { // The first realm must be marked as the default defaultArg := "" - stores, err := GetObjectStores(context) + stores, err := getObjectStores(context) if err != nil { return fmt.Errorf("failed to get object stores. %+v", err) } @@ -189,7 +191,7 @@ func decodeID(data string) (string, error) { return id.ID, err } -func GetObjectStores(context *Context) ([]string, error) { +func getObjectStores(context *Context) ([]string, error) { output, err := runAdminCommandNoRealm(context, "realm", "list") if err != nil { if strings.Index(err.Error(), "exit status 2") != 0 { @@ -274,9 +276,9 @@ func createSimilarPools(context *Context, pools []string, poolSpec model.Pool) e if isECPool { // An EC pool backing an object store does not need to enable EC overwrites, so the pool is // created with that property disabled to avoid unnecessary performance impact. - err = ceph.CreateECPoolForApp(context.context, context.ClusterName, cephConfig, appName, false /* enableECOverwrite */, poolSpec.ErasureCodedConfig) + err = ceph.CreateECPoolForApp(context.context, context.ClusterName, cephConfig, AppName, false /* enableECOverwrite */, poolSpec.ErasureCodedConfig) } else { - err = ceph.CreateReplicatedPoolForApp(context.context, context.ClusterName, cephConfig, appName) + err = ceph.CreateReplicatedPoolForApp(context.context, context.ClusterName, cephConfig, AppName) } if err != nil { return fmt.Errorf("failed to create pool %s for object store %s", name, context.Name) diff --git a/pkg/daemon/ceph/rgw/objectstore_test.go b/pkg/operator/ceph/object/objectstore_test.go similarity index 98% rename from pkg/daemon/ceph/rgw/objectstore_test.go rename to pkg/operator/ceph/object/objectstore_test.go index 6f398d4c3..278be2c15 100644 --- a/pkg/daemon/ceph/rgw/objectstore_test.go +++ b/pkg/operator/ceph/object/objectstore_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package rgw +package object import ( "fmt" @@ -43,9 +43,8 @@ func TestCreateRealm(t *testing.T) { } else if args[0] == "realm" && args[1] == "list" { if defaultStore { return "", fmt.Errorf("failed to run radosgw-admin: Failed to complete : exit status 2") - } else { - return `{"realms": ["myobj"]}`, nil } + return `{"realms": ["myobj"]}`, nil } return idResponse, nil } @@ -146,7 +145,7 @@ func deleteStore(t *testing.T, name string, existingStores string, expectedDelet context := &Context{context: &clusterd.Context{Executor: executor}, Name: "myobj", ClusterName: "ns"} // Delete an object store - err := DeleteObjectStore(context) + err := deleteRealmAndPools(context) assert.Nil(t, err) expectedPoolsDeleted := 5 if expectedDeleteRootPool { diff --git a/pkg/operator/ceph/object/rgw.go b/pkg/operator/ceph/object/rgw.go index 3c4397b5a..0a15f4107 100644 --- a/pkg/operator/ceph/object/rgw.go +++ b/pkg/operator/ceph/object/rgw.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package rgw for the Ceph object store. +// Package object for the Ceph object store. package object import ( @@ -22,43 +22,36 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" - "github.com/rook/rook/pkg/daemon/ceph/client" - rgwdaemon "github.com/rook/rook/pkg/daemon/ceph/rgw" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/pool" "github.com/rook/rook/pkg/operator/k8sutil" - "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) -const ( - appName = "rook-ceph-rgw" - keyringName = "keyring" - certVolumeName = "rook-rgw-cert" - certMountPath = "/etc/rook/private" - certKeyName = "cert" - certFilename = "rgw-cert.pem" -) - -type config struct { +type clusterConfig struct { + clusterInfo *cephconfig.ClusterInfo context *clusterd.Context store cephv1.CephObjectStore rookVersion string cephVersion cephv1.CephVersionSpec hostNetwork bool ownerRefs []metav1.OwnerReference + DataPathMap *config.DataPathMap } // Start the rgw manager -func (c *config) createStore() error { +func (c *clusterConfig) createStore() error { return c.createOrUpdate(false) } -func (c *config) updateStore() error { +func (c *clusterConfig) updateStore() error { return c.createOrUpdate(true) } -func (c *config) createOrUpdate(update bool) error { +func (c *clusterConfig) createOrUpdate(update bool) error { // validate the object store settings if err := validateStore(c.context, c.store); err != nil { return fmt.Errorf("invalid object store %s arguments. %+v", c.store.Name, err) @@ -75,10 +68,6 @@ func (c *config) createOrUpdate(update bool) error { } logger.Infof("creating object store %s in namespace %s", c.store.Name, c.store.Namespace) - err = c.createKeyring() - if err != nil { - return fmt.Errorf("failed to create rgw keyring. %+v", err) - } // start the service serviceIP, err := c.startService() @@ -87,8 +76,8 @@ func (c *config) createOrUpdate(update bool) error { } // create the ceph artifacts for the object store - objContext := rgwdaemon.NewContext(c.context, c.store.Name, c.store.Namespace) - err = rgwdaemon.CreateObjectStore(objContext, *c.store.Spec.MetadataPool.ToModel(""), *c.store.Spec.DataPool.ToModel(""), serviceIP, c.store.Spec.Gateway.Port) + objContext := NewContext(c.context, c.store.Name, c.store.Namespace) + err = createObjectStore(objContext, *c.store.Spec.MetadataPool.ToModel(""), *c.store.Spec.DataPool.ToModel(""), serviceIP, c.store.Spec.Gateway.Port) if err != nil { return fmt.Errorf("failed to create pools. %+v", err) } @@ -101,7 +90,7 @@ func (c *config) createOrUpdate(update bool) error { return nil } -func (c *config) startRGWPods(update bool) error { +func (c *clusterConfig) startRGWPods(update bool) error { // if intended to update, remove the old pods so they can be created with the new spec settings if update { @@ -116,15 +105,49 @@ func (c *config) startRGWPods(update bool) error { } // start the deployment or daemonset + var uid types.UID + var controllerType string if c.store.Spec.Gateway.AllNodes { - return c.startDaemonset() + daemonSet, err := c.startDaemonset() + if err != nil { + return err + } + uid = daemonSet.UID + controllerType = "daemonset" + } else { + deployment, err := c.startDeployment() + if err != nil { + return err + } + uid = deployment.UID + controllerType = "deployment" } - return c.startDeployment() + + resourceControllerOwnerRef := &metav1.OwnerReference{ + UID: uid, + APIVersion: "v1", + Kind: controllerType, + Name: c.instanceName(), + } + + // Generate the keyring after starting the replication controller so that the keyring may use + // the controller as its owner reference; the keyring is deleted with the controller + err := c.generateKeyring(resourceControllerOwnerRef) + if err != nil { + return fmt.Errorf("failed to create rgw keyring. %+v", err) + } + + // Generate the mime.types file after the rep. controller as well for the same reason as keyring + if err := c.generateMimeTypes(resourceControllerOwnerRef); err != nil { + return fmt.Errorf("failed to generate the rgw mime.types config. %+v", err) + } + + return nil } // Delete the object store. // WARNING: This is a very destructive action that deletes all metadata and data pools. -func (c *config) deleteStore() error { +func (c *clusterConfig) deleteStore() error { // check if the object store exists exists, err := c.storeExists() if err != nil { @@ -164,8 +187,8 @@ func (c *config) deleteStore() error { } // Delete the realm and pools - objContext := rgwdaemon.NewContext(c.context, c.store.Name, c.store.Namespace) - err = rgwdaemon.DeleteObjectStore(objContext) + objContext := NewContext(c.context, c.store.Name, c.store.Namespace) + err = deleteRealmAndPools(objContext) if err != nil { return fmt.Errorf("failed to delete the realm and pools. %+v", err) } @@ -175,7 +198,7 @@ func (c *config) deleteStore() error { } // Check if the object store exists depending on either the deployment or the daemonset -func (c *config) storeExists() (bool, error) { +func (c *clusterConfig) storeExists() (bool, error) { _, err := c.context.Clientset.Apps().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) if err == nil { // the deployment was found @@ -198,60 +221,8 @@ func (c *config) storeExists() (bool, error) { return false, nil } -func (c *config) createKeyring() error { - _, err := c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) - if err == nil { - logger.Infof("the rgw keyring was already generated") - return nil - } - if !errors.IsNotFound(err) { - return fmt.Errorf("failed to get rgw secrets. %+v", err) - } - - // create the keyring - logger.Infof("generating rgw keyring") - keyring, err := c.createRGWKeyring() - if err != nil { - return fmt.Errorf("failed to create keyring. %+v", err) - } - - // store the secrets - secrets := map[string]string{ - keyringName: keyring, - } - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: c.instanceName(), - Namespace: c.store.Namespace, - }, - StringData: secrets, - Type: k8sutil.RookType, - } - k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &secret.ObjectMeta, c.ownerRefs) - _, err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Create(secret) - if err != nil { - return fmt.Errorf("failed to save rgw secrets. %+v", err) - } - - return nil -} - -func (c *config) instanceName() string { - return fmt.Sprintf("%s-%s", appName, c.store.Name) -} - -// create a keyring for the rgw client with a limited set of privileges -func (c *config) createRGWKeyring() (string, error) { - username := "client.radosgw.gateway" - access := []string{"osd", "allow rwx", "mon", "allow rw"} - - // get-or-create-key for the user account - key, err := client.AuthGetOrCreateKey(c.context, c.store.Namespace, username, access) - if err != nil { - return "", fmt.Errorf("failed to get or create auth key for %store. %+v", username, err) - } - - return key, err +func (c *clusterConfig) instanceName() string { + return fmt.Sprintf("%s-%s", AppName, c.store.Name) } // Validate the object store arguments diff --git a/pkg/operator/ceph/object/rgw_test.go b/pkg/operator/ceph/object/rgw_test.go index b3da12886..3c224da92 100644 --- a/pkg/operator/ceph/object/rgw_test.go +++ b/pkg/operator/ceph/object/rgw_test.go @@ -23,6 +23,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + cephconfig "github.com/rook/rook/pkg/operator/ceph/config" testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" @@ -44,12 +45,14 @@ func TestStartRGW(t *testing.T) { configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) + info := testop.CreateConfigDir(1) context := &clusterd.Context{Clientset: clientset, Executor: executor, ConfigDir: configDir} store := simpleStore() version := "v1.1.0" + data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "my-fs") // start a basic cluster - c := &config{context, store, version, cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}} + c := &clusterConfig{info, context, store, version, cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}, data} err := c.createStore() assert.Nil(t, err) @@ -63,7 +66,7 @@ func TestStartRGW(t *testing.T) { validateStart(t, c, clientset, true) } -func validateStart(t *testing.T, c *config, clientset *fake.Clientset, allNodes bool) { +func validateStart(t *testing.T, c *clusterConfig, clientset *fake.Clientset, allNodes bool) { if !allNodes { r, err := clientset.Apps().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) assert.Nil(t, err) @@ -83,11 +86,6 @@ func validateStart(t *testing.T, c *config, clientset *fake.Clientset, allNodes s, err := clientset.CoreV1().Services(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) assert.Nil(t, err) assert.Equal(t, c.instanceName(), s.Name) - - secret, err := clientset.CoreV1().Secrets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) - assert.Nil(t, err) - assert.Equal(t, c.instanceName(), secret.Name) - assert.Equal(t, 1, len(secret.StringData)) } func TestCreateObjectStore(t *testing.T) { @@ -114,9 +112,11 @@ func TestCreateObjectStore(t *testing.T) { store := simpleStore() clientset := testop.New(3) context := &clusterd.Context{Executor: executor, Clientset: clientset} + info := testop.CreateConfigDir(1) + data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "my-fs") // create the pools - c := &config{context, store, "1.2.3.4", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}} + c := &clusterConfig{info, context, store, "1.2.3.4", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}, data} err := c.createStore() assert.Nil(t, err) } diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 93521abd2..7e745c73e 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -14,15 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package object for the Ceph object store. package object import ( "fmt" - "path" - rgwdaemon "github.com/rook/rook/pkg/daemon/ceph/rgw" - opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon" + cephconfig "github.com/rook/rook/pkg/operator/ceph/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" @@ -32,11 +29,12 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) -func (c *config) startDeployment() error { +func (c *clusterConfig) startDeployment() (*apps.Deployment, error) { d := &apps.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: c.instanceName(), Namespace: c.store.Namespace, + Labels: c.getLabels(), }, Spec: apps.DeploymentSpec{ Selector: &metav1.LabelSelector{ @@ -51,29 +49,29 @@ func (c *config) startDeployment() error { } k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &d.ObjectMeta, c.ownerRefs) - logger.Debugf("starting mds deployment: %+v", d) - _, err := c.context.Clientset.Apps().Deployments(c.store.Namespace).Create(d) + logger.Debugf("starting rgw deployment: %+v", d) + deployment, err := c.context.Clientset.Apps().Deployments(c.store.Namespace).Create(d) if err != nil { if !errors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create rgw deployment %s: %+v", c.instanceName(), err) + return nil, fmt.Errorf("failed to create rgw deployment %s: %+v", c.instanceName(), err) } logger.Infof("deployment for rgw %s already exists. updating if needed", c.instanceName()) // There may be a *lot* of rgws, and they are stateless, so don't bother waiting until the // entire deployment is updated to move on. - _, err := c.context.Clientset.Apps().Deployments(c.store.Namespace).Update(d) + deployment, err = c.context.Clientset.Apps().Deployments(c.store.Namespace).Update(d) if err != nil { - return fmt.Errorf("failed to update rgw deployment %s. %+v", c.instanceName(), err) + return nil, fmt.Errorf("failed to update rgw deployment %s. %+v", c.instanceName(), err) } } - - return nil + return deployment, nil } -func (c *config) startDaemonset() error { +func (c *clusterConfig) startDaemonset() (*apps.DaemonSet, error) { d := &apps.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: c.instanceName(), Namespace: c.store.Namespace, + Labels: c.getLabels(), }, Spec: apps.DaemonSetSpec{ Selector: &metav1.LabelSelector{ @@ -88,36 +86,36 @@ func (c *config) startDaemonset() error { k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &d.ObjectMeta, c.ownerRefs) logger.Debugf("starting rgw daemonset: %+v", d) - _, err := c.context.Clientset.Apps().DaemonSets(c.store.Namespace).Create(d) + daemonSet, err := c.context.Clientset.Apps().DaemonSets(c.store.Namespace).Create(d) if err != nil { if !errors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create rgw daemonset %s: %+v", c.instanceName(), err) + return nil, fmt.Errorf("failed to create rgw daemonset %s: %+v", c.instanceName(), err) } logger.Infof("daemonset for rgw %s already exists. updating if needed", c.instanceName()) // There may be a *lot* of rgws, and they are stateless, so don't bother waiting until the // entire daemonset is updated to move on. // TODO: is the above statement safe to assume? // TODO: Are there any steps for RGW that need to happen before the daemons upgrade? - _, err := c.context.Clientset.Apps().DaemonSets(c.store.Namespace).Update(d) + daemonSet, err = c.context.Clientset.Apps().DaemonSets(c.store.Namespace).Update(d) if err != nil { - return fmt.Errorf("failed to update rgw daemonset %s. %+v", c.instanceName(), err) + return nil, fmt.Errorf("failed to update rgw daemonset %s. %+v", c.instanceName(), err) } } - - return nil + return daemonSet, nil } -func (c *config) makeRGWPodSpec() v1.PodTemplateSpec { +func (c *clusterConfig) makeRGWPodSpec() v1.PodTemplateSpec { podSpec := v1.PodSpec{ - InitContainers: []v1.Container{ - c.makeConfigInitContainer(), - }, + InitContainers: []v1.Container{}, Containers: []v1.Container{ c.makeDaemonContainer(), }, RestartPolicy: v1.RestartPolicyAlways, - Volumes: opspec.PodVolumes(""), - HostNetwork: c.hostNetwork, + Volumes: append( + opspec.DaemonVolumes(c.DataPathMap, c.instanceName()), + c.mimeTypesVolume(), + ), + HostNetwork: c.hostNetwork, } if c.hostNetwork { podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet @@ -125,10 +123,16 @@ func (c *config) makeRGWPodSpec() v1.PodTemplateSpec { // Set the ssl cert if specified if c.store.Spec.Gateway.SSLCertificateRef != "" { - certVol := v1.Volume{Name: certVolumeName, VolumeSource: v1.VolumeSource{Secret: &v1.SecretVolumeSource{ - SecretName: c.store.Spec.Gateway.SSLCertificateRef, - Items: []v1.KeyToPath{{Key: certKeyName, Path: certFilename}}, - }}} + // Keep the SSL secret as secure as possible in the container. Give only user read perms. + userReadOnly := int32(0400) + certVol := v1.Volume{ + Name: certVolumeName, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: c.store.Spec.Gateway.SSLCertificateRef, + Items: []v1.KeyToPath{ + {Key: certKeyName, Path: certFilename, Mode: &userReadOnly}, + }}}} podSpec.Volumes = append(podSpec.Volumes, certVol) } @@ -144,40 +148,7 @@ func (c *config) makeRGWPodSpec() v1.PodTemplateSpec { } } -func (c *config) makeConfigInitContainer() v1.Container { - container := v1.Container{ - Name: opspec.ConfigInitContainerName, - Image: k8sutil.MakeRookImage(c.rookVersion), - Args: []string{ - "ceph", - "rgw", - fmt.Sprintf("--config-dir=%s", k8sutil.DataDir), - fmt.Sprintf("--rgw-name=%s", c.store.Name), - fmt.Sprintf("--rgw-port=%d", c.store.Spec.Gateway.Port), - fmt.Sprintf("--rgw-secure-port=%d", c.store.Spec.Gateway.SecurePort), - }, - VolumeMounts: opspec.RookVolumeMounts(), - Env: []v1.EnvVar{ - {Name: "ROOK_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: c.instanceName()}, Key: keyringName}}}, - k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), - k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), - opmon.ClusterNameEnvVar(c.store.Namespace), - opmon.EndpointEnvVar(), - opmon.SecretEnvVar(), - k8sutil.ConfigOverrideEnvVar(), - }, - Resources: c.store.Spec.Gateway.Resources, - } - - if c.store.Spec.Gateway.SSLCertificateRef != "" { - path := path.Join(certMountPath, certFilename) - container.Args = append(container.Args, fmt.Sprintf("--rgw-cert=%s", path)) - } - - return container -} - -func (c *config) makeDaemonContainer() v1.Container { +func (c *clusterConfig) makeDaemonContainer() v1.Container { // start the rgw daemon in the foreground container := v1.Container{ @@ -186,26 +157,33 @@ func (c *config) makeDaemonContainer() v1.Container { Command: []string{ "radosgw", }, - Args: []string{ - "--foreground", - "--name=client.radosgw.gateway", - fmt.Sprintf("--rgw-mime-types-file=%s", rgwdaemon.GetMimeTypesPath(k8sutil.DataDir)), - }, - VolumeMounts: opspec.CephVolumeMounts(), - Env: k8sutil.ClusterDaemonEnvVars(c.cephVersion.Image), - Resources: c.store.Spec.Gateway.Resources, + Args: append( + append( + opspec.DaemonFlags(c.clusterInfo, cephconfig.RgwType, c.store.Name), + "--foreground", + "--name=client.radosgw.gateway", + cephconfig.NewFlag("host", opspec.ContainerEnvVarReference("POD_NAME")), + cephconfig.NewFlag("rgw-mime-types-file", mimeTypesMountPath()), + ), c.defaultSettings().GlobalFlags()..., // use default settings as flags until mon kv store supported + ), + VolumeMounts: append( + opspec.DaemonVolumeMounts(c.DataPathMap, c.instanceName()), + c.mimeTypesVolumeMount(), + ), + Env: opspec.DaemonEnvVars(c.cephVersion.Image), + Resources: c.store.Spec.Gateway.Resources, } if c.store.Spec.Gateway.SSLCertificateRef != "" { // Add a volume mount for the ssl certificate - mount := v1.VolumeMount{Name: certVolumeName, MountPath: certMountPath, ReadOnly: true} + mount := v1.VolumeMount{Name: certVolumeName, MountPath: certDir, ReadOnly: true} container.VolumeMounts = append(container.VolumeMounts, mount) } return container } -func (c *config) startService() (string, error) { +func (c *clusterConfig) startService() (string, error) { labels := c.getLabels() svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -253,10 +231,8 @@ func addPort(service *v1.Service, name string, port int32) { }) } -func (c *config) getLabels() map[string]string { - return map[string]string{ - k8sutil.AppAttr: appName, - k8sutil.ClusterAttr: c.store.Namespace, - "rook_object_store": c.store.Name, - } +func (c *clusterConfig) getLabels() map[string]string { + labels := opspec.PodLabels(AppName, c.store.Namespace, "rgw", c.store.Name) + labels["rook_object_store"] = c.store.Name + return labels } diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index a6521bb50..f1f5a1c53 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -17,13 +17,13 @@ limitations under the License. package object import ( - "fmt" "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" - cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" - "github.com/rook/rook/pkg/operator/k8sutil" + cephconfig "github.com/rook/rook/pkg/operator/ceph/config" + cephtest "github.com/rook/rook/pkg/operator/ceph/test" + testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" @@ -40,80 +40,59 @@ func TestPodSpecs(t *testing.T) { v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI), }, } + info := testop.CreateConfigDir(1) + data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "default") + + c := &clusterConfig{ + clusterInfo: info, + store: store, + rookVersion: "rook/rook:myversion", + cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.1", Name: "mimic"}, + hostNetwork: true, + DataPathMap: data, + } - c := &config{store: store, rookVersion: "rook/rook:myversion", cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.1", Name: "mimic"}, hostNetwork: true} s := c.makeRGWPodSpec() - assert.NotNil(t, s) - //assert.Equal(t, instanceName(store), s.Name) - assert.Equal(t, v1.RestartPolicyAlways, s.Spec.RestartPolicy) - assert.Equal(t, 3, len(s.Spec.Volumes)) - assert.Equal(t, "rook-data", s.Spec.Volumes[0].Name) - assert.Equal(t, cephconfig.DefaultConfigMountName, s.Spec.Volumes[1].Name) - assert.Equal(t, c.instanceName(), s.ObjectMeta.Name) - assert.Equal(t, appName, s.ObjectMeta.Labels["app"]) - assert.Equal(t, store.Namespace, s.ObjectMeta.Labels["rook_cluster"]) - assert.Equal(t, store.Name, s.ObjectMeta.Labels["rook_object_store"]) - assert.Equal(t, 0, len(s.ObjectMeta.Annotations)) - - cont := s.Spec.InitContainers[0] - assert.Equal(t, 1, len(s.Spec.InitContainers)) - assert.Equal(t, "rook/rook:myversion", cont.Image) - assert.Equal(t, 3, len(cont.VolumeMounts)) - assert.Equal(t, 0, len(cont.Command)) - assert.Equal(t, 6, len(cont.Args)) - assert.Equal(t, "ceph", cont.Args[0]) - assert.Equal(t, "rgw", cont.Args[1]) - assert.Equal(t, "--config-dir=/var/lib/rook", cont.Args[2]) - assert.Equal(t, "--rgw-name=default", cont.Args[3]) - assert.Equal(t, "--rgw-port=123", cont.Args[4]) - assert.Equal(t, "--rgw-secure-port=0", cont.Args[5]) - - cont = s.Spec.Containers[0] - assert.Equal(t, "ceph/ceph:v13.2.1", cont.Image) - assert.Equal(t, 2, len(cont.VolumeMounts)) - - assert.Equal(t, 1, len(cont.Command)) - assert.Equal(t, "radosgw", cont.Command[0]) - assert.Equal(t, 3, len(cont.Args)) - assert.Equal(t, "--foreground", cont.Args[0]) - assert.Equal(t, "--name=client.radosgw.gateway", cont.Args[1]) - assert.Equal(t, "--rgw-mime-types-file=/var/lib/rook/rgw/mime.types", cont.Args[2]) - - assert.Equal(t, len(k8sutil.ClusterDaemonEnvVars("ceph/ceph:v14")), len(cont.Env)) - - assert.Equal(t, "100", cont.Resources.Limits.Cpu().String()) - assert.Equal(t, "1337", cont.Resources.Requests.Memory().String()) + podTemplate := cephtest.NewPodTemplateSpecTester(t, &s) + podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion", + "100", "1337" /* resources */) } func TestSSLPodSpec(t *testing.T) { store := simpleStore() + store.Spec.Gateway.Resources = v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(100.0, resource.BinarySI), + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI), + }, + } + info := testop.CreateConfigDir(1) + data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "default") store.Spec.Gateway.SSLCertificateRef = "mycert" store.Spec.Gateway.SecurePort = 443 - c := &config{store: store, rookVersion: "v1.0", hostNetwork: true} + c := &clusterConfig{ + clusterInfo: info, + store: store, + rookVersion: "rook/rook:myversion", + cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.1", Name: "mimic"}, + hostNetwork: true, + DataPathMap: data, + } + c.hostNetwork = true + s := c.makeRGWPodSpec() - assert.NotNil(t, s) - assert.Equal(t, c.instanceName(), s.Name) - assert.Equal(t, 4, len(s.Spec.Volumes)) - assert.Equal(t, certVolumeName, s.Spec.Volumes[3].Name) + podTemplate := cephtest.NewPodTemplateSpecTester(t, &s) + podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion", + "100", "1337" /* resources */) + assert.True(t, s.Spec.HostNetwork) assert.Equal(t, v1.DNSClusterFirstWithHostNet, s.Spec.DNSPolicy) - initCont := s.Spec.InitContainers[0] - - assert.Equal(t, 3, len(initCont.VolumeMounts)) - assert.Equal(t, 7, len(initCont.Args)) - assert.Equal(t, fmt.Sprintf("--rgw-secure-port=%d", 443), initCont.Args[5]) - assert.Equal(t, fmt.Sprintf("--rgw-cert=%s/%s", certMountPath, certFilename), initCont.Args[6]) - - rgwCont := s.Spec.Containers[0] - - assert.Equal(t, 3, len(rgwCont.VolumeMounts)) - assert.Equal(t, certVolumeName, rgwCont.VolumeMounts[2].Name) - assert.Equal(t, certMountPath, rgwCont.VolumeMounts[2].MountPath) - assert.Equal(t, 3, len(rgwCont.Args)) } func TestValidateSpec(t *testing.T) { diff --git a/pkg/daemon/ceph/rgw/user.go b/pkg/operator/ceph/object/user.go similarity index 91% rename from pkg/daemon/ceph/rgw/user.go rename to pkg/operator/ceph/object/user.go index be83aab5d..f1eb3f999 100644 --- a/pkg/daemon/ceph/rgw/user.go +++ b/pkg/operator/ceph/object/user.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package rgw +package object import ( "encoding/json" @@ -24,13 +24,14 @@ import ( ) const ( - RGWErrorNone = iota - RGWErrorUnknown = iota - RGWErrorNotFound = iota - RGWErrorBadData = iota - RGWErrorParse = iota + RGWErrorNone = iota + RGWErrorUnknown + RGWErrorNotFound + RGWErrorBadData + RGWErrorParse ) +// An ObjectUser defines the details of an object store user. type ObjectUser struct { UserID string `json:"userId"` DisplayName *string `json:"displayName"` @@ -39,6 +40,7 @@ type ObjectUser struct { SecretKey *string `json:"secretKey"` } +// ListUsers lists the object pool users. func ListUsers(c *Context) ([]string, int, error) { result, err := runAdminCommand(c, "user", "list") if err != nil { @@ -80,6 +82,7 @@ func decodeUser(data string) (*ObjectUser, int, error) { return &rookUser, RGWErrorNone, nil } +// GetUser returns the user with the given ID. func GetUser(c *Context, id string) (*ObjectUser, int, error) { logger.Infof("Getting user: %s", id) @@ -93,6 +96,7 @@ func GetUser(c *Context, id string) (*ObjectUser, int, error) { return decodeUser(result) } +// CreateUser creates a new user with the information given. func CreateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) { logger.Infof("Creating user: %s", user.UserID) @@ -131,6 +135,7 @@ func CreateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) { return decodeUser(result) } +// UpdateUser updates the user whose ID matches the user. func UpdateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) { logger.Infof("Updating user: %s", user.UserID) @@ -155,6 +160,7 @@ func UpdateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) { return decodeUser(body) } +// DeleteUser deletes the user with the given ID. func DeleteUser(c *Context, id string) (string, int, error) { logger.Infof("Deleting user: %s", id) result, err := runAdminCommand(c, "user", "rm", "--uid", id) diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go index d6d4d4498..87f99e651 100644 --- a/pkg/operator/ceph/object/user/controller.go +++ b/pkg/operator/ceph/object/user/controller.go @@ -25,7 +25,7 @@ import ( opkit "github.com/rook/operator-kit" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" - cephrgw "github.com/rook/rook/pkg/daemon/ceph/rgw" + "github.com/rook/rook/pkg/operator/ceph/object" "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/api/core/v1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" @@ -34,12 +34,12 @@ import ( ) const ( - AppName = "rook-ceph-rgw" + appName = object.AppName ) var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-object") -// ObjectStoreResource represents the object store user custom resource +// ObjectStoreUserResource represents the object store user custom resource var ObjectStoreUserResource = opkit.CustomResource{ Name: "cephobjectstoreuser", Plural: "cephobjectstoreusers", @@ -139,13 +139,13 @@ func (c *ObjectStoreUserController) createUser(context *clusterd.Context, u *cep // create the user logger.Infof("creating user %s in namespace %s", u.Name, u.Namespace) - userConfig := cephrgw.ObjectUser{ + userConfig := object.ObjectUser{ UserID: u.Name, DisplayName: &displayName, } - objContext := cephrgw.NewContext(context, u.Spec.Store, u.Namespace) + objContext := object.NewContext(context, u.Spec.Store, u.Namespace) - user, rgwerr, err := cephrgw.CreateUser(objContext, userConfig) + user, rgwerr, err := object.CreateUser(objContext, userConfig) if err != nil { return fmt.Errorf("failed to create user %s. RadosGW returned error %d: %+v", u.Name, rgwerr, err) } @@ -160,7 +160,7 @@ func (c *ObjectStoreUserController) createUser(context *clusterd.Context, u *cep Name: fmt.Sprintf("rook-ceph-object-user-%s-%s", u.Spec.Store, u.Name), Namespace: u.Namespace, Labels: map[string]string{ - "app": AppName, + "app": appName, "user": u.Name, "rook_cluster": u.Namespace, "rook_object_store": u.Spec.Store, @@ -181,8 +181,8 @@ func (c *ObjectStoreUserController) createUser(context *clusterd.Context, u *cep // Delete the user func deleteUser(context *clusterd.Context, u *cephv1.CephObjectStoreUser) error { - objContext := cephrgw.NewContext(context, u.Spec.Store, u.Namespace) - _, rgwerr, err := cephrgw.DeleteUser(objContext, u.Name) + objContext := object.NewContext(context, u.Spec.Store, u.Namespace) + _, rgwerr, err := object.DeleteUser(objContext, u.Name) if err != nil { if rgwerr == 3 { logger.Infof("user %s does not exist in store %s", u.Name, u.Spec.Store) @@ -200,7 +200,7 @@ func deleteUser(context *clusterd.Context, u *cephv1.CephObjectStoreUser) error return nil } -// Validate the user arguments +// ValidateUser validates the user arguments func ValidateUser(context *clusterd.Context, u *cephv1.CephObjectStoreUser) error { if u.Name == "" { return fmt.Errorf("missing name") diff --git a/pkg/operator/ceph/spec/spec.go b/pkg/operator/ceph/spec/spec.go index 52c66e20d..e9e15d919 100644 --- a/pkg/operator/ceph/spec/spec.go +++ b/pkg/operator/ceph/spec/spec.go @@ -95,9 +95,9 @@ func DaemonVolumeMounts(dataPaths *config.DataPathMap, keyringResourceName strin func DaemonFlags(cluster *cephconfig.ClusterInfo, daemonType config.DaemonType, daemonID string) []string { return append( config.DefaultFlags(cluster.FSID, keyring.VolumeMount().KeyringFilePath()), - // all daemons are named in the format . - config.NewFlag("name", fmt.Sprintf("%s.%s", string(daemonType), daemonID)), + config.NewFlag("id", daemonID), ) + } // AdminFlags returns the command line flags used for Ceph commands requiring admin authentication. diff --git a/tests/framework/clients/object_user.go b/tests/framework/clients/object_user.go index aefe5f948..547f046f2 100644 --- a/tests/framework/clients/object_user.go +++ b/tests/framework/clients/object_user.go @@ -18,7 +18,8 @@ package clients import ( "fmt" - "github.com/rook/rook/pkg/daemon/ceph/rgw" + + rgw "github.com/rook/rook/pkg/operator/ceph/object" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" ) diff --git a/tests/integration/base_object_test.go b/tests/integration/base_object_test.go index 24be17ac2..51b19d7c9 100644 --- a/tests/integration/base_object_test.go +++ b/tests/integration/base_object_test.go @@ -19,13 +19,14 @@ package integration import ( "errors" - rgwdaemon "github.com/rook/rook/pkg/daemon/ceph/rgw" + "time" + + rgw "github.com/rook/rook/pkg/operator/ceph/object" "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "time" ) var ( @@ -167,17 +168,17 @@ func objectTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, na helper.ObjectClient.DeleteUser(storeName, userid)*/ } -func getBucket(bucketname string, bucketList []rgwdaemon.ObjectBucket) (rgwdaemon.ObjectBucket, error) { +func getBucket(bucketname string, bucketList []rgw.ObjectBucket) (rgw.ObjectBucket, error) { for _, bucket := range bucketList { if bucket.Name == bucketname { return bucket, nil } } - return rgwdaemon.ObjectBucket{}, errors.New("Bucket not found") + return rgw.ObjectBucket{}, errors.New("Bucket not found") } -func getBucketSizeAndObjects(bucketname string, bucketList []rgwdaemon.ObjectBucket) (uint64, uint64, error) { +func getBucketSizeAndObjects(bucketname string, bucketList []rgw.ObjectBucket) (uint64, uint64, error) { bkt, err := getBucket(bucketname, bucketList) if err != nil { return 0, 0, errors.New("Bucket not found")