diff --git a/Documentation/ceph-object-store-crd.md b/Documentation/ceph-object-store-crd.md index aac5239f9..4f26ae6b0 100644 --- a/Documentation/ceph-object-store-crd.md +++ b/Documentation/ceph-object-store-crd.md @@ -86,3 +86,18 @@ The gateway settings correspond to the RGW daemon settings. - `allNodes`: Whether RGW pods should be started on all nodes. If true, a daemonset is created. If false, `instances` must be set. - `placement`: The Kubernetes placement settings to determine where the RGW pods should be started in the cluster. - `resources`: Set resource requests/limits for the Gateway Pod(s), see [Resource Requirements/Limits](ceph-cluster-crd.md#resource-requirementslimits). + +## Runtime settings + +### MIME types + +Rook provides a default `mime.types` file for each Ceph object store. This file is stored in a +Kubernetes ConfigMap with the name `rook-ceph-rgw--mime-types`. For most users, the +default file should suffice, however, the option is available to users to edit the `mime.types` +file in the ConfigMap as they desire. Users may have their own special file types, and particularly +security conscious users may wish to pare down the file to reduce the possibility of a file type +execution attack. + +Rook will not overwrite an existing `mime.types` ConfigMap so that user modifications will not be +destroyed. If the object store is destroyed and recreated, the ConfigMap will also be destroyed and +created anew. diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index bb4fdc703..a46420db1 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -29,9 +29,10 @@ - Rook will no longer create a directory-based osd in the `dataDirHostPath` if no directories or devices are specified or if there are no disks on the host. -- Containers in `mon`, `mgr`, and `mds` pods have been removed and/or changed names. -- Config paths in `mon`, `mgr`, and `mds` containers are now always the Ceph default paths - (`/etc/ceph`, `/var/lib/ceph/...`) regardless of the `dataDirHostPath` setting. +- Containers in `mon`, `mgr`, `mds`, and `rgw` pods have been removed and/or changed names. +- Config paths in `mon`, `mgr`, `mds` and `rgw` containers are now always under + `/etc/ceph` or `/var/lib/ceph` and as close to Ceph's default path as possible regardless of the + `dataDirHostPath` setting. ## Known Issues diff --git a/cmd/rook/ceph/ceph.go b/cmd/rook/ceph/ceph.go index 70e0f7df3..1e71da6ad 100644 --- a/cmd/rook/ceph/ceph.go +++ b/cmd/rook/ceph/ceph.go @@ -58,7 +58,6 @@ func init() { Cmd.AddCommand(operatorCmd, agentCmd, osdCmd, - rgwCmd, configCmd, nfsCmd) } diff --git a/cmd/rook/ceph/rgw.go b/cmd/rook/ceph/rgw.go deleted file mode 100644 index 871a73578..000000000 --- a/cmd/rook/ceph/rgw.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ceph - -import ( - "fmt" - "os" - - "github.com/rook/rook/cmd/rook/rook" - rgwdaemon "github.com/rook/rook/pkg/daemon/ceph/rgw" - "github.com/rook/rook/pkg/operator/ceph/cluster/mon" - "github.com/rook/rook/pkg/util/flags" - "github.com/spf13/cobra" -) - -var rgwCmd = &cobra.Command{ - Use: "rgw", - Short: "Generates rgw config and runs the rgw daemon", -} - -var ( - rgwName string - rgwKeyring string - rgwHost string - rgwCert string - rgwPort int - rgwSecurePort int -) - -func init() { - rgwCmd.Flags().StringVar(&rgwName, "rgw-name", "", "name of the object store") - rgwCmd.Flags().StringVar(&rgwKeyring, "rgw-keyring", "", "the rgw keyring") - rgwCmd.Flags().StringVar(&rgwHost, "rgw-host", os.Getenv("HOSTNAME"), "RGW host name. Becomes the only accepted hostname if the rgw dns name property is unset. Defaults to the pod hostname") - rgwCmd.Flags().StringVar(&rgwCert, "rgw-cert", "", "path to the ssl certificate in pem format") - rgwCmd.Flags().IntVar(&rgwPort, "rgw-port", 0, "rgw port (http)") - rgwCmd.Flags().IntVar(&rgwSecurePort, "rgw-secure-port", 0, "rgw secure port number (https)") - addCephFlags(rgwCmd) - - flags.SetFlagsFromEnv(rgwCmd.Flags(), rook.RookEnvVarPrefix) - - rgwCmd.RunE = initRGW -} - -func initRGW(cmd *cobra.Command, args []string) error { - required := []string{"mon-endpoints", "cluster-name", "rgw-name", "rgw-keyring"} - if err := flags.VerifyRequiredFlags(rgwCmd, required); err != nil { - return err - } - - if err := verifyRenamedFlags(rgwCmd); err != nil { - return err - } - - if rgwPort == 0 && rgwSecurePort == 0 { - return fmt.Errorf("port or secure port are required") - } - - rook.SetLogLevel() - - rook.LogStartupInfo(rgwCmd.Flags()) - - clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints) - config := &rgwdaemon.Config{ - ClusterInfo: &clusterInfo, - Name: rgwName, - Keyring: rgwKeyring, - Host: rgwHost, - Port: rgwPort, - SecurePort: rgwSecurePort, - CertificatePath: rgwCert, - } - - err := rgwdaemon.Initialize(createContext(), config) - if err != nil { - rook.TerminateFatal(err) - } - - return nil -} diff --git a/pkg/daemon/ceph/rgw/init.go b/pkg/daemon/ceph/rgw/init.go deleted file mode 100644 index 64bd943af..000000000 --- a/pkg/daemon/ceph/rgw/init.go +++ /dev/null @@ -1,147 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rgw - -import ( - "fmt" - "io/ioutil" - "os" - "path" - "strconv" - - "github.com/coreos/pkg/capnslog" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - "github.com/rook/rook/pkg/clusterd" - cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" - "github.com/rook/rook/pkg/util" -) - -var ( - logger = capnslog.NewPackageLogger("github.com/rook/rook", "cephrgw") - rgwFrontendName = "civetweb" -) - -const ( - keyringTemplate = `[client.radosgw.gateway] -key = %s -caps mon = "allow rw" -caps osd = "allow *" -` -) - -type Config struct { - Name string - Host string - Port int - SecurePort int - Keyring string - CertificatePath string - ClusterInfo *cephconfig.ClusterInfo -} - -func Initialize(context *clusterd.Context, config *Config) error { - - err := generateConfigFiles(context, config) - if err != nil { - return fmt.Errorf("failed to generate rgw config files. %+v", err) - } - - return err -} - -func portString(config *Config) string { - - var portString string - if config.Port != 0 { - portString = strconv.Itoa(config.Port) - } - if config.SecurePort != 0 && config.CertificatePath != "" { - var separator string - if config.Port != 0 { - separator = "+" - } - // the suffix is intended to be appended to the end of the rgw_frontends arg, immediately after the port. - // with ssl enabled, the port number must end with the letter s. - portString = fmt.Sprintf("%s%s%ds ssl_certificate=%s", portString, separator, config.SecurePort, config.CertificatePath) - } - - return portString -} - -func rgwFrontend(config *Config) string { - if cephv1.VersionAtLeast(config.ClusterInfo.CephVersionName, cephv1.Nautilus) { - rgwFrontendName = "beast" - } - return rgwFrontendName -} - -func generateConfigFiles(context *clusterd.Context, config *Config) error { - - // create the rgw data directory - dataDir := path.Join(getRGWConfDir(context.ConfigDir), "data") - if err := os.MkdirAll(dataDir, 0744); err != nil { - logger.Warningf("failed to create data directory %s: %+v", dataDir, err) - } - - settings := map[string]string{ - "host": config.Host, - "rgw data": dataDir, - "rgw log nonexistent bucket": "true", - "rgw intent log object name utc": "true", - "rgw enable usage log": "true", - "rgw_frontends": fmt.Sprintf("%s port=%s", rgwFrontend(config), portString(config)), - "rgw_zone": config.Name, - "rgw_zonegroup": config.Name, - } - configFile, err := cephconfig.GenerateConfigFile(context, config.ClusterInfo, getRGWConfDir(context.ConfigDir), - "client.radosgw.gateway", getRGWKeyringPath(context.ConfigDir), nil, settings) - if err != nil { - return fmt.Errorf("failed to create config file. %+v", err) - } - util.WriteFileToLog(logger, configFile) - - keyringEval := func(key string) string { - return fmt.Sprintf(keyringTemplate, key) - } - - // create rgw config - err = cephconfig.WriteKeyring(getRGWKeyringPath(context.ConfigDir), config.Keyring, keyringEval) - if err != nil { - return fmt.Errorf("failed to save keyring. %+v", err) - } - - // write the mime types config - mimeTypesPath := GetMimeTypesPath(context.ConfigDir) - logger.Infof("Writing mime types to: %s", mimeTypesPath) - if err := ioutil.WriteFile(mimeTypesPath, []byte(mimeTypes), 0644); err != nil { - return fmt.Errorf("failed to write mime types to %s: %+v", mimeTypesPath, err) - } - - return nil -} - -func getRGWConfDir(configDir string) string { - return path.Join(configDir, "rgw") -} - -func getRGWKeyringPath(configDir string) string { - return path.Join(getRGWConfDir(configDir), "keyring") -} - -func GetMimeTypesPath(configDir string) string { - return path.Join(getRGWConfDir(configDir), "mime.types") -} diff --git a/pkg/daemon/ceph/rgw/init_test.go b/pkg/daemon/ceph/rgw/init_test.go deleted file mode 100644 index 5135f2279..000000000 --- a/pkg/daemon/ceph/rgw/init_test.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rgw - -import ( - "testing" - - cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" - "github.com/stretchr/testify/assert" -) - -func TestPortString(t *testing.T) { - // No port or secure port - cfg := &Config{} - result := portString(cfg) - assert.Equal(t, "", result) - - // Insecure port - cfg = &Config{Port: 80} - result = portString(cfg) - assert.Equal(t, "80", result) - - // Secure port - cfg = &Config{SecurePort: 443, CertificatePath: "/etc/rgw/cert.pem"} - result = portString(cfg) - assert.Equal(t, "443s ssl_certificate=/etc/rgw/cert.pem", result) - - // Both ports - cfg = &Config{Port: 80, SecurePort: 443, CertificatePath: "/etc/rgw/cert.pem"} - result = portString(cfg) - assert.Equal(t, "80+443s ssl_certificate=/etc/rgw/cert.pem", result) - - // Secure port requires the cert - cfg = &Config{SecurePort: 443} - result = portString(cfg) - assert.Equal(t, "", result) -} - -func TestFrontend(t *testing.T) { - cfg := &Config{} - cfg.ClusterInfo = &cephconfig.ClusterInfo{CephVersionName: "mimic"} - - result := rgwFrontend(cfg) - assert.Equal(t, "civetweb", result) - - cfg.ClusterInfo.CephVersionName = "nautilus" - result = rgwFrontend(cfg) - assert.Equal(t, "beast", result) - - cfg.ClusterInfo.CephVersionName = "octopus" - result = rgwFrontend(cfg) - assert.Equal(t, "beast", result) -} diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 40271f06e..2f13bae93 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -229,7 +229,7 @@ func (c *ClusterController) onAdd(obj interface{}) { poolController.StartWatch(cluster.Namespace, cluster.stopCh) // Start object store CRD watcher - objectStoreController := object.NewObjectStoreController(c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef) + objectStoreController := object.NewObjectStoreController(cluster.Info, c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef) objectStoreController.StartWatch(cluster.Namespace, cluster.stopCh) // Start object store user CRD watcher diff --git a/pkg/operator/ceph/config/store.go b/pkg/operator/ceph/config/store.go index d78a63462..47651f51a 100644 --- a/pkg/operator/ceph/config/store.go +++ b/pkg/operator/ceph/config/store.go @@ -161,7 +161,7 @@ func (s *Store) createOrUpdateMonHostSecrets(clusterInfo *cephconfig.ClusterInfo monIP := cephutil.GetIPFromEndpoint(m.Endpoint) // This tries to detect the current port if the mon already exists - // This basically handles the transtion between monitors running on 6790 to msgr2 + // This basically handles the transition between monitors running on 6790 to msgr2 // So whatever the previous monitor port was we keep it currentMonPort := cephutil.GetPortFromEndpoint(m.Endpoint) diff --git a/pkg/daemon/ceph/rgw/admin.go b/pkg/operator/ceph/object/admin.go similarity index 93% rename from pkg/daemon/ceph/rgw/admin.go rename to pkg/operator/ceph/object/admin.go index 5028b4490..03ab593c3 100644 --- a/pkg/daemon/ceph/rgw/admin.go +++ b/pkg/operator/ceph/object/admin.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package rgw +package object import ( "fmt" @@ -23,12 +23,14 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" ) +// Context holds the context for the object store. type Context struct { context *clusterd.Context Name string ClusterName string } +// NewContext creates a new object store context. func NewContext(context *clusterd.Context, name, clusterName string) *Context { return &Context{context: context, Name: name, ClusterName: clusterName} } diff --git a/pkg/daemon/ceph/rgw/bucket.go b/pkg/operator/ceph/object/bucket.go similarity index 99% rename from pkg/daemon/ceph/rgw/bucket.go rename to pkg/operator/ceph/object/bucket.go index 7cea9b83a..3151f284b 100644 --- a/pkg/daemon/ceph/rgw/bucket.go +++ b/pkg/operator/ceph/object/bucket.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package rgw +package object import ( "encoding/json" diff --git a/pkg/operator/ceph/object/config.go b/pkg/operator/ceph/object/config.go new file mode 100644 index 000000000..de5b8b1f0 --- /dev/null +++ b/pkg/operator/ceph/object/config.go @@ -0,0 +1,100 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package object + +import ( + "fmt" + "path" + "strconv" + + cephconfig "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/config/keyring" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + keyringTemplate = ` +[client.radosgw.gateway] +key = %s +caps mon = "allow rw" +caps osd = "allow rwx" +` + + certVolumeName = "rook-ceph-rgw-cert" + certDir = "/etc/ceph/private" + certKeyName = "cert" + certFilename = "rgw-cert.pem" +) + +// TODO: these should be set in the mon's central kv store for mimic+ +func (c *clusterConfig) defaultSettings() *cephconfig.Config { + s := cephconfig.NewConfig() + s.Section("global"). + Set("rgw log nonexistent bucket", "true"). + Set("rgw intent log object name utc", "true"). + Set("rgw enable usage log", "true"). + Set("rgw frontends", fmt.Sprintf("civetweb port=%s", c.portString())). + Set("rgw zone", c.store.Name). + Set("rgw zonegroup", c.store.Name) + return s +} + +func (c *clusterConfig) portString() string { + var portString string + port := c.store.Spec.Gateway.Port + if port != 0 { + portString = strconv.Itoa(int(port)) + } + if c.store.Spec.Gateway.SecurePort != 0 && c.store.Spec.Gateway.SSLCertificateRef != "" { + var separator string + if port != 0 { + separator = "+" + } + certPath := path.Join(certDir, certFilename) + // with ssl enabled, the port number must end with the letter s. + // e.g., "443s ssl_certificate=/etc/ceph/private/keyandcert.pem" + portString = fmt.Sprintf("%s%s%ds ssl_certificate=%s", + portString, separator, c.store.Spec.Gateway.SecurePort, certPath) + } + return portString +} + +func (c *clusterConfig) generateKeyring(replicationControllerOwnerRef *metav1.OwnerReference) error { + user := "client.radosgw.gateway" + /* TODO: this says `osd allow rwx` while template says `osd allow *`; which is correct? */ + access := []string{"osd", "allow rwx", "mon", "allow rw"} + s := keyring.GetSecretStore(c.context, c.store.Namespace, replicationControllerOwnerRef) + + key, err := s.GenerateKey(c.instanceName(), user, access) + if err != nil { + return err + } + + // Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x + err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Delete(c.instanceName(), &metav1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + logger.Debugf("legacy rgw key %s is already removed", c.instanceName()) + } else { + logger.Warningf("legacy rgw key %s could not be removed. %+v", c.instanceName(), err) + } + } + + keyring := fmt.Sprintf(keyringTemplate, key) + return s.CreateOrUpdate(c.instanceName(), keyring) +} diff --git a/pkg/operator/ceph/object/config_test.go b/pkg/operator/ceph/object/config_test.go new file mode 100644 index 000000000..086566bcb --- /dev/null +++ b/pkg/operator/ceph/object/config_test.go @@ -0,0 +1,66 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package object + +import ( + "testing" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/stretchr/testify/assert" +) + +func newConfig() *clusterConfig { + return &clusterConfig{ + store: cephv1.CephObjectStore{ + Spec: cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{}, + }}} +} + +func TestPortString(t *testing.T) { + // No port or secure port + cfg := newConfig() + result := cfg.portString() + assert.Equal(t, "", result) + + // Insecure port + cfg = newConfig() + cfg.store.Spec.Gateway.Port = 80 + result = cfg.portString() + assert.Equal(t, "80", result) + + // Secure port + cfg = newConfig() + cfg.store.Spec.Gateway.SecurePort = 443 + cfg.store.Spec.Gateway.SSLCertificateRef = "some-k8s-key-secret" + result = cfg.portString() + assert.Equal(t, "443s ssl_certificate=/etc/ceph/private/rgw-cert.pem", result) + + // Both ports + cfg = newConfig() + cfg.store.Spec.Gateway.Port = 80 + cfg.store.Spec.Gateway.SecurePort = 443 + cfg.store.Spec.Gateway.SSLCertificateRef = "some-k8s-key-secret" + result = cfg.portString() + assert.Equal(t, "80+443s ssl_certificate=/etc/ceph/private/rgw-cert.pem", result) + + // Secure port requires the cert + cfg = newConfig() + cfg.store.Spec.Gateway.SecurePort = 443 + result = cfg.portString() + assert.Equal(t, "", result) +} diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index 22aed6c1c..4ba616709 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -14,7 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package rgw to manage a rook object store. package object import ( @@ -26,6 +25,8 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" cephbeta "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1" "github.com/rook/rook/pkg/clusterd" + daemonconfig "github.com/rook/rook/pkg/daemon/ceph/config" + cephconfig "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/pool" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/api/errors" @@ -56,6 +57,7 @@ var ObjectStoreResourceRookLegacy = opkit.CustomResource{ // ObjectStoreController represents a controller object for object store custom resources type ObjectStoreController struct { + clusterInfo *daemonconfig.ClusterInfo context *clusterd.Context rookImage string cephVersion cephv1.CephVersionSpec @@ -64,8 +66,16 @@ type ObjectStoreController struct { } // NewObjectStoreController create controller for watching object store custom resources created -func NewObjectStoreController(context *clusterd.Context, rookImage string, cephVersion cephv1.CephVersionSpec, hostNetwork bool, ownerRef metav1.OwnerReference) *ObjectStoreController { +func NewObjectStoreController( + clusterInfo *daemonconfig.ClusterInfo, + context *clusterd.Context, + rookImage string, + cephVersion cephv1.CephVersionSpec, + hostNetwork bool, + ownerRef metav1.OwnerReference, +) *ObjectStoreController { return &ObjectStoreController{ + clusterInfo: clusterInfo, context: context, rookImage: rookImage, cephVersion: cephVersion, @@ -107,7 +117,16 @@ func (c *ObjectStoreController) onAdd(obj interface{}) { return } - cfg := config{c.context, *objectstore, c.rookImage, c.cephVersion, c.hostNetwork, c.storeOwners(objectstore)} + cfg := clusterConfig{ + clusterInfo: c.clusterInfo, + context: c.context, + store: *objectstore, + rookVersion: c.rookImage, + cephVersion: c.cephVersion, + hostNetwork: c.hostNetwork, + ownerRefs: c.storeOwners(objectstore), + DataPathMap: cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, objectstore.Name), + } if err = cfg.createStore(); err != nil { logger.Errorf("failed to create object store %s. %+v", objectstore.Name, err) } @@ -139,7 +158,16 @@ func (c *ObjectStoreController) onUpdate(oldObj, newObj interface{}) { } logger.Infof("applying object store %s changes", newStore.Name) - cfg := config{c.context, *newStore, c.rookImage, c.cephVersion, c.hostNetwork, c.storeOwners(newStore)} + cfg := clusterConfig{ + c.clusterInfo, + c.context, + *newStore, + c.rookImage, + c.cephVersion, + c.hostNetwork, + c.storeOwners(newStore), + cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, newStore.Name), + } if err = cfg.updateStore(); err != nil { logger.Errorf("failed to create (modify) object store %s. %+v", newStore.Name, err) } @@ -157,7 +185,7 @@ func (c *ObjectStoreController) onDelete(obj interface{}) { return } - cfg := config{context: c.context, store: *objectstore} + cfg := clusterConfig{context: c.context, store: *objectstore} if err = cfg.deleteStore(); err != nil { logger.Errorf("failed to delete object store %s. %+v", objectstore.Name, err) } diff --git a/pkg/operator/ceph/object/controller_test.go b/pkg/operator/ceph/object/controller_test.go index 235eed4dd..bdd0ba2fd 100644 --- a/pkg/operator/ceph/object/controller_test.go +++ b/pkg/operator/ceph/object/controller_test.go @@ -91,7 +91,8 @@ func TestMigrateObjectStoreObject(t *testing.T) { Clientset: clientset, RookClientset: rookfake.NewSimpleClientset(legacyObjectStore), } - controller := NewObjectStoreController(context, "", cephv1.CephVersionSpec{}, false, metav1.OwnerReference{}) + info := testop.CreateConfigDir(1) + controller := NewObjectStoreController(info, context, "", cephv1.CephVersionSpec{}, false, metav1.OwnerReference{}) // convert the legacy objectstore object in memory and assert that a migration is needed convertedObjectStore, migrationNeeded, err := getObjectStoreObject(legacyObjectStore) diff --git a/pkg/daemon/ceph/rgw/mime.go b/pkg/operator/ceph/object/mime.go similarity index 91% rename from pkg/daemon/ceph/rgw/mime.go rename to pkg/operator/ceph/object/mime.go index 28ce4120d..858c27a0d 100644 --- a/pkg/daemon/ceph/rgw/mime.go +++ b/pkg/operator/ceph/object/mime.go @@ -14,7 +14,67 @@ See the License for the specific language governing permissions and limitations under the License. */ -package rgw +package object + +import ( + "fmt" + "path" + + "github.com/rook/rook/pkg/operator/k8sutil" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // these control the location where the mime.types file is mounted in containers. + // Ceph's default is /etc/mime.types, but we can't mount a configmap to `/etc` without breaking + // the container. /etc/ceph/rgw won't overwrite ceph.conf (if it exists), and it will be easy to + // find manually by inspection of the `/etc/ceph` dir. + mimeTypesMountDir = "/etc/ceph/rgw" + mimeTypesFileName = "mime.types" +) + +func (c *clusterConfig) mimeTypesConfigMapName() string { + return fmt.Sprintf("%s-mime-types", c.instanceName()) +} + +func mimeTypesMountPath() string { + return path.Join(mimeTypesMountDir, mimeTypesFileName) +} + +// store mime.types file in a config map +func (c *clusterConfig) generateMimeTypes(ownerRef *metav1.OwnerReference) error { + k := k8sutil.NewConfigMapKVStore(c.store.Namespace, c.context.Clientset, *ownerRef) + if _, err := k.GetValue(c.mimeTypesConfigMapName(), mimeTypesFileName); err == nil || !errors.IsNotFound(err) { + logger.Infof("config map for object pool %s already exists, not overwriting", c.store.Name) + return nil + } + // is not found + if err := k.SetValue(c.mimeTypesConfigMapName(), mimeTypesFileName, mimeTypes); err != nil { + return fmt.Errorf("failed to create config map for object pool %s. %+v", c.store.Name, err) + } + return nil +} + +func (c *clusterConfig) mimeTypesVolume() v1.Volume { + return v1.Volume{ + Name: c.mimeTypesConfigMapName(), + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{LocalObjectReference: v1.LocalObjectReference{ + Name: c.mimeTypesConfigMapName(), + }}}} +} + +func (c *clusterConfig) mimeTypesVolumeMount() v1.VolumeMount { + return v1.VolumeMount{ + Name: c.mimeTypesConfigMapName(), + MountPath: mimeTypesMountDir, + ReadOnly: true, + // read-only so malicious actors can't edit, which could allow them to add file type + // execution vulnerabilities that an admin has taken care to eliminate + } +} const mimeTypes = ` application/activemessage diff --git a/pkg/daemon/ceph/rgw/objectstore.go b/pkg/operator/ceph/object/objectstore.go similarity index 94% rename from pkg/daemon/ceph/rgw/objectstore.go rename to pkg/operator/ceph/object/objectstore.go index 121da1b0a..ea047305f 100644 --- a/pkg/daemon/ceph/rgw/objectstore.go +++ b/pkg/operator/ceph/object/objectstore.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package rgw +package object import ( "encoding/json" @@ -27,7 +27,9 @@ import ( const ( rootPool = ".rgw.root" - appName = "rgw" + + // AppName is the name Rook uses for the object store's application + AppName = "rook-ceph-rgw" ) var ( @@ -51,7 +53,7 @@ type realmType struct { Realms []string `json:"realms"` } -func CreateObjectStore(context *Context, metadataSpec, dataSpec model.Pool, serviceIP string, port int32) error { +func createObjectStore(context *Context, metadataSpec, dataSpec model.Pool, serviceIP string, port int32) error { err := createPools(context, metadataSpec, dataSpec) if err != nil { return fmt.Errorf("failed to create object pools. %+v", err) @@ -64,8 +66,8 @@ func CreateObjectStore(context *Context, metadataSpec, dataSpec model.Pool, serv return nil } -func DeleteObjectStore(context *Context) error { - stores, err := GetObjectStores(context) +func deleteRealmAndPools(context *Context) error { + stores, err := getObjectStores(context) if err != nil { return fmt.Errorf("failed to detect object stores during deletion. %+v", err) } @@ -95,7 +97,7 @@ func createRealm(context *Context, serviceIP string, port int32) error { // The first realm must be marked as the default defaultArg := "" - stores, err := GetObjectStores(context) + stores, err := getObjectStores(context) if err != nil { return fmt.Errorf("failed to get object stores. %+v", err) } @@ -189,7 +191,7 @@ func decodeID(data string) (string, error) { return id.ID, err } -func GetObjectStores(context *Context) ([]string, error) { +func getObjectStores(context *Context) ([]string, error) { output, err := runAdminCommandNoRealm(context, "realm", "list") if err != nil { if strings.Index(err.Error(), "exit status 2") != 0 { @@ -274,9 +276,9 @@ func createSimilarPools(context *Context, pools []string, poolSpec model.Pool) e if isECPool { // An EC pool backing an object store does not need to enable EC overwrites, so the pool is // created with that property disabled to avoid unnecessary performance impact. - err = ceph.CreateECPoolForApp(context.context, context.ClusterName, cephConfig, appName, false /* enableECOverwrite */, poolSpec.ErasureCodedConfig) + err = ceph.CreateECPoolForApp(context.context, context.ClusterName, cephConfig, AppName, false /* enableECOverwrite */, poolSpec.ErasureCodedConfig) } else { - err = ceph.CreateReplicatedPoolForApp(context.context, context.ClusterName, cephConfig, appName) + err = ceph.CreateReplicatedPoolForApp(context.context, context.ClusterName, cephConfig, AppName) } if err != nil { return fmt.Errorf("failed to create pool %s for object store %s", name, context.Name) diff --git a/pkg/daemon/ceph/rgw/objectstore_test.go b/pkg/operator/ceph/object/objectstore_test.go similarity index 98% rename from pkg/daemon/ceph/rgw/objectstore_test.go rename to pkg/operator/ceph/object/objectstore_test.go index 6f398d4c3..278be2c15 100644 --- a/pkg/daemon/ceph/rgw/objectstore_test.go +++ b/pkg/operator/ceph/object/objectstore_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package rgw +package object import ( "fmt" @@ -43,9 +43,8 @@ func TestCreateRealm(t *testing.T) { } else if args[0] == "realm" && args[1] == "list" { if defaultStore { return "", fmt.Errorf("failed to run radosgw-admin: Failed to complete : exit status 2") - } else { - return `{"realms": ["myobj"]}`, nil } + return `{"realms": ["myobj"]}`, nil } return idResponse, nil } @@ -146,7 +145,7 @@ func deleteStore(t *testing.T, name string, existingStores string, expectedDelet context := &Context{context: &clusterd.Context{Executor: executor}, Name: "myobj", ClusterName: "ns"} // Delete an object store - err := DeleteObjectStore(context) + err := deleteRealmAndPools(context) assert.Nil(t, err) expectedPoolsDeleted := 5 if expectedDeleteRootPool { diff --git a/pkg/operator/ceph/object/rgw.go b/pkg/operator/ceph/object/rgw.go index 3c4397b5a..0a15f4107 100644 --- a/pkg/operator/ceph/object/rgw.go +++ b/pkg/operator/ceph/object/rgw.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package rgw for the Ceph object store. +// Package object for the Ceph object store. package object import ( @@ -22,43 +22,36 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" - "github.com/rook/rook/pkg/daemon/ceph/client" - rgwdaemon "github.com/rook/rook/pkg/daemon/ceph/rgw" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/pool" "github.com/rook/rook/pkg/operator/k8sutil" - "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) -const ( - appName = "rook-ceph-rgw" - keyringName = "keyring" - certVolumeName = "rook-rgw-cert" - certMountPath = "/etc/rook/private" - certKeyName = "cert" - certFilename = "rgw-cert.pem" -) - -type config struct { +type clusterConfig struct { + clusterInfo *cephconfig.ClusterInfo context *clusterd.Context store cephv1.CephObjectStore rookVersion string cephVersion cephv1.CephVersionSpec hostNetwork bool ownerRefs []metav1.OwnerReference + DataPathMap *config.DataPathMap } // Start the rgw manager -func (c *config) createStore() error { +func (c *clusterConfig) createStore() error { return c.createOrUpdate(false) } -func (c *config) updateStore() error { +func (c *clusterConfig) updateStore() error { return c.createOrUpdate(true) } -func (c *config) createOrUpdate(update bool) error { +func (c *clusterConfig) createOrUpdate(update bool) error { // validate the object store settings if err := validateStore(c.context, c.store); err != nil { return fmt.Errorf("invalid object store %s arguments. %+v", c.store.Name, err) @@ -75,10 +68,6 @@ func (c *config) createOrUpdate(update bool) error { } logger.Infof("creating object store %s in namespace %s", c.store.Name, c.store.Namespace) - err = c.createKeyring() - if err != nil { - return fmt.Errorf("failed to create rgw keyring. %+v", err) - } // start the service serviceIP, err := c.startService() @@ -87,8 +76,8 @@ func (c *config) createOrUpdate(update bool) error { } // create the ceph artifacts for the object store - objContext := rgwdaemon.NewContext(c.context, c.store.Name, c.store.Namespace) - err = rgwdaemon.CreateObjectStore(objContext, *c.store.Spec.MetadataPool.ToModel(""), *c.store.Spec.DataPool.ToModel(""), serviceIP, c.store.Spec.Gateway.Port) + objContext := NewContext(c.context, c.store.Name, c.store.Namespace) + err = createObjectStore(objContext, *c.store.Spec.MetadataPool.ToModel(""), *c.store.Spec.DataPool.ToModel(""), serviceIP, c.store.Spec.Gateway.Port) if err != nil { return fmt.Errorf("failed to create pools. %+v", err) } @@ -101,7 +90,7 @@ func (c *config) createOrUpdate(update bool) error { return nil } -func (c *config) startRGWPods(update bool) error { +func (c *clusterConfig) startRGWPods(update bool) error { // if intended to update, remove the old pods so they can be created with the new spec settings if update { @@ -116,15 +105,49 @@ func (c *config) startRGWPods(update bool) error { } // start the deployment or daemonset + var uid types.UID + var controllerType string if c.store.Spec.Gateway.AllNodes { - return c.startDaemonset() + daemonSet, err := c.startDaemonset() + if err != nil { + return err + } + uid = daemonSet.UID + controllerType = "daemonset" + } else { + deployment, err := c.startDeployment() + if err != nil { + return err + } + uid = deployment.UID + controllerType = "deployment" } - return c.startDeployment() + + resourceControllerOwnerRef := &metav1.OwnerReference{ + UID: uid, + APIVersion: "v1", + Kind: controllerType, + Name: c.instanceName(), + } + + // Generate the keyring after starting the replication controller so that the keyring may use + // the controller as its owner reference; the keyring is deleted with the controller + err := c.generateKeyring(resourceControllerOwnerRef) + if err != nil { + return fmt.Errorf("failed to create rgw keyring. %+v", err) + } + + // Generate the mime.types file after the rep. controller as well for the same reason as keyring + if err := c.generateMimeTypes(resourceControllerOwnerRef); err != nil { + return fmt.Errorf("failed to generate the rgw mime.types config. %+v", err) + } + + return nil } // Delete the object store. // WARNING: This is a very destructive action that deletes all metadata and data pools. -func (c *config) deleteStore() error { +func (c *clusterConfig) deleteStore() error { // check if the object store exists exists, err := c.storeExists() if err != nil { @@ -164,8 +187,8 @@ func (c *config) deleteStore() error { } // Delete the realm and pools - objContext := rgwdaemon.NewContext(c.context, c.store.Name, c.store.Namespace) - err = rgwdaemon.DeleteObjectStore(objContext) + objContext := NewContext(c.context, c.store.Name, c.store.Namespace) + err = deleteRealmAndPools(objContext) if err != nil { return fmt.Errorf("failed to delete the realm and pools. %+v", err) } @@ -175,7 +198,7 @@ func (c *config) deleteStore() error { } // Check if the object store exists depending on either the deployment or the daemonset -func (c *config) storeExists() (bool, error) { +func (c *clusterConfig) storeExists() (bool, error) { _, err := c.context.Clientset.Apps().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) if err == nil { // the deployment was found @@ -198,60 +221,8 @@ func (c *config) storeExists() (bool, error) { return false, nil } -func (c *config) createKeyring() error { - _, err := c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) - if err == nil { - logger.Infof("the rgw keyring was already generated") - return nil - } - if !errors.IsNotFound(err) { - return fmt.Errorf("failed to get rgw secrets. %+v", err) - } - - // create the keyring - logger.Infof("generating rgw keyring") - keyring, err := c.createRGWKeyring() - if err != nil { - return fmt.Errorf("failed to create keyring. %+v", err) - } - - // store the secrets - secrets := map[string]string{ - keyringName: keyring, - } - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: c.instanceName(), - Namespace: c.store.Namespace, - }, - StringData: secrets, - Type: k8sutil.RookType, - } - k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &secret.ObjectMeta, c.ownerRefs) - _, err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Create(secret) - if err != nil { - return fmt.Errorf("failed to save rgw secrets. %+v", err) - } - - return nil -} - -func (c *config) instanceName() string { - return fmt.Sprintf("%s-%s", appName, c.store.Name) -} - -// create a keyring for the rgw client with a limited set of privileges -func (c *config) createRGWKeyring() (string, error) { - username := "client.radosgw.gateway" - access := []string{"osd", "allow rwx", "mon", "allow rw"} - - // get-or-create-key for the user account - key, err := client.AuthGetOrCreateKey(c.context, c.store.Namespace, username, access) - if err != nil { - return "", fmt.Errorf("failed to get or create auth key for %store. %+v", username, err) - } - - return key, err +func (c *clusterConfig) instanceName() string { + return fmt.Sprintf("%s-%s", AppName, c.store.Name) } // Validate the object store arguments diff --git a/pkg/operator/ceph/object/rgw_test.go b/pkg/operator/ceph/object/rgw_test.go index b3da12886..3c224da92 100644 --- a/pkg/operator/ceph/object/rgw_test.go +++ b/pkg/operator/ceph/object/rgw_test.go @@ -23,6 +23,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + cephconfig "github.com/rook/rook/pkg/operator/ceph/config" testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" @@ -44,12 +45,14 @@ func TestStartRGW(t *testing.T) { configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) + info := testop.CreateConfigDir(1) context := &clusterd.Context{Clientset: clientset, Executor: executor, ConfigDir: configDir} store := simpleStore() version := "v1.1.0" + data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "my-fs") // start a basic cluster - c := &config{context, store, version, cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}} + c := &clusterConfig{info, context, store, version, cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}, data} err := c.createStore() assert.Nil(t, err) @@ -63,7 +66,7 @@ func TestStartRGW(t *testing.T) { validateStart(t, c, clientset, true) } -func validateStart(t *testing.T, c *config, clientset *fake.Clientset, allNodes bool) { +func validateStart(t *testing.T, c *clusterConfig, clientset *fake.Clientset, allNodes bool) { if !allNodes { r, err := clientset.Apps().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) assert.Nil(t, err) @@ -83,11 +86,6 @@ func validateStart(t *testing.T, c *config, clientset *fake.Clientset, allNodes s, err := clientset.CoreV1().Services(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) assert.Nil(t, err) assert.Equal(t, c.instanceName(), s.Name) - - secret, err := clientset.CoreV1().Secrets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) - assert.Nil(t, err) - assert.Equal(t, c.instanceName(), secret.Name) - assert.Equal(t, 1, len(secret.StringData)) } func TestCreateObjectStore(t *testing.T) { @@ -114,9 +112,11 @@ func TestCreateObjectStore(t *testing.T) { store := simpleStore() clientset := testop.New(3) context := &clusterd.Context{Executor: executor, Clientset: clientset} + info := testop.CreateConfigDir(1) + data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "my-fs") // create the pools - c := &config{context, store, "1.2.3.4", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}} + c := &clusterConfig{info, context, store, "1.2.3.4", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{}, data} err := c.createStore() assert.Nil(t, err) } diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 93521abd2..7e745c73e 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -14,15 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package object for the Ceph object store. package object import ( "fmt" - "path" - rgwdaemon "github.com/rook/rook/pkg/daemon/ceph/rgw" - opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon" + cephconfig "github.com/rook/rook/pkg/operator/ceph/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" @@ -32,11 +29,12 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) -func (c *config) startDeployment() error { +func (c *clusterConfig) startDeployment() (*apps.Deployment, error) { d := &apps.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: c.instanceName(), Namespace: c.store.Namespace, + Labels: c.getLabels(), }, Spec: apps.DeploymentSpec{ Selector: &metav1.LabelSelector{ @@ -51,29 +49,29 @@ func (c *config) startDeployment() error { } k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &d.ObjectMeta, c.ownerRefs) - logger.Debugf("starting mds deployment: %+v", d) - _, err := c.context.Clientset.Apps().Deployments(c.store.Namespace).Create(d) + logger.Debugf("starting rgw deployment: %+v", d) + deployment, err := c.context.Clientset.Apps().Deployments(c.store.Namespace).Create(d) if err != nil { if !errors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create rgw deployment %s: %+v", c.instanceName(), err) + return nil, fmt.Errorf("failed to create rgw deployment %s: %+v", c.instanceName(), err) } logger.Infof("deployment for rgw %s already exists. updating if needed", c.instanceName()) // There may be a *lot* of rgws, and they are stateless, so don't bother waiting until the // entire deployment is updated to move on. - _, err := c.context.Clientset.Apps().Deployments(c.store.Namespace).Update(d) + deployment, err = c.context.Clientset.Apps().Deployments(c.store.Namespace).Update(d) if err != nil { - return fmt.Errorf("failed to update rgw deployment %s. %+v", c.instanceName(), err) + return nil, fmt.Errorf("failed to update rgw deployment %s. %+v", c.instanceName(), err) } } - - return nil + return deployment, nil } -func (c *config) startDaemonset() error { +func (c *clusterConfig) startDaemonset() (*apps.DaemonSet, error) { d := &apps.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: c.instanceName(), Namespace: c.store.Namespace, + Labels: c.getLabels(), }, Spec: apps.DaemonSetSpec{ Selector: &metav1.LabelSelector{ @@ -88,36 +86,36 @@ func (c *config) startDaemonset() error { k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &d.ObjectMeta, c.ownerRefs) logger.Debugf("starting rgw daemonset: %+v", d) - _, err := c.context.Clientset.Apps().DaemonSets(c.store.Namespace).Create(d) + daemonSet, err := c.context.Clientset.Apps().DaemonSets(c.store.Namespace).Create(d) if err != nil { if !errors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create rgw daemonset %s: %+v", c.instanceName(), err) + return nil, fmt.Errorf("failed to create rgw daemonset %s: %+v", c.instanceName(), err) } logger.Infof("daemonset for rgw %s already exists. updating if needed", c.instanceName()) // There may be a *lot* of rgws, and they are stateless, so don't bother waiting until the // entire daemonset is updated to move on. // TODO: is the above statement safe to assume? // TODO: Are there any steps for RGW that need to happen before the daemons upgrade? - _, err := c.context.Clientset.Apps().DaemonSets(c.store.Namespace).Update(d) + daemonSet, err = c.context.Clientset.Apps().DaemonSets(c.store.Namespace).Update(d) if err != nil { - return fmt.Errorf("failed to update rgw daemonset %s. %+v", c.instanceName(), err) + return nil, fmt.Errorf("failed to update rgw daemonset %s. %+v", c.instanceName(), err) } } - - return nil + return daemonSet, nil } -func (c *config) makeRGWPodSpec() v1.PodTemplateSpec { +func (c *clusterConfig) makeRGWPodSpec() v1.PodTemplateSpec { podSpec := v1.PodSpec{ - InitContainers: []v1.Container{ - c.makeConfigInitContainer(), - }, + InitContainers: []v1.Container{}, Containers: []v1.Container{ c.makeDaemonContainer(), }, RestartPolicy: v1.RestartPolicyAlways, - Volumes: opspec.PodVolumes(""), - HostNetwork: c.hostNetwork, + Volumes: append( + opspec.DaemonVolumes(c.DataPathMap, c.instanceName()), + c.mimeTypesVolume(), + ), + HostNetwork: c.hostNetwork, } if c.hostNetwork { podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet @@ -125,10 +123,16 @@ func (c *config) makeRGWPodSpec() v1.PodTemplateSpec { // Set the ssl cert if specified if c.store.Spec.Gateway.SSLCertificateRef != "" { - certVol := v1.Volume{Name: certVolumeName, VolumeSource: v1.VolumeSource{Secret: &v1.SecretVolumeSource{ - SecretName: c.store.Spec.Gateway.SSLCertificateRef, - Items: []v1.KeyToPath{{Key: certKeyName, Path: certFilename}}, - }}} + // Keep the SSL secret as secure as possible in the container. Give only user read perms. + userReadOnly := int32(0400) + certVol := v1.Volume{ + Name: certVolumeName, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: c.store.Spec.Gateway.SSLCertificateRef, + Items: []v1.KeyToPath{ + {Key: certKeyName, Path: certFilename, Mode: &userReadOnly}, + }}}} podSpec.Volumes = append(podSpec.Volumes, certVol) } @@ -144,40 +148,7 @@ func (c *config) makeRGWPodSpec() v1.PodTemplateSpec { } } -func (c *config) makeConfigInitContainer() v1.Container { - container := v1.Container{ - Name: opspec.ConfigInitContainerName, - Image: k8sutil.MakeRookImage(c.rookVersion), - Args: []string{ - "ceph", - "rgw", - fmt.Sprintf("--config-dir=%s", k8sutil.DataDir), - fmt.Sprintf("--rgw-name=%s", c.store.Name), - fmt.Sprintf("--rgw-port=%d", c.store.Spec.Gateway.Port), - fmt.Sprintf("--rgw-secure-port=%d", c.store.Spec.Gateway.SecurePort), - }, - VolumeMounts: opspec.RookVolumeMounts(), - Env: []v1.EnvVar{ - {Name: "ROOK_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: c.instanceName()}, Key: keyringName}}}, - k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), - k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), - opmon.ClusterNameEnvVar(c.store.Namespace), - opmon.EndpointEnvVar(), - opmon.SecretEnvVar(), - k8sutil.ConfigOverrideEnvVar(), - }, - Resources: c.store.Spec.Gateway.Resources, - } - - if c.store.Spec.Gateway.SSLCertificateRef != "" { - path := path.Join(certMountPath, certFilename) - container.Args = append(container.Args, fmt.Sprintf("--rgw-cert=%s", path)) - } - - return container -} - -func (c *config) makeDaemonContainer() v1.Container { +func (c *clusterConfig) makeDaemonContainer() v1.Container { // start the rgw daemon in the foreground container := v1.Container{ @@ -186,26 +157,33 @@ func (c *config) makeDaemonContainer() v1.Container { Command: []string{ "radosgw", }, - Args: []string{ - "--foreground", - "--name=client.radosgw.gateway", - fmt.Sprintf("--rgw-mime-types-file=%s", rgwdaemon.GetMimeTypesPath(k8sutil.DataDir)), - }, - VolumeMounts: opspec.CephVolumeMounts(), - Env: k8sutil.ClusterDaemonEnvVars(c.cephVersion.Image), - Resources: c.store.Spec.Gateway.Resources, + Args: append( + append( + opspec.DaemonFlags(c.clusterInfo, cephconfig.RgwType, c.store.Name), + "--foreground", + "--name=client.radosgw.gateway", + cephconfig.NewFlag("host", opspec.ContainerEnvVarReference("POD_NAME")), + cephconfig.NewFlag("rgw-mime-types-file", mimeTypesMountPath()), + ), c.defaultSettings().GlobalFlags()..., // use default settings as flags until mon kv store supported + ), + VolumeMounts: append( + opspec.DaemonVolumeMounts(c.DataPathMap, c.instanceName()), + c.mimeTypesVolumeMount(), + ), + Env: opspec.DaemonEnvVars(c.cephVersion.Image), + Resources: c.store.Spec.Gateway.Resources, } if c.store.Spec.Gateway.SSLCertificateRef != "" { // Add a volume mount for the ssl certificate - mount := v1.VolumeMount{Name: certVolumeName, MountPath: certMountPath, ReadOnly: true} + mount := v1.VolumeMount{Name: certVolumeName, MountPath: certDir, ReadOnly: true} container.VolumeMounts = append(container.VolumeMounts, mount) } return container } -func (c *config) startService() (string, error) { +func (c *clusterConfig) startService() (string, error) { labels := c.getLabels() svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -253,10 +231,8 @@ func addPort(service *v1.Service, name string, port int32) { }) } -func (c *config) getLabels() map[string]string { - return map[string]string{ - k8sutil.AppAttr: appName, - k8sutil.ClusterAttr: c.store.Namespace, - "rook_object_store": c.store.Name, - } +func (c *clusterConfig) getLabels() map[string]string { + labels := opspec.PodLabels(AppName, c.store.Namespace, "rgw", c.store.Name) + labels["rook_object_store"] = c.store.Name + return labels } diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index a6521bb50..f1f5a1c53 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -17,13 +17,13 @@ limitations under the License. package object import ( - "fmt" "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" - cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" - "github.com/rook/rook/pkg/operator/k8sutil" + cephconfig "github.com/rook/rook/pkg/operator/ceph/config" + cephtest "github.com/rook/rook/pkg/operator/ceph/test" + testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" @@ -40,80 +40,59 @@ func TestPodSpecs(t *testing.T) { v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI), }, } + info := testop.CreateConfigDir(1) + data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "default") + + c := &clusterConfig{ + clusterInfo: info, + store: store, + rookVersion: "rook/rook:myversion", + cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.1", Name: "mimic"}, + hostNetwork: true, + DataPathMap: data, + } - c := &config{store: store, rookVersion: "rook/rook:myversion", cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.1", Name: "mimic"}, hostNetwork: true} s := c.makeRGWPodSpec() - assert.NotNil(t, s) - //assert.Equal(t, instanceName(store), s.Name) - assert.Equal(t, v1.RestartPolicyAlways, s.Spec.RestartPolicy) - assert.Equal(t, 3, len(s.Spec.Volumes)) - assert.Equal(t, "rook-data", s.Spec.Volumes[0].Name) - assert.Equal(t, cephconfig.DefaultConfigMountName, s.Spec.Volumes[1].Name) - assert.Equal(t, c.instanceName(), s.ObjectMeta.Name) - assert.Equal(t, appName, s.ObjectMeta.Labels["app"]) - assert.Equal(t, store.Namespace, s.ObjectMeta.Labels["rook_cluster"]) - assert.Equal(t, store.Name, s.ObjectMeta.Labels["rook_object_store"]) - assert.Equal(t, 0, len(s.ObjectMeta.Annotations)) - - cont := s.Spec.InitContainers[0] - assert.Equal(t, 1, len(s.Spec.InitContainers)) - assert.Equal(t, "rook/rook:myversion", cont.Image) - assert.Equal(t, 3, len(cont.VolumeMounts)) - assert.Equal(t, 0, len(cont.Command)) - assert.Equal(t, 6, len(cont.Args)) - assert.Equal(t, "ceph", cont.Args[0]) - assert.Equal(t, "rgw", cont.Args[1]) - assert.Equal(t, "--config-dir=/var/lib/rook", cont.Args[2]) - assert.Equal(t, "--rgw-name=default", cont.Args[3]) - assert.Equal(t, "--rgw-port=123", cont.Args[4]) - assert.Equal(t, "--rgw-secure-port=0", cont.Args[5]) - - cont = s.Spec.Containers[0] - assert.Equal(t, "ceph/ceph:v13.2.1", cont.Image) - assert.Equal(t, 2, len(cont.VolumeMounts)) - - assert.Equal(t, 1, len(cont.Command)) - assert.Equal(t, "radosgw", cont.Command[0]) - assert.Equal(t, 3, len(cont.Args)) - assert.Equal(t, "--foreground", cont.Args[0]) - assert.Equal(t, "--name=client.radosgw.gateway", cont.Args[1]) - assert.Equal(t, "--rgw-mime-types-file=/var/lib/rook/rgw/mime.types", cont.Args[2]) - - assert.Equal(t, len(k8sutil.ClusterDaemonEnvVars("ceph/ceph:v14")), len(cont.Env)) - - assert.Equal(t, "100", cont.Resources.Limits.Cpu().String()) - assert.Equal(t, "1337", cont.Resources.Requests.Memory().String()) + podTemplate := cephtest.NewPodTemplateSpecTester(t, &s) + podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion", + "100", "1337" /* resources */) } func TestSSLPodSpec(t *testing.T) { store := simpleStore() + store.Spec.Gateway.Resources = v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(100.0, resource.BinarySI), + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI), + }, + } + info := testop.CreateConfigDir(1) + data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "default") store.Spec.Gateway.SSLCertificateRef = "mycert" store.Spec.Gateway.SecurePort = 443 - c := &config{store: store, rookVersion: "v1.0", hostNetwork: true} + c := &clusterConfig{ + clusterInfo: info, + store: store, + rookVersion: "rook/rook:myversion", + cephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v13.2.1", Name: "mimic"}, + hostNetwork: true, + DataPathMap: data, + } + c.hostNetwork = true + s := c.makeRGWPodSpec() - assert.NotNil(t, s) - assert.Equal(t, c.instanceName(), s.Name) - assert.Equal(t, 4, len(s.Spec.Volumes)) - assert.Equal(t, certVolumeName, s.Spec.Volumes[3].Name) + podTemplate := cephtest.NewPodTemplateSpecTester(t, &s) + podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion", + "100", "1337" /* resources */) + assert.True(t, s.Spec.HostNetwork) assert.Equal(t, v1.DNSClusterFirstWithHostNet, s.Spec.DNSPolicy) - initCont := s.Spec.InitContainers[0] - - assert.Equal(t, 3, len(initCont.VolumeMounts)) - assert.Equal(t, 7, len(initCont.Args)) - assert.Equal(t, fmt.Sprintf("--rgw-secure-port=%d", 443), initCont.Args[5]) - assert.Equal(t, fmt.Sprintf("--rgw-cert=%s/%s", certMountPath, certFilename), initCont.Args[6]) - - rgwCont := s.Spec.Containers[0] - - assert.Equal(t, 3, len(rgwCont.VolumeMounts)) - assert.Equal(t, certVolumeName, rgwCont.VolumeMounts[2].Name) - assert.Equal(t, certMountPath, rgwCont.VolumeMounts[2].MountPath) - assert.Equal(t, 3, len(rgwCont.Args)) } func TestValidateSpec(t *testing.T) { diff --git a/pkg/daemon/ceph/rgw/user.go b/pkg/operator/ceph/object/user.go similarity index 91% rename from pkg/daemon/ceph/rgw/user.go rename to pkg/operator/ceph/object/user.go index be83aab5d..f1eb3f999 100644 --- a/pkg/daemon/ceph/rgw/user.go +++ b/pkg/operator/ceph/object/user.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package rgw +package object import ( "encoding/json" @@ -24,13 +24,14 @@ import ( ) const ( - RGWErrorNone = iota - RGWErrorUnknown = iota - RGWErrorNotFound = iota - RGWErrorBadData = iota - RGWErrorParse = iota + RGWErrorNone = iota + RGWErrorUnknown + RGWErrorNotFound + RGWErrorBadData + RGWErrorParse ) +// An ObjectUser defines the details of an object store user. type ObjectUser struct { UserID string `json:"userId"` DisplayName *string `json:"displayName"` @@ -39,6 +40,7 @@ type ObjectUser struct { SecretKey *string `json:"secretKey"` } +// ListUsers lists the object pool users. func ListUsers(c *Context) ([]string, int, error) { result, err := runAdminCommand(c, "user", "list") if err != nil { @@ -80,6 +82,7 @@ func decodeUser(data string) (*ObjectUser, int, error) { return &rookUser, RGWErrorNone, nil } +// GetUser returns the user with the given ID. func GetUser(c *Context, id string) (*ObjectUser, int, error) { logger.Infof("Getting user: %s", id) @@ -93,6 +96,7 @@ func GetUser(c *Context, id string) (*ObjectUser, int, error) { return decodeUser(result) } +// CreateUser creates a new user with the information given. func CreateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) { logger.Infof("Creating user: %s", user.UserID) @@ -131,6 +135,7 @@ func CreateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) { return decodeUser(result) } +// UpdateUser updates the user whose ID matches the user. func UpdateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) { logger.Infof("Updating user: %s", user.UserID) @@ -155,6 +160,7 @@ func UpdateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) { return decodeUser(body) } +// DeleteUser deletes the user with the given ID. func DeleteUser(c *Context, id string) (string, int, error) { logger.Infof("Deleting user: %s", id) result, err := runAdminCommand(c, "user", "rm", "--uid", id) diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go index d6d4d4498..87f99e651 100644 --- a/pkg/operator/ceph/object/user/controller.go +++ b/pkg/operator/ceph/object/user/controller.go @@ -25,7 +25,7 @@ import ( opkit "github.com/rook/operator-kit" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" - cephrgw "github.com/rook/rook/pkg/daemon/ceph/rgw" + "github.com/rook/rook/pkg/operator/ceph/object" "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/api/core/v1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" @@ -34,12 +34,12 @@ import ( ) const ( - AppName = "rook-ceph-rgw" + appName = object.AppName ) var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-object") -// ObjectStoreResource represents the object store user custom resource +// ObjectStoreUserResource represents the object store user custom resource var ObjectStoreUserResource = opkit.CustomResource{ Name: "cephobjectstoreuser", Plural: "cephobjectstoreusers", @@ -139,13 +139,13 @@ func (c *ObjectStoreUserController) createUser(context *clusterd.Context, u *cep // create the user logger.Infof("creating user %s in namespace %s", u.Name, u.Namespace) - userConfig := cephrgw.ObjectUser{ + userConfig := object.ObjectUser{ UserID: u.Name, DisplayName: &displayName, } - objContext := cephrgw.NewContext(context, u.Spec.Store, u.Namespace) + objContext := object.NewContext(context, u.Spec.Store, u.Namespace) - user, rgwerr, err := cephrgw.CreateUser(objContext, userConfig) + user, rgwerr, err := object.CreateUser(objContext, userConfig) if err != nil { return fmt.Errorf("failed to create user %s. RadosGW returned error %d: %+v", u.Name, rgwerr, err) } @@ -160,7 +160,7 @@ func (c *ObjectStoreUserController) createUser(context *clusterd.Context, u *cep Name: fmt.Sprintf("rook-ceph-object-user-%s-%s", u.Spec.Store, u.Name), Namespace: u.Namespace, Labels: map[string]string{ - "app": AppName, + "app": appName, "user": u.Name, "rook_cluster": u.Namespace, "rook_object_store": u.Spec.Store, @@ -181,8 +181,8 @@ func (c *ObjectStoreUserController) createUser(context *clusterd.Context, u *cep // Delete the user func deleteUser(context *clusterd.Context, u *cephv1.CephObjectStoreUser) error { - objContext := cephrgw.NewContext(context, u.Spec.Store, u.Namespace) - _, rgwerr, err := cephrgw.DeleteUser(objContext, u.Name) + objContext := object.NewContext(context, u.Spec.Store, u.Namespace) + _, rgwerr, err := object.DeleteUser(objContext, u.Name) if err != nil { if rgwerr == 3 { logger.Infof("user %s does not exist in store %s", u.Name, u.Spec.Store) @@ -200,7 +200,7 @@ func deleteUser(context *clusterd.Context, u *cephv1.CephObjectStoreUser) error return nil } -// Validate the user arguments +// ValidateUser validates the user arguments func ValidateUser(context *clusterd.Context, u *cephv1.CephObjectStoreUser) error { if u.Name == "" { return fmt.Errorf("missing name") diff --git a/pkg/operator/ceph/spec/spec.go b/pkg/operator/ceph/spec/spec.go index 52c66e20d..e9e15d919 100644 --- a/pkg/operator/ceph/spec/spec.go +++ b/pkg/operator/ceph/spec/spec.go @@ -95,9 +95,9 @@ func DaemonVolumeMounts(dataPaths *config.DataPathMap, keyringResourceName strin func DaemonFlags(cluster *cephconfig.ClusterInfo, daemonType config.DaemonType, daemonID string) []string { return append( config.DefaultFlags(cluster.FSID, keyring.VolumeMount().KeyringFilePath()), - // all daemons are named in the format . - config.NewFlag("name", fmt.Sprintf("%s.%s", string(daemonType), daemonID)), + config.NewFlag("id", daemonID), ) + } // AdminFlags returns the command line flags used for Ceph commands requiring admin authentication. diff --git a/tests/framework/clients/object_user.go b/tests/framework/clients/object_user.go index aefe5f948..547f046f2 100644 --- a/tests/framework/clients/object_user.go +++ b/tests/framework/clients/object_user.go @@ -18,7 +18,8 @@ package clients import ( "fmt" - "github.com/rook/rook/pkg/daemon/ceph/rgw" + + rgw "github.com/rook/rook/pkg/operator/ceph/object" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" ) diff --git a/tests/integration/base_object_test.go b/tests/integration/base_object_test.go index 24be17ac2..51b19d7c9 100644 --- a/tests/integration/base_object_test.go +++ b/tests/integration/base_object_test.go @@ -19,13 +19,14 @@ package integration import ( "errors" - rgwdaemon "github.com/rook/rook/pkg/daemon/ceph/rgw" + "time" + + rgw "github.com/rook/rook/pkg/operator/ceph/object" "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "time" ) var ( @@ -167,17 +168,17 @@ func objectTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, na helper.ObjectClient.DeleteUser(storeName, userid)*/ } -func getBucket(bucketname string, bucketList []rgwdaemon.ObjectBucket) (rgwdaemon.ObjectBucket, error) { +func getBucket(bucketname string, bucketList []rgw.ObjectBucket) (rgw.ObjectBucket, error) { for _, bucket := range bucketList { if bucket.Name == bucketname { return bucket, nil } } - return rgwdaemon.ObjectBucket{}, errors.New("Bucket not found") + return rgw.ObjectBucket{}, errors.New("Bucket not found") } -func getBucketSizeAndObjects(bucketname string, bucketList []rgwdaemon.ObjectBucket) (uint64, uint64, error) { +func getBucketSizeAndObjects(bucketname string, bucketList []rgw.ObjectBucket) (uint64, uint64, error) { bkt, err := getBucket(bucketname, bucketList) if err != nil { return 0, 0, errors.New("Bucket not found")