diff --git a/cluster/charts/rook-ceph/templates/deployment.yaml b/cluster/charts/rook-ceph/templates/deployment.yaml index 9e6fcde5e..5ffc90367 100644 --- a/cluster/charts/rook-ceph/templates/deployment.yaml +++ b/cluster/charts/rook-ceph/templates/deployment.yaml @@ -169,7 +169,7 @@ spec: {{- if .Values.csi.allowUnsupportedVersion }} - name: ROOK_CSI_ALLOW_UNSUPPORTED_VERSION value: {{ .Values.csi.allowUnsupportedVersion | quote }} -{{- end }} +{{- end }} {{- if .Values.csi.pluginTolerations }} - name: CSI_PLUGIN_TOLERATIONS value: {{ toYaml .Values.csi.pluginTolerations | quote }} @@ -223,6 +223,8 @@ spec: value: "{{ .Values.enableFlexDriver }}" - name: ROOK_ENABLE_DISCOVERY_DAEMON value: "{{ .Values.enableDiscoveryDaemon }}" + - name: ROOK_OBC_WATCH_OPERATOR_NAMESPACE + value: "{{ .Values.enableOBCWatchOperatorNamespace }}" - name: NODE_NAME valueFrom: diff --git a/cluster/charts/rook-ceph/values.yaml b/cluster/charts/rook-ceph/values.yaml index dcd6d29ef..4aa46a9fd 100644 --- a/cluster/charts/rook-ceph/values.yaml +++ b/cluster/charts/rook-ceph/values.yaml @@ -312,3 +312,6 @@ discoverDaemonUdev: # imagePullSecrets option allow to pull docker images from private docker registry. Option will be passed to all service accounts. # imagePullSecrets: # - name: my-registry-secret + +# Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used +enableOBCWatchOperatorNamespace: true \ No newline at end of file diff --git a/cluster/examples/kubernetes/ceph/create-external-cluster-resources.sh b/cluster/examples/kubernetes/ceph/create-external-cluster-resources.sh index 5e7cea591..bfb6ca0c9 100644 --- a/cluster/examples/kubernetes/ceph/create-external-cluster-resources.sh +++ b/cluster/examples/kubernetes/ceph/create-external-cluster-resources.sh @@ -9,6 +9,7 @@ set -Eeuo pipefail ############# : "${CLIENT_CHECKER_NAME:=client.healthchecker}" +: "${RGW_POOL_PREFIX:=default}" ############# # FUNCTIONS # @@ -31,7 +32,7 @@ fi } function createCheckerKey() { - checkerKey=$(ceph auth get-or-create "$CLIENT_CHECKER_NAME" mon 'allow r, allow command quorum_status'|awk '/key =/ { print $3}') + checkerKey=$(ceph auth get-or-create "$CLIENT_CHECKER_NAME" mon 'allow r, allow command quorum_status' osd 'allow rwx pool='"$RGW_POOL_PREFIX"'.rgw.meta, allow r pool=.rgw.root, allow rw pool='"$RGW_POOL_PREFIX"'.rgw.control, allow x pool='"$RGW_POOL_PREFIX"'.rgw.buckets.index'|awk '/key =/ { print $3}') echo "export ROOK_EXTERNAL_USER_SECRET=$checkerKey" echo "export ROOK_EXTERNAL_USERNAME=$CLIENT_CHECKER_NAME" } diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index 81691e553..27a5a7941 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -85,7 +85,7 @@ data: # - effect: NoExecute # key: node-role.kubernetes.io/etcd # operator: Exists - + # (Optional) CEPH CSI RBD provisioner resource requirement list, Put here list of resource # requests and limits you want to apply for provisioner pod # CSI_RBD_PROVISIONER_RESOURCE: | @@ -234,13 +234,16 @@ data: # limits: # memory: 256Mi # cpu: 100m - + # Configure CSI CSI Ceph FS grpc and liveness metrics port # CSI_CEPHFS_GRPC_METRICS_PORT: "9091" # CSI_CEPHFS_LIVENESS_METRICS_PORT: "9081" # Configure CSI RBD grpc and liveness metrics port # CSI_RBD_GRPC_METRICS_PORT: "9090" # CSI_RBD_LIVENESS_METRICS_PORT: "9080" + + # Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used + ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true" --- # OLM: BEGIN OPERATOR DEPLOYMENT apiVersion: apps/v1 diff --git a/cluster/examples/kubernetes/ceph/storageclass-bucket-delete.yaml b/cluster/examples/kubernetes/ceph/storageclass-bucket-delete.yaml index 82e0810d3..3ace9d5d4 100644 --- a/cluster/examples/kubernetes/ceph/storageclass-bucket-delete.yaml +++ b/cluster/examples/kubernetes/ceph/storageclass-bucket-delete.yaml @@ -2,7 +2,7 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: rook-ceph-delete-bucket -provisioner: ceph.rook.io/bucket +provisioner: rook-ceph.ceph.rook.io/bucket # set the reclaim policy to delete the bucket and all objects # when its OBC is deleted. reclaimPolicy: Delete @@ -10,7 +10,7 @@ parameters: objectStoreName: my-store objectStoreNamespace: rook-ceph region: us-east-1 - # To accomodate brownfield cases reference the existing bucket name here instead + # To accommodate brownfield cases reference the existing bucket name here instead # of in the ObjectBucketClaim (OBC). In this case the provisioner will grant # access to the bucket by creating a new user, attaching it to the bucket, and # providing the credentials via a Secret in the namespace of the requesting OBC. diff --git a/cluster/examples/kubernetes/ceph/storageclass-bucket-retain-external.yaml b/cluster/examples/kubernetes/ceph/storageclass-bucket-retain-external.yaml new file mode 100644 index 000000000..766c95ec0 --- /dev/null +++ b/cluster/examples/kubernetes/ceph/storageclass-bucket-retain-external.yaml @@ -0,0 +1,18 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: rook-ceph-retain-bucket-external +provisioner: rook-ceph-external.ceph.rook.io/bucket +# set the reclaim policy to retain the bucket when its OBC is deleted +reclaimPolicy: Retain +parameters: + # represents the endpoint IP and Port + endpoint: # ip:port + # this is the namespace of the external cluster, identical to the namespace it is running on + objectStoreNamespace: rook-ceph + region: us-east-1 + # To accommodate brownfield cases reference the existing bucket name here instead + # of in the ObjectBucketClaim (OBC). In this case the provisioner will grant + # access to the bucket by creating a new user, attaching it to the bucket, and + # providing the credentials via a Secret in the namespace of the requesting OBC. + #bucketName: diff --git a/cluster/examples/kubernetes/ceph/storageclass-bucket-retain.yaml b/cluster/examples/kubernetes/ceph/storageclass-bucket-retain.yaml index 2d1667702..1b87c1db7 100644 --- a/cluster/examples/kubernetes/ceph/storageclass-bucket-retain.yaml +++ b/cluster/examples/kubernetes/ceph/storageclass-bucket-retain.yaml @@ -2,14 +2,14 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: rook-ceph-retain-bucket -provisioner: ceph.rook.io/bucket +provisioner: rook-ceph.ceph.rook.io/bucket # set the reclaim policy to retain the bucket when its OBC is deleted reclaimPolicy: Retain parameters: objectStoreName: my-store # port 80 assumed objectStoreNamespace: rook-ceph region: us-east-1 - # To accomodate brownfield cases reference the existing bucket name here instead + # To accommodate brownfield cases reference the existing bucket name here instead # of in the ObjectBucketClaim (OBC). In this case the provisioner will grant # access to the bucket by creating a new user, attaching it to the bucket, and # providing the credentials via a Secret in the namespace of the requesting OBC. diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 5cfbe3eec..bd3375363 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -208,8 +208,14 @@ func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *ceph clientController := cephclient.NewClientController(c.context, cluster.Namespace) clientController.StartWatch(cluster.stopCh) + clientToUse := client.AdminUsername + if cluster.Spec.External.Enable { + clientToUse = cluster.Info.ExternalCred.Username + } // Start the object bucket provisioner - bucketProvisioner := bucket.NewProvisioner(c.context, cluster.Namespace) + bucketProvisioner := bucket.NewProvisioner(c.context, cluster.Namespace, clientToUse) + // If cluster is external, pass down the user to the bucket controller + // note: the error return below is ignored and is expected to be removed from the // bucket library's `NewProvisioner` function bucketController, _ := bucket.NewBucketController(c.context.KubeConfig, bucketProvisioner) diff --git a/pkg/operator/ceph/object/admin.go b/pkg/operator/ceph/object/admin.go index 7284eb4fa..c5890b3e7 100644 --- a/pkg/operator/ceph/object/admin.go +++ b/pkg/operator/ceph/object/admin.go @@ -29,15 +29,16 @@ type Context struct { Context *clusterd.Context Name string ClusterName string + RunAsUser string } // NewContext creates a new object store context. func NewContext(context *clusterd.Context, name, clusterName string) *Context { - return &Context{Context: context, Name: name, ClusterName: clusterName} + return &Context{Context: context, Name: name, ClusterName: clusterName, RunAsUser: client.AdminUsername} } func runAdminCommandNoRealm(c *Context, args ...string) (string, error) { - command, args := client.FinalizeCephCommandArgs("radosgw-admin", args, c.Context.ConfigDir, c.ClusterName, client.AdminUsername) + command, args := client.FinalizeCephCommandArgs("radosgw-admin", args, c.Context.ConfigDir, c.ClusterName, c.RunAsUser) // start the rgw admin command output, err := c.Context.Executor.ExecuteCommandWithOutput(command, args...) @@ -49,9 +50,17 @@ func runAdminCommandNoRealm(c *Context, args ...string) (string, error) { } func runAdminCommand(c *Context, args ...string) (string, error) { - options := []string{ - fmt.Sprintf("--rgw-realm=%s", c.Name), - fmt.Sprintf("--rgw-zonegroup=%s", c.Name), + // If the objectStoreName is not passed in the storage class + // This means we are pointing to an external cluster so these commands are not needed + // simply because the external cluster mode does not support that yet + if c.Name != "" { + options := []string{ + fmt.Sprintf("--rgw-realm=%s", c.Name), + fmt.Sprintf("--rgw-zonegroup=%s", c.Name), + } + + return runAdminCommandNoRealm(c, append(args, options...)...) } - return runAdminCommandNoRealm(c, append(args, options...)...) + + return runAdminCommandNoRealm(c, args...) } diff --git a/pkg/operator/ceph/object/bucket/provisioner.go b/pkg/operator/ceph/object/bucket/provisioner.go index b07c1655c..9fb4a87bc 100644 --- a/pkg/operator/ceph/object/bucket/provisioner.go +++ b/pkg/operator/ceph/object/bucket/provisioner.go @@ -28,6 +28,7 @@ import ( "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" + cephutil "github.com/rook/rook/pkg/daemon/ceph/util" cephObject "github.com/rook/rook/pkg/operator/ceph/object" ) @@ -46,18 +47,22 @@ type Provisioner struct { accessKeyID string secretAccessKey string + // RunAsUser cephx user to use on radosgw-admin to create s3 users + RunRgwCmdAsUser string + s3Svc *s3.S3 objectStoreName string objectStoreNamespace string + endpoint string secretName string secretNamespace string } var _ apibkt.Provisioner = &Provisioner{} -func NewProvisioner(context *clusterd.Context, namespace string) *Provisioner { - return &Provisioner{context: context, namespace: namespace} +func NewProvisioner(context *clusterd.Context, namespace, runRgwCmdAsUser string) *Provisioner { + return &Provisioner{context: context, namespace: namespace, RunRgwCmdAsUser: runRgwCmdAsUser} } const maxBuckets = 1 @@ -79,7 +84,6 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje } s3svc, err := NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint()) - if err != nil { return nil, err } @@ -294,15 +298,16 @@ func (p *Provisioner) initializeCreateOrGrant(options *apibkt.BucketOptions) err p.setObjectStoreName(sc) p.setObjectStoreNamespace(sc) p.setRegion(sc) + p.setEndpoint(sc) err = p.setObjectContext() if err != nil { return err } - if err = p.setObjectStoreDomainName(sc); err != nil { - return err - } - if err = p.setObjectStorePort(sc); err != nil { - return err + + // If an endpoint is declared let's use it + err = p.populateDomainAndPort(sc) + if err != nil { + return errors.Wrap(err, "failed to set domain and port") } return nil @@ -324,7 +329,9 @@ func (p *Provisioner) initializeDeleteOrRevoke(ob *bktv1alpha1.ObjectBucket) err if err != nil { return err } - if err = p.setObjectStoreDomainName(sc); err != nil { + + err = p.populateDomainAndPort(sc) + if err != nil { return err } @@ -361,12 +368,15 @@ func (p *Provisioner) composeObjectBucket() *bktv1alpha1.ObjectBucket { func (p *Provisioner) setObjectContext() error { msg := "error building object.Context: store %s cannot be empty" - if p.objectStoreName == "" { + // p.endpoint means we point to an external cluster + if p.objectStoreName == "" && p.endpoint == "" { return errors.Errorf(msg, "name") } else if p.objectStoreNamespace == "" { return errors.Errorf(msg, "namespace") } p.objectContext = cephObject.NewContext(p.context, p.objectStoreName, p.objectStoreNamespace) + p.objectContext.RunAsUser = p.RunRgwCmdAsUser + return nil } @@ -410,6 +420,10 @@ func (p *Provisioner) setBucketName(name string) { p.bucketName = name } +func (p *Provisioner) setEndpoint(sc *storagev1.StorageClass) { + p.endpoint = sc.Parameters[objectStoreEndpoint] +} + func (p *Provisioner) setRegion(sc *storagev1.StorageClass) { const key = "region" p.region = sc.Parameters[key] @@ -418,3 +432,28 @@ func (p *Provisioner) setRegion(sc *storagev1.StorageClass) { func (p Provisioner) getObjectStoreEndpoint() string { return fmt.Sprintf("%s:%d", p.storeDomainName, p.storePort) } + +func (p *Provisioner) populateDomainAndPort(sc *storagev1.StorageClass) error { + endpoint := getObjectStoreEndpoint(sc) + // if endpoint is present, let's introspect it + if endpoint != "" { + p.storeDomainName = cephutil.GetIPFromEndpoint(endpoint) + if p.storeDomainName == "" { + return errors.New("failed to discover endpoint IP (is empty)") + } + p.storePort = cephutil.GetPortFromEndpoint(endpoint) + if p.storePort == 0 { + return errors.New("failed to discover endpoint port (is empty)") + } + // If no endpoint exists let's see if CephObjectStore exists + } else { + if err := p.setObjectStoreDomainName(sc); err != nil { + return errors.Wrap(err, "failed to set object store domain name") + } + if err := p.setObjectStorePort(sc); err != nil { + return errors.Wrap(err, "failed to set object store port") + } + } + + return nil +} diff --git a/pkg/operator/ceph/object/bucket/provisionner_test.go b/pkg/operator/ceph/object/bucket/provisionner_test.go new file mode 100644 index 000000000..5c47ec7e7 --- /dev/null +++ b/pkg/operator/ceph/object/bucket/provisionner_test.go @@ -0,0 +1,106 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bucket + +import ( + "fmt" + "testing" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/test" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + name = "my-user" + namespace = "rook-ceph" + store = "my-store" +) + +func TestPopulateDomainAndPort(t *testing.T) { + p := NewProvisioner(&clusterd.Context{RookClientset: rookclient.NewSimpleClientset(), Clientset: test.New(t, 1)}, namespace, client.AdminUsername) + sc := &storagev1.StorageClass{ + Parameters: map[string]string{ + "foo": "bar", + }, + } + + // No endpoint and no CephObjectStore + err := p.populateDomainAndPort(sc) + assert.Error(t, err) + + // Endpoint is set but port is missing + sc.Parameters["endpoint"] = "192.168.0.1" + err = p.populateDomainAndPort(sc) + assert.Error(t, err) + + // Endpoint is set but IP is missing + sc.Parameters["endpoint"] = ":80" + err = p.populateDomainAndPort(sc) + assert.Error(t, err) + + // Endpoint is correct + sc.Parameters["endpoint"] = "192.168.0.1:80" + err = p.populateDomainAndPort(sc) + assert.NoError(t, err) + assert.Equal(t, "192.168.0.1", p.storeDomainName) + assert.Equal(t, int32(80), p.storePort) + + // No endpoint but a CephObjectStore + sc.Parameters["endpoint"] = "" + sc.Parameters["objectStoreNamespace"] = namespace + sc.Parameters["objectStoreName"] = store + cephObjectStore := &cephv1.CephObjectStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: store, + Namespace: namespace, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "CephObjectStore"}, + Spec: cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{ + Port: int32(80), + }, + }, + } + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", prefixObjectStoreSvc, store), + Namespace: namespace, + }, + Spec: v1.ServiceSpec{ + ClusterIP: "192.168.0.1", + Ports: []v1.ServicePort{{Name: "port", Port: int32(80)}}, + }, + } + + _, err = p.context.RookClientset.CephV1().CephObjectStores(namespace).Create(cephObjectStore) + assert.NoError(t, err) + _, err = p.context.Clientset.CoreV1().Services(namespace).Create(svc) + assert.NoError(t, err) + p.objectStoreNamespace = namespace + p.objectStoreName = store + err = p.populateDomainAndPort(sc) + assert.NoError(t, err) + assert.Equal(t, "rook-ceph-rgw-my-store.rook-ceph", p.storeDomainName) +} diff --git a/pkg/operator/ceph/object/bucket/util.go b/pkg/operator/ceph/object/bucket/util.go index 8ad91be82..21f5b1297 100644 --- a/pkg/operator/ceph/object/bucket/util.go +++ b/pkg/operator/ceph/object/bucket/util.go @@ -17,7 +17,9 @@ limitations under the License. package bucket import ( + "fmt" "math/rand" + "strings" "time" "github.com/coreos/pkg/capnslog" @@ -33,6 +35,8 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" cephclientset "github.com/rook/rook/pkg/client/clientset/versioned/typed/ceph.rook.io/v1" + controllerutil "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/k8sutil" ) var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-bucket-prov") @@ -46,12 +50,24 @@ const ( secretSecretKeyKey = "secretAccessKey" objectStoreName = "objectStoreName" objectStoreNamespace = "objectStoreNamespace" + objectStoreEndpoint = "endpoint" ) func NewBucketController(cfg *rest.Config, p *Provisioner) (*provisioner.Provisioner, error) { const allNamespaces = "" - logger.Infof("Ceph Bucket Provisioner launched") - return provisioner.NewProvisioner(cfg, provisionerName, p, allNamespaces) + provName := provisionerName + + obcWatchOnNamespace, err := k8sutil.GetOperatorSetting(p.context.Clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_OBC_WATCH_OPERATOR_NAMESPACE", "false") + if err != nil { + logger.Warningf("failed to verify if obc should watch the operator namespace or all of them, watching all") + } else { + if strings.EqualFold(obcWatchOnNamespace, "true") { + provName = fmt.Sprintf("%s.%s", p.namespace, provisionerName) + } + } + + logger.Infof("ceph bucket provisioner launched watching for provisioner %q", provName) + return provisioner.NewProvisioner(cfg, provName, p, allNamespaces) } // Return the secret namespace and name from the passed storage class. @@ -80,6 +96,10 @@ func getObjectStoreNameSpace(sc *storagev1.StorageClass) string { return sc.Parameters[objectStoreNamespace] } +func getObjectStoreEndpoint(sc *storagev1.StorageClass) string { + return sc.Parameters[objectStoreEndpoint] +} + func getBucketName(ob *bktv1alpha1.ObjectBucket) string { return ob.Spec.Endpoint.BucketName } diff --git a/pkg/operator/ceph/object/user.go b/pkg/operator/ceph/object/user.go index 95efab724..6556aea81 100644 --- a/pkg/operator/ceph/object/user.go +++ b/pkg/operator/ceph/object/user.go @@ -213,7 +213,7 @@ func LinkUser(c *Context, id, bucket string) (string, int, error) { } func UnlinkUser(c *Context, id, bucket string) (string, int, error) { - logger.Info("Unlinking (user: %s) (bucket: %s)", id, bucket) + logger.Infof("Unlinking (user: %s) (bucket: %s)", id, bucket) args := []string{"bucket", "unlink", "--uid", id, "--bucket", bucket} result, err := runAdminCommand(c, args...) if err != nil { diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index c06e45892..f15bece10 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -1593,6 +1593,7 @@ data: ROOK_CSI_ENABLE_CEPHFS: "true" ROOK_CSI_ENABLE_RBD: "true" ROOK_CSI_ENABLE_GRPC_METRICS: "true" + ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true" ` } @@ -2151,7 +2152,7 @@ func (m *CephManifestsMaster) GetBucketStorageClass(storeNameSpace string, store kind: StorageClass metadata: name: ` + storageClassName + ` -provisioner: ceph.rook.io/bucket +provisioner: ` + storeNameSpace + `.ceph.rook.io/bucket reclaimPolicy: ` + reclaimPolicy + ` parameters: objectStoreName: ` + storeName + `