ceph: add support for OBC on external cluster

We can now consume an external s3 endpoint which is not managed by Rook.

Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
Sébastien Han committed 2020-05-04 18:14:26 +02:00
1 parent 4017aa8de5
commit b86bb9c528
14 files changed
+237 -29

No files matched your search

@@ -169,7 +169,7 @@ spec:
{{- if .Values.csi.allowUnsupportedVersion }}
- name: ROOK_CSI_ALLOW_UNSUPPORTED_VERSION
value: {{ .Values.csi.allowUnsupportedVersion | quote }}
{{- end }}
{{- end }}
{{- if .Values.csi.pluginTolerations }}
- name: CSI_PLUGIN_TOLERATIONS
value: {{ toYaml .Values.csi.pluginTolerations | quote }}
@@ -223,6 +223,8 @@ spec:
value: "{{ .Values.enableFlexDriver }}"
- name: ROOK_ENABLE_DISCOVERY_DAEMON
value: "{{ .Values.enableDiscoveryDaemon }}"
- name: ROOK_OBC_WATCH_OPERATOR_NAMESPACE
value: "{{ .Values.enableOBCWatchOperatorNamespace }}"
- name: NODE_NAME
valueFrom:
+3
View File
@@ -312,3 +312,6 @@ discoverDaemonUdev:
# imagePullSecrets option allow to pull docker images from private docker registry. Option will be passed to all service accounts.
# imagePullSecrets:
# - name: my-registry-secret
# Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used
enableOBCWatchOperatorNamespace: true
@@ -9,6 +9,7 @@ set -Eeuo pipefail
#############
: "${CLIENT_CHECKER_NAME:=client.healthchecker}"
: "${RGW_POOL_PREFIX:=default}"
#############
# FUNCTIONS #
@@ -31,7 +32,7 @@ fi
}
function createCheckerKey() {
checkerKey=$(ceph auth get-or-create "$CLIENT_CHECKER_NAME" mon 'allow r, allow command quorum_status'|awk '/key =/ { print $3}')
checkerKey=$(ceph auth get-or-create "$CLIENT_CHECKER_NAME" mon 'allow r, allow command quorum_status' osd 'allow rwx pool='"$RGW_POOL_PREFIX"'.rgw.meta, allow r pool=.rgw.root, allow rw pool='"$RGW_POOL_PREFIX"'.rgw.control, allow x pool='"$RGW_POOL_PREFIX"'.rgw.buckets.index'|awk '/key =/ { print $3}')
echo "export ROOK_EXTERNAL_USER_SECRET=$checkerKey"
echo "export ROOK_EXTERNAL_USERNAME=$CLIENT_CHECKER_NAME"
}
@@ -85,7 +85,7 @@ data:
# - effect: NoExecute
# key: node-role.kubernetes.io/etcd
# operator: Exists
# (Optional) CEPH CSI RBD provisioner resource requirement list, Put here list of resource
# requests and limits you want to apply for provisioner pod
# CSI_RBD_PROVISIONER_RESOURCE: |
@@ -234,13 +234,16 @@ data:
# limits:
# memory: 256Mi
# cpu: 100m
# Configure CSI CSI Ceph FS grpc and liveness metrics port
# CSI_CEPHFS_GRPC_METRICS_PORT: "9091"
# CSI_CEPHFS_LIVENESS_METRICS_PORT: "9081"
# Configure CSI RBD grpc and liveness metrics port
# CSI_RBD_GRPC_METRICS_PORT: "9090"
# CSI_RBD_LIVENESS_METRICS_PORT: "9080"
# Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used
ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true"
---
# OLM: BEGIN OPERATOR DEPLOYMENT
apiVersion: apps/v1
@@ -2,7 +2,7 @@ apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: rook-ceph-delete-bucket
provisioner: ceph.rook.io/bucket
provisioner: rook-ceph.ceph.rook.io/bucket
# set the reclaim policy to delete the bucket and all objects
# when its OBC is deleted.
reclaimPolicy: Delete
@@ -10,7 +10,7 @@ parameters:
objectStoreName: my-store
objectStoreNamespace: rook-ceph
region: us-east-1
# To accomodate brownfield cases reference the existing bucket name here instead
# To accommodate brownfield cases reference the existing bucket name here instead
# of in the ObjectBucketClaim (OBC). In this case the provisioner will grant
# access to the bucket by creating a new user, attaching it to the bucket, and
# providing the credentials via a Secret in the namespace of the requesting OBC.
@@ -0,0 +1,18 @@
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: rook-ceph-retain-bucket-external
provisioner: rook-ceph-external.ceph.rook.io/bucket
# set the reclaim policy to retain the bucket when its OBC is deleted
reclaimPolicy: Retain
parameters:
# represents the endpoint IP and Port
endpoint: # ip:port
# this is the namespace of the external cluster, identical to the namespace it is running on
objectStoreNamespace: rook-ceph
region: us-east-1
# To accommodate brownfield cases reference the existing bucket name here instead
# of in the ObjectBucketClaim (OBC). In this case the provisioner will grant
# access to the bucket by creating a new user, attaching it to the bucket, and
# providing the credentials via a Secret in the namespace of the requesting OBC.
#bucketName:
@@ -2,14 +2,14 @@ apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: rook-ceph-retain-bucket
provisioner: ceph.rook.io/bucket
provisioner: rook-ceph.ceph.rook.io/bucket
# set the reclaim policy to retain the bucket when its OBC is deleted
reclaimPolicy: Retain
parameters:
objectStoreName: my-store # port 80 assumed
objectStoreNamespace: rook-ceph
region: us-east-1
# To accomodate brownfield cases reference the existing bucket name here instead
# To accommodate brownfield cases reference the existing bucket name here instead
# of in the ObjectBucketClaim (OBC). In this case the provisioner will grant
# access to the bucket by creating a new user, attaching it to the bucket, and
# providing the credentials via a Secret in the namespace of the requesting OBC.
+7 -1
View File
@@ -208,8 +208,14 @@ func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *ceph
clientController := cephclient.NewClientController(c.context, cluster.Namespace)
clientController.StartWatch(cluster.stopCh)
clientToUse := client.AdminUsername
if cluster.Spec.External.Enable {
clientToUse = cluster.Info.ExternalCred.Username
}
// Start the object bucket provisioner
bucketProvisioner := bucket.NewProvisioner(c.context, cluster.Namespace)
bucketProvisioner := bucket.NewProvisioner(c.context, cluster.Namespace, clientToUse)
// If cluster is external, pass down the user to the bucket controller
// note: the error return below is ignored and is expected to be removed from the
// bucket library's `NewProvisioner` function
bucketController, _ := bucket.NewBucketController(c.context.KubeConfig, bucketProvisioner)
+15 -6
View File
@@ -29,15 +29,16 @@ type Context struct {
Context *clusterd.Context
Name string
ClusterName string
RunAsUser string
}
// NewContext creates a new object store context.
func NewContext(context *clusterd.Context, name, clusterName string) *Context {
return &Context{Context: context, Name: name, ClusterName: clusterName}
return &Context{Context: context, Name: name, ClusterName: clusterName, RunAsUser: client.AdminUsername}
}
func runAdminCommandNoRealm(c *Context, args ...string) (string, error) {
command, args := client.FinalizeCephCommandArgs("radosgw-admin", args, c.Context.ConfigDir, c.ClusterName, client.AdminUsername)
command, args := client.FinalizeCephCommandArgs("radosgw-admin", args, c.Context.ConfigDir, c.ClusterName, c.RunAsUser)
// start the rgw admin command
output, err := c.Context.Executor.ExecuteCommandWithOutput(command, args...)
@@ -49,9 +50,17 @@ func runAdminCommandNoRealm(c *Context, args ...string) (string, error) {
}
func runAdminCommand(c *Context, args ...string) (string, error) {
options := []string{
fmt.Sprintf("--rgw-realm=%s", c.Name),
fmt.Sprintf("--rgw-zonegroup=%s", c.Name),
// If the objectStoreName is not passed in the storage class
// This means we are pointing to an external cluster so these commands are not needed
// simply because the external cluster mode does not support that yet
if c.Name != "" {
options := []string{
fmt.Sprintf("--rgw-realm=%s", c.Name),
fmt.Sprintf("--rgw-zonegroup=%s", c.Name),
}
return runAdminCommandNoRealm(c, append(args, options...)...)
}
return runAdminCommandNoRealm(c, append(args, options...)...)
return runAdminCommandNoRealm(c, args...)
}
+49 -10
View File
@@ -28,6 +28,7 @@ import (
"github.com/pkg/errors"
"github.com/rook/rook/pkg/clusterd"
cephutil "github.com/rook/rook/pkg/daemon/ceph/util"
cephObject "github.com/rook/rook/pkg/operator/ceph/object"
)
@@ -46,18 +47,22 @@ type Provisioner struct {
accessKeyID string
secretAccessKey string
// RunAsUser cephx user to use on radosgw-admin to create s3 users
RunRgwCmdAsUser string
s3Svc *s3.S3
objectStoreName string
objectStoreNamespace string
endpoint string
secretName string
secretNamespace string
}
var _ apibkt.Provisioner = &Provisioner{}
func NewProvisioner(context *clusterd.Context, namespace string) *Provisioner {
return &Provisioner{context: context, namespace: namespace}
func NewProvisioner(context *clusterd.Context, namespace, runRgwCmdAsUser string) *Provisioner {
return &Provisioner{context: context, namespace: namespace, RunRgwCmdAsUser: runRgwCmdAsUser}
}
const maxBuckets = 1
@@ -79,7 +84,6 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje
}
s3svc, err := NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint())
if err != nil {
return nil, err
}
@@ -294,15 +298,16 @@ func (p *Provisioner) initializeCreateOrGrant(options *apibkt.BucketOptions) err
p.setObjectStoreName(sc)
p.setObjectStoreNamespace(sc)
p.setRegion(sc)
p.setEndpoint(sc)
err = p.setObjectContext()
if err != nil {
return err
}
if err = p.setObjectStoreDomainName(sc); err != nil {
return err
}
if err = p.setObjectStorePort(sc); err != nil {
return err
// If an endpoint is declared let's use it
err = p.populateDomainAndPort(sc)
if err != nil {
return errors.Wrap(err, "failed to set domain and port")
}
return nil
@@ -324,7 +329,9 @@ func (p *Provisioner) initializeDeleteOrRevoke(ob *bktv1alpha1.ObjectBucket) err
if err != nil {
return err
}
if err = p.setObjectStoreDomainName(sc); err != nil {
err = p.populateDomainAndPort(sc)
if err != nil {
return err
}
@@ -361,12 +368,15 @@ func (p *Provisioner) composeObjectBucket() *bktv1alpha1.ObjectBucket {
func (p *Provisioner) setObjectContext() error {
msg := "error building object.Context: store %s cannot be empty"
if p.objectStoreName == "" {
// p.endpoint means we point to an external cluster
if p.objectStoreName == "" && p.endpoint == "" {
return errors.Errorf(msg, "name")
} else if p.objectStoreNamespace == "" {
return errors.Errorf(msg, "namespace")
}
p.objectContext = cephObject.NewContext(p.context, p.objectStoreName, p.objectStoreNamespace)
p.objectContext.RunAsUser = p.RunRgwCmdAsUser
return nil
}
@@ -410,6 +420,10 @@ func (p *Provisioner) setBucketName(name string) {
p.bucketName = name
}
func (p *Provisioner) setEndpoint(sc *storagev1.StorageClass) {
p.endpoint = sc.Parameters[objectStoreEndpoint]
}
func (p *Provisioner) setRegion(sc *storagev1.StorageClass) {
const key = "region"
p.region = sc.Parameters[key]
@@ -418,3 +432,28 @@ func (p *Provisioner) setRegion(sc *storagev1.StorageClass) {
func (p Provisioner) getObjectStoreEndpoint() string {
return fmt.Sprintf("%s:%d", p.storeDomainName, p.storePort)
}
func (p *Provisioner) populateDomainAndPort(sc *storagev1.StorageClass) error {
endpoint := getObjectStoreEndpoint(sc)
// if endpoint is present, let's introspect it
if endpoint != "" {
p.storeDomainName = cephutil.GetIPFromEndpoint(endpoint)
if p.storeDomainName == "" {
return errors.New("failed to discover endpoint IP (is empty)")
}
p.storePort = cephutil.GetPortFromEndpoint(endpoint)
if p.storePort == 0 {
return errors.New("failed to discover endpoint port (is empty)")
}
// If no endpoint exists let's see if CephObjectStore exists
} else {
if err := p.setObjectStoreDomainName(sc); err != nil {
return errors.Wrap(err, "failed to set object store domain name")
}
if err := p.setObjectStorePort(sc); err != nil {
return errors.Wrap(err, "failed to set object store port")
}
}
return nil
}
@@ -0,0 +1,106 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package bucket
import (
"fmt"
"testing"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/test"
"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
var (
name = "my-user"
namespace = "rook-ceph"
store = "my-store"
)
func TestPopulateDomainAndPort(t *testing.T) {
p := NewProvisioner(&clusterd.Context{RookClientset: rookclient.NewSimpleClientset(), Clientset: test.New(t, 1)}, namespace, client.AdminUsername)
sc := &storagev1.StorageClass{
Parameters: map[string]string{
"foo": "bar",
},
}
// No endpoint and no CephObjectStore
err := p.populateDomainAndPort(sc)
assert.Error(t, err)
// Endpoint is set but port is missing
sc.Parameters["endpoint"] = "192.168.0.1"
err = p.populateDomainAndPort(sc)
assert.Error(t, err)
// Endpoint is set but IP is missing
sc.Parameters["endpoint"] = ":80"
err = p.populateDomainAndPort(sc)
assert.Error(t, err)
// Endpoint is correct
sc.Parameters["endpoint"] = "192.168.0.1:80"
err = p.populateDomainAndPort(sc)
assert.NoError(t, err)
assert.Equal(t, "192.168.0.1", p.storeDomainName)
assert.Equal(t, int32(80), p.storePort)
// No endpoint but a CephObjectStore
sc.Parameters["endpoint"] = ""
sc.Parameters["objectStoreNamespace"] = namespace
sc.Parameters["objectStoreName"] = store
cephObjectStore := &cephv1.CephObjectStore{
ObjectMeta: metav1.ObjectMeta{
Name: store,
Namespace: namespace,
},
TypeMeta: metav1.TypeMeta{
Kind: "CephObjectStore"},
Spec: cephv1.ObjectStoreSpec{
Gateway: cephv1.GatewaySpec{
Port: int32(80),
},
},
}
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%s", prefixObjectStoreSvc, store),
Namespace: namespace,
},
Spec: v1.ServiceSpec{
ClusterIP: "192.168.0.1",
Ports: []v1.ServicePort{{Name: "port", Port: int32(80)}},
},
}
_, err = p.context.RookClientset.CephV1().CephObjectStores(namespace).Create(cephObjectStore)
assert.NoError(t, err)
_, err = p.context.Clientset.CoreV1().Services(namespace).Create(svc)
assert.NoError(t, err)
p.objectStoreNamespace = namespace
p.objectStoreName = store
err = p.populateDomainAndPort(sc)
assert.NoError(t, err)
assert.Equal(t, "rook-ceph-rgw-my-store.rook-ceph", p.storeDomainName)
}
+22 -2
View File
@@ -17,7 +17,9 @@ limitations under the License.
package bucket
import (
"fmt"
"math/rand"
"strings"
"time"
"github.com/coreos/pkg/capnslog"
@@ -33,6 +35,8 @@ import (
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
cephclientset "github.com/rook/rook/pkg/client/clientset/versioned/typed/ceph.rook.io/v1"
controllerutil "github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-bucket-prov")
@@ -46,12 +50,24 @@ const (
secretSecretKeyKey = "secretAccessKey"
objectStoreName = "objectStoreName"
objectStoreNamespace = "objectStoreNamespace"
objectStoreEndpoint = "endpoint"
)
func NewBucketController(cfg *rest.Config, p *Provisioner) (*provisioner.Provisioner, error) {
const allNamespaces = ""
logger.Infof("Ceph Bucket Provisioner launched")
return provisioner.NewProvisioner(cfg, provisionerName, p, allNamespaces)
provName := provisionerName
obcWatchOnNamespace, err := k8sutil.GetOperatorSetting(p.context.Clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_OBC_WATCH_OPERATOR_NAMESPACE", "false")
if err != nil {
logger.Warningf("failed to verify if obc should watch the operator namespace or all of them, watching all")
} else {
if strings.EqualFold(obcWatchOnNamespace, "true") {
provName = fmt.Sprintf("%s.%s", p.namespace, provisionerName)
}
}
logger.Infof("ceph bucket provisioner launched watching for provisioner %q", provName)
return provisioner.NewProvisioner(cfg, provName, p, allNamespaces)
}
// Return the secret namespace and name from the passed storage class.
@@ -80,6 +96,10 @@ func getObjectStoreNameSpace(sc *storagev1.StorageClass) string {
return sc.Parameters[objectStoreNamespace]
}
func getObjectStoreEndpoint(sc *storagev1.StorageClass) string {
return sc.Parameters[objectStoreEndpoint]
}
func getBucketName(ob *bktv1alpha1.ObjectBucket) string {
return ob.Spec.Endpoint.BucketName
}
+1 -1
View File
@@ -213,7 +213,7 @@ func LinkUser(c *Context, id, bucket string) (string, int, error) {
}
func UnlinkUser(c *Context, id, bucket string) (string, int, error) {
logger.Info("Unlinking (user: %s) (bucket: %s)", id, bucket)
logger.Infof("Unlinking (user: %s) (bucket: %s)", id, bucket)
args := []string{"bucket", "unlink", "--uid", id, "--bucket", bucket}
result, err := runAdminCommand(c, args...)
if err != nil {
+2 -1
View File
@@ -1593,6 +1593,7 @@ data:
ROOK_CSI_ENABLE_CEPHFS: "true"
ROOK_CSI_ENABLE_RBD: "true"
ROOK_CSI_ENABLE_GRPC_METRICS: "true"
ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true"
`
}
@@ -2151,7 +2152,7 @@ func (m *CephManifestsMaster) GetBucketStorageClass(storeNameSpace string, store
kind: StorageClass
metadata:
name: ` + storageClassName + `
provisioner: ceph.rook.io/bucket
provisioner: ` + storeNameSpace + `.ceph.rook.io/bucket
reclaimPolicy: ` + reclaimPolicy + `
parameters:
objectStoreName: ` + storeName + `