Merge pull request #7998 from leseb/replace-radosgw-admin-cli-with-goceph

ceph: stop using radosgw-admin CLI for s3 user management
This commit is contained in:
Sébastien Han authored and GitHub committed 2021-06-09 16:01:14 +02:00
commit e17fa5be51
29 files changed
+560 -376

No files matched your search

@@ -14,12 +14,14 @@ See the License for the specific language governing permissions and
limitations under the License.
'''
import errno
import sys
import json
import argparse
import unittest
import re
import requests
import subprocess
from os import linesep as LINESEP
from os import path
@@ -138,6 +140,7 @@ class DummyRados(object):
class RadosJSON:
EXTERNAL_USER_NAME = "client.healthchecker"
EXTERNAL_RGW_ADMIN_OPS_USER_NAME = "rgw-admin-ops-user"
EMPTY_OUTPUT_LIST = "Empty output list"
DEFAULT_RGW_POOL_PREFIX = "default"
DEFAULT_MONITORING_ENDPOINT_PORT = "9283"
@@ -347,12 +350,13 @@ class RadosJSON:
raise ExecutionFailureException(
"'mgr services' command failed.\n" +
"Error: {}".format(err_msg if ret_val != 0 else self.EMPTY_OUTPUT_LIST))
monitoring_endpoint = json_out.get('mgrmap', {}).get('services', {}).get('prometheus', '')
monitoring_endpoint = json_out.get('mgrmap', {}).get(
'services', {}).get('prometheus', '')
if not monitoring_endpoint:
raise ExecutionFailureException(
"'prometheus' service not found, is the exporter enabled?'.\n")
# now check the stand-by mgr-s
standby_arr = json_out.get('mgrmap',{}).get('standbys', [])
standby_arr = json_out.get('mgrmap', {}).get('standbys', [])
for each_standby in standby_arr:
if 'name' in each_standby.keys():
standby_mgrs.append(each_standby['name'])
@@ -378,10 +382,10 @@ class RadosJSON:
for each_standby_mgr in standby_mgrs:
failed_ip = each_standby_mgr
mgr_ips.append(
self._convert_hostname_to_ip(each_standby_mgr))
self._convert_hostname_to_ip(each_standby_mgr))
except:
raise ExecutionFailureException(
"Conversion of host: {} to IP failed. "\
"Conversion of host: {} to IP failed. "
"Please enter the IP addresses of all the ceph-mgrs with the '--monitoring-endpoint' flag".format(failed_ip))
monitoring_endpoint = self._join_host_port(
monitoring_endpoint_ip, monitoring_endpoint_port)
@@ -555,6 +559,33 @@ class RadosJSON:
"Error: {}".format(err_msg if ret_val != 0 else self.EMPTY_OUTPUT_LIST))
return str(json_out[0]['key'])
def create_rgw_admin_ops_user(self):
cmd = ['radosgw-admin', 'user', 'create', '--uid', self.EXTERNAL_RGW_ADMIN_OPS_USER_NAME, '--display-name',
'Rook RGW Admin Ops user', '--caps', 'buckets=*;users=*;usage=read;metadata=read;zone=read']
try:
output = subprocess.check_output(cmd,
stderr=subprocess.PIPE)
except subprocess.CalledProcessError as exec:
# if the user already exists, we just query it
if exec.returncode == errno.EEXIST:
cmd = ['radosgw-admin', 'user', 'info',
'--uid', self.EXTERNAL_RGW_ADMIN_OPS_USER_NAME
]
try:
output = subprocess.check_output(cmd,
stderr=subprocess.PIPE)
except subprocess.CalledProcessError as exec:
err_msg = "failed to execute command %s. Output: %s. Code: %s. Error: %s" % (
cmd, exec.output, exec.returncode, exec.stderr)
raise Exception(err_msg)
else:
err_msg = "failed to execute command %s. Output: %s. Code: %s. Error: %s" % (
cmd, exec.output, exec.returncode, exec.stderr)
raise Exception(err_msg)
jsonoutput = json.loads(output)
return jsonoutput["keys"][0]['access_key'], jsonoutput["keys"][0]['secret_key']
def _gen_output_map(self):
if self.out_map:
return
@@ -598,6 +629,7 @@ class RadosJSON:
self.out_map['MONITORING_ENDPOINT_PORT'] = self.get_active_and_standby_mgrs()
self.out_map['RBD_POOL_NAME'] = self._arg_parser.rbd_data_pool_name
self.out_map['RGW_POOL_PREFIX'] = self._arg_parser.rgw_pool_prefix
self.out_map['ACCESS_KEY'], self.out_map['SECRET_KEY'] = self.create_rgw_admin_ops_user()
def gen_shell_out(self):
self._gen_output_map()
@@ -660,6 +692,14 @@ class RadosJSON:
"MonitoringEndpoint": self.out_map['MONITORING_ENDPOINT'],
"MonitoringPort": self.out_map['MONITORING_ENDPOINT_PORT']
}
},
{
"name": "rgw-admin-ops-user",
"kind": "Secret",
"data": {
"accessKey": self.out_map['ACCESS_KEY'],
"secretKey": self.out_map['SECRET_KEY']
}
}
]
@@ -78,6 +78,15 @@ function namespace() {
export NAMESPACE=$ns
}
function createRGWAdminOpsUser() {
createRGWAdminOpsUserKeys=$(radosgw-admin user create --uid rgw-admin-ops-user --display-name "Rook RGW Admin Ops user" --caps "buckets=*;users=*;usage=read;metadata=read;zone=read"|jq --raw-output .keys[0])
createRGWAdminOpsUserAccessKey=$(echo "$createRGWAdminOpsUserKeys"|jq --raw-output .access_key)
createRGWAdminOpsUserSecretKey=$(echo "$createRGWAdminOpsUserKeys"|jq --raw-output .secret_key)
echo "export RGW_ADMIN_OPS_USER_ACCESS_KEY=$createRGWAdminOpsUserAccessKey"
echo "export RGW_ADMIN_OPS_USER_SECRET_KEY=$createRGWAdminOpsUserSecretKey"
}
########
# MAIN #
########
@@ -90,3 +99,4 @@ createCephCSIKeyringCephFSProvisioner
getFSID
externalMonData
namespace
createRGWAdminOpsUser
@@ -9,6 +9,7 @@ CSI_RBD_NODE_SECRET_NAME=rook-csi-rbd-node
CSI_RBD_PROVISIONER_SECRET_NAME=rook-csi-rbd-provisioner
CSI_CEPHFS_NODE_SECRET_NAME=rook-csi-cephfs-node
CSI_CEPHFS_PROVISIONER_SECRET_NAME=rook-csi-cephfs-provisioner
RGW_ADMIN_OPS_USER_SECRET_NAME=rgw-admin-ops-user
MON_SECRET_CLUSTER_NAME_KEYNAME=cluster-name
MON_SECRET_FSID_KEYNAME=fsid
MON_SECRET_ADMIN_KEYRING_KEYNAME=admin-secret
@@ -140,6 +141,17 @@ function importCsiCephFSProvisionerSecret() {
--from-literal=adminKey="$CSI_CEPHFS_PROVISIONER_SECRET"
}
function importRGWAdminOpsUser() {
kubectl -n "$NAMESPACE" \
create \
secret \
generic \
--type="kubernetes.io/rook" \
"$RGW_ADMIN_OPS_USER_SECRET_NAME" \
--from-literal=accessKey="$RGW_ADMIN_OPS_USER_ACCESS_KEY" \
--from-literal=secretKey="$RGW_ADMIN_OPS_USER_SECRET_KEY"
}
########
# MAIN #
########
@@ -150,3 +162,4 @@ importCsiRBDNodeSecret
importCsiRBDProvisionerSecret
importCsiCephFSNodeSecret
importCsiCephFSProvisionerSecret
importRGWAdminOpsUser
+1
View File
@@ -5,6 +5,7 @@ go 1.16
require (
github.com/aws/aws-sdk-go v1.35.24
github.com/banzaicloud/k8s-objectmatcher v1.1.0
github.com/ceph/go-ceph v0.9.1-0.20210607162346-8179bd4437f9
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f
github.com/corpix/uarand v0.1.1 // indirect
github.com/csi-addons/volume-replication-operator v0.1.1-0.20210525040814-ab575a2879fb
+3
View File
@@ -242,6 +242,8 @@ github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QH
github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/centrify/cloud-golang-sdk v0.0.0-20190214225812-119110094d0f/go.mod h1:C0rtzmGXgN78pYR0tGJFhtHgkbAs0lIbHwkB81VxDQE=
github.com/ceph/go-ceph v0.9.1-0.20210607162346-8179bd4437f9 h1:M2eYrt7HSr7WImn5eGbonTcVwl2JddiKjnUNZ+LTs0s=
github.com/ceph/go-ceph v0.9.1-0.20210607162346-8179bd4437f9/go.mod h1:mafFpf5Vg8Ai8Bd+FAMvKBHLmtdpTXdRP/TNq8XWegY=
github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
github.com/cespare/xxhash v0.0.0-20181017004759-096ff4a8a059/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
@@ -1857,6 +1859,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+1 -1
View File
@@ -210,7 +210,7 @@ func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Re
err = r.createOrUpdateClient(cephClient)
if err != nil {
if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
logger.Info("skipping reconcile since operator is still initializing")
logger.Info(opcontroller.OperatorNotInitializedMessage)
return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil
}
updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure)
+1 -1
View File
@@ -210,7 +210,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, daemon)
if err != nil {
if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
logger.Info("skipping reconcile since operator is still initializing")
logger.Info(opcontroller.OperatorNotInitializedMessage)
return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil
}
return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to retrieve current ceph %q version", daemon)
@@ -40,6 +40,9 @@ const (
// UninitializedCephConfigError refers to the error message printed by the Ceph CLI when there is no ceph configuration file
// This typically is raised when the operator has not finished initializing
UninitializedCephConfigError = "error calling conf_read_file"
// OperatorNotInitializedMessage is the message we print when the Operator is not ready to reconcile, typically the ceph.conf has not been generated yet
OperatorNotInitializedMessage = "skipping reconcile since operator is still initializing"
)
var (
+2 -2
View File
@@ -216,7 +216,7 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil
currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, opconfig.MonType)
if err != nil {
if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
logger.Info("skipping reconcile since operator is still initializing")
logger.Info(opcontroller.OperatorNotInitializedMessage)
return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil
}
return reconcile.Result{}, errors.Wrapf(err, "failed to retrieve current ceph %q version", opconfig.MonType)
@@ -244,7 +244,7 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil
// validate the filesystem settings
if err := validateFilesystem(r.context, r.clusterInfo, r.cephClusterSpec, cephFilesystem); err != nil {
if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
logger.Info("skipping reconcile since operator is still initializing")
logger.Info(opcontroller.OperatorNotInitializedMessage)
return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil
}
return reconcile.Result{}, errors.Wrapf(err, "invalid object filesystem %q arguments", cephFilesystem.Name)
+1 -1
View File
@@ -197,7 +197,7 @@ func (r *ReconcileFilesystemMirror) reconcile(request reconcile.Request) (reconc
currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, daemon)
if err != nil {
if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
logger.Info("skipping reconcile since operator is still initializing")
logger.Info(opcontroller.OperatorNotInitializedMessage)
return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil
}
return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to retrieve current ceph %q version", daemon)
+1 -1
View File
@@ -196,7 +196,7 @@ func (r *ReconcileCephNFS) reconcile(request reconcile.Request) (reconcile.Resul
currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, opconfig.MonType)
if err != nil {
if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
logger.Info("skipping reconcile since operator is still initializing")
logger.Info(opcontroller.OperatorNotInitializedMessage)
return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil
}
return reconcile.Result{}, errors.Wrapf(err, "failed to retrieve current ceph %q version", opconfig.MonType)
+77 -14
View File
@@ -17,34 +17,54 @@ limitations under the License.
package object
import (
"context"
"fmt"
"regexp"
"github.com/ceph/go-ceph/rgw/admin"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/util/exec"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
)
// Context holds the context for the object store.
type Context struct {
Context *clusterd.Context
clusterInfo *client.ClusterInfo
Name string
UID string
Endpoint string
Realm string
ZoneGroup string
Zone string
Context *clusterd.Context
clusterInfo *cephclient.ClusterInfo
Name string
UID string
Endpoint string
Realm string
ZoneGroup string
Zone string
adminOpsUserAccessKey string
adminOpsUserSecretKey string
adminOpsClient *admin.API
}
const (
// RGWAdminOpsUserSecretName is the secret name of the admin ops user
// #nosec G101 since this is not leaking any hardcoded credentials, it's just the secret name
RGWAdminOpsUserSecretName = "rgw-admin-ops-user"
rgwAdminOpsUserAccessKey = "accessKey"
rgwAdminOpsUserSecretKey = "secretKey"
rgwAdminOpsUserCaps = "buckets=*;users=*;usage=read;metadata=read;zone=read"
)
var (
rgwAdminOpsUserDisplayName = "RGW Admin Ops User"
)
// NewContext creates a new object store context.
func NewContext(context *clusterd.Context, clusterInfo *client.ClusterInfo, name string) *Context {
func NewContext(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, name string) *Context {
return &Context{Context: context, Name: name, clusterInfo: clusterInfo}
}
func NewMultisiteContext(context *clusterd.Context, clusterInfo *client.ClusterInfo, store *cephv1.CephObjectStore) (*Context, error) {
func NewMultisiteContext(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, store *cephv1.CephObjectStore) (*Context, error) {
objContext := &Context{Context: context, Name: store.Name, clusterInfo: clusterInfo}
realmName, zoneGroupName, zoneName, err := getMultisiteForObjectStore(context, store)
if err != nil {
@@ -71,10 +91,10 @@ func extractJSON(output string) (string, error) {
// RunAdminCommandNoMultisite is for running radosgw-admin commands in scenarios where an object-store has not been created yet or for commands on the realm or zonegroup (ex: radosgw-admin zonegroup get)
// This function times out after a fixed interval if no response is received.
func RunAdminCommandNoMultisite(c *Context, expectJSON bool, args ...string) (string, error) {
command, args := client.FinalizeCephCommandArgs("radosgw-admin", c.clusterInfo, args, c.Context.ConfigDir)
command, args := cephclient.FinalizeCephCommandArgs("radosgw-admin", c.clusterInfo, args, c.Context.ConfigDir)
// start the rgw admin command
output, err := c.Context.Executor.ExecuteCommandWithTimeout(client.CephCommandTimeout, command, args...)
output, err := c.Context.Executor.ExecuteCommandWithTimeout(cephclient.CephCommandTimeout, command, args...)
if err != nil {
return output, err
}
@@ -98,7 +118,7 @@ func runAdminCommand(c *Context, expectJSON bool, args ...string) (string, error
// The following conditions tries to determine if the cluster is external
// When connecting to an external cluster, the Ceph user is different than client.admin
// This is not perfect though since "client.admin" is somehow supported...
if c.Name != "" && c.clusterInfo.CephCred.Username == client.AdminUsername {
if c.Name != "" && c.clusterInfo.CephCred.Username == cephclient.AdminUsername {
options := []string{
fmt.Sprintf("--rgw-realm=%s", c.Realm),
fmt.Sprintf("--rgw-zonegroup=%s", c.ZoneGroup),
@@ -149,3 +169,46 @@ func isInvalidFlagError(err error) bool {
// it's also returned from some other failures, but this should be rare for Rook
return exitCode == 22
}
func GetAdminOPSUserCredentials(ctx *clusterd.Context, clusterInfo *cephclient.ClusterInfo, objContext *Context, cephObjectStore *cephv1.CephObjectStore) (string, string, error) {
if cephObjectStore.Spec.IsExternal() {
// Fetch the secret for admin ops user
s := &v1.Secret{}
err := ctx.Client.Get(context.TODO(), types.NamespacedName{Name: RGWAdminOpsUserSecretName, Namespace: cephObjectStore.Namespace}, s)
if err != nil {
return "", "", err
}
accessKey, ok := s.Data[rgwAdminOpsUserAccessKey]
if !ok {
return "", "", errors.Errorf("failed to find accessKey %q for rgw admin ops in secret %q", rgwAdminOpsUserAccessKey, RGWAdminOpsUserSecretName)
}
secretKey, ok := s.Data[rgwAdminOpsUserSecretKey]
if !ok {
return "", "", errors.Errorf("failed to find secretKey %q for rgw admin ops in secret %q", rgwAdminOpsUserSecretKey, RGWAdminOpsUserSecretName)
}
// Set the keys for further usage
return string(accessKey), string(secretKey), nil
}
// Fetch the admin ops user locally
userConfig := ObjectUser{
UserID: RGWAdminOpsUserSecretName,
DisplayName: &rgwAdminOpsUserDisplayName,
AdminOpsUser: true,
}
logger.Debugf("creating s3 user object %q for object store %q", userConfig.UserID, cephObjectStore.Namespace)
user, rgwerr, err := CreateUser(objContext, userConfig)
if err != nil {
if rgwerr == ErrorCodeFileExists {
user, _, err = GetUser(objContext, userConfig.UserID)
if err != nil {
return "", "", errors.Wrapf(err, "failed to get details from ceph object user %q for object store %q", userConfig.UserID, cephObjectStore.Name)
}
} else {
return "", "", errors.Wrapf(err, "failed to create object user %q. error code %d for object store %q", userConfig.UserID, rgwerr, cephObjectStore.Name)
}
}
return *user.AccessKey, *user.SecretKey, nil
}
-44
View File
@@ -173,28 +173,6 @@ func getBucketMetadata(c *Context, bucket string) (*ObjectBucketMetadata, bool,
return &ObjectBucketMetadata{Owner: s.Data.Owner, CreatedAt: createdAt}, false, nil
}
func ListBuckets(c *Context) ([]ObjectBucket, error) {
logger.Infof("Listing buckets")
stats, err := GetBucketsStats(c)
if err != nil {
return nil, errors.Wrap(err, "Failed to get bucket stats")
}
buckets := []ObjectBucket{}
for bucket, stat := range stats {
metadata, _, err := getBucketMetadata(c, bucket)
if err != nil {
return nil, err
}
buckets = append(buckets, ObjectBucket{Name: bucket, ObjectBucketMetadata: ObjectBucketMetadata{Owner: metadata.Owner, CreatedAt: metadata.CreatedAt}, ObjectBucketStats: stat})
}
return buckets, nil
}
func GetBucket(c *Context, bucket string) (*ObjectBucket, int, error) {
stat, notFound, err := GetBucketStats(c, bucket)
if notFound {
@@ -216,25 +194,3 @@ func GetBucket(c *Context, bucket string) (*ObjectBucket, int, error) {
return &ObjectBucket{Name: bucket, ObjectBucketMetadata: ObjectBucketMetadata{Owner: metadata.Owner, CreatedAt: metadata.CreatedAt}, ObjectBucketStats: *stat}, RGWErrorNone, nil
}
func DeleteObjectBucket(c *Context, bucketName string, purge bool) (int, error) {
options := []string{"bucket", "rm", "--bucket", bucketName}
if purge {
options = append(options, "--purge-objects")
}
result, err := runAdminCommand(c, false, options...)
if err != nil {
return RGWErrorUnknown, errors.Wrap(err, "failed to delete bucket")
}
if result == "" {
return RGWErrorNone, nil
}
if strings.Contains(result, "could not get bucket info for bucket=") {
return RGWErrorNotFound, errors.New("Bucket not found")
}
return RGWErrorUnknown, errors.Wrap(err, "failed to delete bucket")
}
+124 -35
View File
@@ -17,12 +17,20 @@ limitations under the License.
package bucket
import (
"context"
"fmt"
"net/http"
"strconv"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/ceph/go-ceph/rgw/admin"
bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
apibkt "github.com/kube-object-storage/lib-bucket-provisioner/pkg/provisioner/api"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
storagev1 "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/api/resource"
"github.com/pkg/errors"
"github.com/rook/rook/pkg/clusterd"
@@ -47,6 +55,7 @@ type Provisioner struct {
endpoint string
additionalConfigData map[string]string
tlsCert []byte
adminOpsClient *admin.API
}
var _ apibkt.Provisioner = &Provisioner{}
@@ -55,8 +64,6 @@ func NewProvisioner(context *clusterd.Context, clusterInfo *client.ClusterInfo)
return &Provisioner{context: context, clusterInfo: clusterInfo}
}
const maxBuckets = 1
// Provision creates an s3 bucket and returns a connection info
// representing the bucket's endpoint and user access credentials.
func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBucket, error) {
@@ -89,12 +96,13 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje
return nil, err
}
_, err = cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, maxBuckets)
singleBucketQuota := 1
_, err = p.adminOpsClient.ModifyUser(context.TODO(), admin.User{ID: p.cephUserName, MaxBuckets: &singleBucketQuota})
if err != nil {
p.deleteOBCResourceLogError(p.bucketName)
return nil, err
}
logger.Infof("set user %q bucket max to %d", p.cephUserName, maxBuckets)
logger.Infof("set user %q bucket max to %d", p.cephUserName, singleBucketQuota)
// setting quota limit if it is enabled
err = p.setAdditionalSettings(options)
@@ -130,25 +138,27 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu
}
// need to quota into -1 for restricting creation of new buckets in rgw
_, err = cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, -1)
restrictBucketCreation := -1
_, err = p.adminOpsClient.ModifyUser(context.TODO(), admin.User{ID: p.cephUserName, MaxBuckets: &restrictBucketCreation})
if err != nil {
p.deleteOBCResourceLogError("")
return nil, err
}
// get the bucket's owner via the bucket metadata
stats, _, err := cephObject.GetBucket(p.objectContext, p.bucketName)
stats, err := p.adminOpsClient.GetBucketInfo(context.TODO(), admin.Bucket{Bucket: p.bucketName})
if err != nil {
p.deleteOBCResourceLogError("")
return nil, errors.Wrapf(err, "could not get bucket stats (bucket: %s)", p.bucketName)
}
objectUser, _, err := cephObject.GetUser(p.objectContext, stats.Owner)
if err != nil {
p.deleteOBCResourceLogError("")
return nil, errors.Wrapf(err, "could not get user (user: %s)", stats.Owner)
return nil, errors.Wrapf(err, "failed to get bucket %q stats", p.bucketName)
}
s3svc, err := cephObject.NewS3Agent(*objectUser.AccessKey, *objectUser.SecretKey, p.getObjectStoreEndpoint(), true, p.tlsCert)
objectUser, err := p.adminOpsClient.GetUser(context.TODO(), admin.User{ID: stats.Owner})
if err != nil {
p.deleteOBCResourceLogError("")
return nil, errors.Wrapf(err, "failed to get user %q", stats.Owner)
}
s3svc, err := cephObject.NewS3Agent(objectUser.Keys[0].AccessKey, objectUser.Keys[0].SecretKey, p.getObjectStoreEndpoint(), true, p.tlsCert)
if err != nil {
p.deleteOBCResourceLogError("")
return nil, err
@@ -209,7 +219,7 @@ func (p Provisioner) Delete(ob *bktv1alpha1.ObjectBucket) error {
logger.Infof("Delete: deleting bucket %q for OB %q", p.bucketName, ob.Name)
if err := p.deleteOBCResource(p.bucketName); err != nil {
return errors.Wrapf(err, "error deleting OBCResource bucket %q", p.bucketName)
return errors.Wrapf(err, "failed to delete OBCResource bucket %q", p.bucketName)
}
return nil
}
@@ -225,25 +235,26 @@ func (p Provisioner) Revoke(ob *bktv1alpha1.ObjectBucket) error {
}
logger.Infof("Revoke: denying access to bucket %q for OB %q", p.bucketName, ob.Name)
bucket, _, err := cephObject.GetBucket(p.objectContext, p.bucketName)
bucket, err := p.adminOpsClient.GetBucketInfo(context.TODO(), admin.Bucket{Bucket: p.bucketName})
if err != nil {
logger.Errorf("%v", err)
} else {
if bucket.Owner == "" {
return errors.New("cannot find bucket owner")
return errors.Errorf("failed to find bucket %q owner", p.bucketName)
}
user, code, err := cephObject.GetUser(p.objectContext, bucket.Owner)
// The user may not exist. Ignore this in order to ensure the PolicyStatement does not contain the
// stale user.
if err != nil && code != cephObject.RGWErrorNotFound {
user, err := p.adminOpsClient.GetUser(context.TODO(), admin.User{ID: bucket.Owner})
if err != nil {
if errors.Is(err, admin.ErrNoSuchUser) {
// The user may not exist. Ignore this in order to ensure the PolicyStatement does not contain the
// stale user.
return nil
}
return err
} else if user == nil {
logger.Errorf("querying user %q returned nil", p.cephUserName)
return nil
}
s3svc, err := cephObject.NewS3Agent(*user.AccessKey, *user.SecretKey, p.getObjectStoreEndpoint(), true, p.tlsCert)
s3svc, err := cephObject.NewS3Agent(user.Keys[0].AccessKey, user.Keys[0].SecretKey, p.getObjectStoreEndpoint(), true, p.tlsCert)
if err != nil {
return err
}
@@ -343,6 +354,17 @@ func (p *Provisioner) initializeCreateOrGrant(options *apibkt.BucketOptions) err
return errors.Wrapf(err, "failed to set CA cert for the OBC %q to connect with object store %q via TLS", obc.Name, p.objectStoreName)
}
// Set admin ops api client
err = p.setAdminOpsAPIClient()
if err != nil {
// Replace the error with a nicer more comprehensive one
// If the ceph config is not initialized yet, the radosgw-admin command will fail to retrieve the user
if strings.Contains(err.Error(), opcontroller.OperatorNotInitializedMessage) {
return errors.New(opcontroller.OperatorNotInitializedMessage)
}
return errors.Wrap(err, "failed to set admin ops api client")
}
return nil
}
@@ -372,6 +394,18 @@ func (p *Provisioner) initializeDeleteOrRevoke(ob *bktv1alpha1.ObjectBucket) err
if err != nil {
return errors.Wrapf(err, "failed to set CA cert for the OB %q to connect with object store %q via TLS", ob.Name, p.objectStoreName)
}
// Set admin ops api client
err = p.setAdminOpsAPIClient()
if err != nil {
// Replace the error with a nicer more comprehensive one
// If the ceph config is not initialized yet, the radosgw-admin command will fail to retrieve the user
if strings.Contains(err.Error(), opcontroller.OperatorNotInitializedMessage) {
return errors.New(opcontroller.OperatorNotInitializedMessage)
}
return errors.Wrap(err, "failed to set admin ops api client")
}
return nil
}
@@ -522,33 +556,53 @@ func (p *Provisioner) deleteOBCResourceLogError(bucketname string) {
// Check for additional options mentioned in OBC and set them accordingly
func (p Provisioner) setAdditionalSettings(options *apibkt.BucketOptions) error {
quotaEnabled := true
maxObjects := MaxObjectQuota(options)
maxSize := MaxSizeQuota(options)
if maxObjects == "" && maxSize == "" {
return nil
}
var err error
// Enabling quota for the user
err := p.adminOpsClient.SetUserQuota(context.TODO(), admin.QuotaSpec{UID: p.cephUserName, Enabled: &quotaEnabled})
if err != nil {
return errors.Wrapf(err, "failed to enable user %q quota for obc", p.cephUserName)
}
if maxObjects != "" {
if _, err = cephObject.SetQuotaUserObjectMax(p.objectContext, p.cephUserName, maxObjects); err != nil {
logger.Errorf("Setting MaxObject failed %v", err)
return err
maxObjectsInt, err := strconv.Atoi(maxObjects)
if err != nil {
return errors.Wrap(err, "failed to convert maxObjects to integer")
}
maxObjectsInt64 := int64(maxObjectsInt)
err = p.adminOpsClient.SetUserQuota(context.TODO(), admin.QuotaSpec{UID: p.cephUserName, MaxObjects: &maxObjectsInt64})
if err != nil {
return errors.Wrapf(err, "failed to set MaxObject to user %q", p.cephUserName)
}
}
if maxSize != "" {
if _, err = cephObject.SetQuotaUserMaxSize(p.objectContext, p.cephUserName, maxSize); err != nil {
logger.Errorf("Setting MaxSize failed %v", err)
return err
maxSizeInt, err := maxSizeToInt64(maxSize)
if err != nil {
return errors.Wrapf(err, "failed to parse maxSize quota for user %q", p.cephUserName)
}
err = p.adminOpsClient.SetUserQuota(context.TODO(), admin.QuotaSpec{UID: p.cephUserName, MaxSize: &maxSizeInt})
if err != nil {
return errors.Wrapf(err, "failed to set MaxSize to user %q", p.cephUserName)
}
}
if _, err = cephObject.EnableUserQuota(p.objectContext, p.cephUserName); err != nil {
logger.Errorf("Enabling user quota for obc failed %v", err)
return err
}
return nil
}
func maxSizeToInt64(maxSize string) (int64, error) {
maxSizeInt, err := resource.ParseQuantity(maxSize)
if err != nil {
return 0, errors.Wrap(err, "failed to parse quantity")
}
return maxSizeInt.Value(), nil
}
func (p *Provisioner) setTlsCaCert() error {
objStore, err := p.getObjectStore()
if err != nil {
@@ -564,3 +618,38 @@ func (p *Provisioner) setTlsCaCert() error {
return nil
}
func (p *Provisioner) setAdminOpsAPIClient() error {
// Build TLS transport for the HTTP client if needed
httpClient := &http.Client{
Timeout: time.Second * 15,
}
if p.tlsCert != nil {
httpClient.Transport = cephObject.BuildTransportTLS(p.tlsCert)
}
// Fetch the ceph object store
cephObjectStore, err := p.getObjectStore()
if err != nil {
return errors.Wrapf(err, "failed to get ceph object store %q", p.objectStoreName)
}
// Fetch the object store admin ops user
accessKey, secretKey, err := cephObject.GetAdminOPSUserCredentials(p.context, p.clusterInfo, p.objectContext, cephObjectStore)
if err != nil {
return errors.Wrap(err, "failed to retrieve rgw admin ops user")
}
// Build endpoint
s3endpoint := cephObject.BuildDNSEndpoint(cephObject.BuildDomainName(p.objectContext.Name, cephObjectStore.Namespace), p.storePort, cephObjectStore.Spec.IsTLSEnabled())
// Initialize object store admin ops API
adminOpsClient, err := admin.New(s3endpoint, accessKey, secretKey, httpClient)
if err != nil {
return errors.Wrap(err, "failed to build object store admin ops API connection")
}
p.adminOpsClient = adminOpsClient
return nil
}
@@ -104,3 +104,31 @@ func TestPopulateDomainAndPort(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, "rook-ceph-rgw-test-store.ns.svc", p.storeDomainName)
}
func TestMaxSizeToInt64(t *testing.T) {
type args struct {
maxSize string
}
tests := []struct {
name string
args args
want int64
wantErr bool
}{
{"invalid size", args{maxSize: "foo"}, 0, true},
{"2gb size is invalid", args{maxSize: "2g"}, 0, true},
{"2G size is valid", args{maxSize: "2G"}, 2000000000, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := maxSizeToInt64(tt.args.maxSize)
if (err != nil) != tt.wantErr {
t.Errorf("maxSizeToInt64() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("maxSizeToInt64() = %v, want %v", got, tt.want)
}
})
}
}
+33 -24
View File
@@ -1,28 +1,34 @@
package bucket
import (
"context"
"fmt"
"github.com/ceph/go-ceph/rgw/admin"
"github.com/pkg/errors"
cephObject "github.com/rook/rook/pkg/operator/ceph/object"
)
func (p *Provisioner) bucketExists(name string) (bool, error) {
_, errCode, err := cephObject.GetBucket(p.objectContext, name)
if errCode != 0 {
return false, errors.Wrapf(err, "error getting ceph bucket %q", name)
_, err := p.adminOpsClient.GetBucketInfo(context.TODO(), admin.Bucket{Bucket: name})
if err != nil {
if errors.Is(err, admin.ErrNoSuchBucket) {
return false, nil
}
return false, errors.Wrapf(err, "failed to get ceph bucket %q", name)
}
return true, nil
}
func (p *Provisioner) userExists(name string) (bool, error) {
_, errCode, err := cephObject.GetUser(p.objectContext, name)
if errCode == cephObject.RGWErrorNotFound {
return false, nil
}
if errCode > 0 {
return false, errors.Wrapf(err, "error getting ceph user %q", name)
_, err := p.adminOpsClient.GetUser(context.TODO(), admin.User{ID: name})
if err != nil {
if errors.Is(err, admin.ErrNoSuchUser) {
return false, nil
} else {
return false, errors.Wrapf(err, "failed to get ceph user %q", name)
}
}
return true, nil
}
@@ -38,18 +44,18 @@ func (p *Provisioner) createCephUser(username string) (accKey string, secKey str
p.cephUserName = username
logger.Infof("creating Ceph user %q", username)
userConfig := cephObject.ObjectUser{
UserID: username,
DisplayName: &p.cephUserName,
userConfig := admin.User{
ID: username,
DisplayName: p.cephUserName,
}
u, errCode, err := cephObject.CreateUser(p.objectContext, userConfig)
if err != nil || errCode != 0 {
return "", "", errors.Wrapf(err, "error creating ceph user %q: %v", username, err)
u, err := p.adminOpsClient.CreateUser(context.TODO(), userConfig)
if err != nil {
return "", "", errors.Wrapf(err, "failed to create ceph user %q: %v", username, err)
}
logger.Infof("successfully created Ceph user %q with access keys", username)
return *u.AccessKey, *u.SecretKey, nil
return u.Keys[0].AccessKey, u.Keys[0].SecretKey, nil
}
// returns "" if unable to generate a unique name.
@@ -81,21 +87,24 @@ func (p *Provisioner) deleteOBCResource(bucketName string) error {
logger.Infof("deleting Ceph user %q and bucket %q", p.cephUserName, bucketName)
if len(bucketName) > 0 {
// delete bucket with purge option to remove all objects
errCode, err := cephObject.DeleteObjectBucket(p.objectContext, bucketName, true)
if errCode == cephObject.RGWErrorNone {
thePurge := true
err := p.adminOpsClient.RemoveBucket(context.TODO(), admin.Bucket{Bucket: bucketName, PurgeObject: &thePurge})
if err == nil {
logger.Infof("bucket %q successfully deleted", p.bucketName)
} else if errCode == cephObject.RGWErrorNotFound {
} else if errors.Is(err, admin.ErrNoSuchBucket) {
// opinion: "not found" is not an error
logger.Infof("bucket %q does not exist", p.bucketName)
} else {
return errors.Wrapf(err, "failed to delete bucket %q: errCode: %d", bucketName, errCode)
return errors.Wrapf(err, "failed to delete bucket %q", bucketName)
}
}
if len(p.cephUserName) > 0 {
output, err := cephObject.DeleteUser(p.objectContext, p.cephUserName)
err := p.adminOpsClient.RemoveUser(context.TODO(), admin.User{ID: p.cephUserName})
if err != nil {
logger.Warningf("failed to delete user %q. %s. %v", p.cephUserName, output, err)
if errors.Is(err, admin.ErrNoSuchUser) {
logger.Warningf("user %q does not exist, nothing to delete. %v", p.cephUserName, err)
}
logger.Warningf("failed to delete user %q. %v", p.cephUserName, err)
} else {
logger.Infof("user %q successfully deleted", p.cephUserName)
}
+2 -2
View File
@@ -88,7 +88,7 @@ func (p *Provisioner) getObjectStore() (*cephv1.CephObjectStore, error) {
if kerrors.IsNotFound(err) {
return nil, errors.Wrap(err, "cephObjectStore not found")
}
return nil, errors.Wrap(err, "error getting cephObjectStore")
return nil, errors.Wrapf(err, "failed to get ceph object store %q", p.objectStoreName)
}
return store, err
}
@@ -101,7 +101,7 @@ func getService(c kubernetes.Interface, namespace, name string) (*v1.Service, er
if kerrors.IsNotFound(err) {
return nil, errors.Wrap(err, "cephObjectStore service not found")
}
return nil, errors.Wrap(err, "error getting cephObjectStore service")
return nil, errors.Wrapf(err, "failed to get ceph object store service %q", name)
}
return svc, nil
}
+12 -2
View File
@@ -102,6 +102,7 @@ func newReconciler(mgr manager.Manager, context *clusterd.Context) reconcile.Rec
if err := cephv1.AddToScheme(mgr.GetScheme()); err != nil {
panic(err)
}
context.Client = mgr.GetClient()
return &ReconcileCephObjectStore{
client: mgr.GetClient(),
scheme: mgrScheme,
@@ -241,7 +242,7 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, opconfig.MonType)
if err != nil {
if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
logger.Info("skipping reconcile since operator is still initializing")
logger.Info(opcontroller.OperatorNotInitializedMessage)
return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil
}
return reconcile.Result{}, errors.Wrapf(err, "failed to retrieve current ceph %q version", opconfig.MonType)
@@ -357,7 +358,6 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
if err != nil {
return r.setFailedStatus(namespacedName, "failed to reconcile external endpoint", err)
}
} else {
logger.Info("reconciling object store deployments")
@@ -522,6 +522,16 @@ func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjec
}
rgwChecker := newBucketChecker(r.context, objContext, port, r.client, namespacedName, &objectstore.Spec)
// Fetch the admin ops user
accessKey, secretKey, err := GetAdminOPSUserCredentials(r.context, r.clusterInfo, objContext, objectstore)
if err != nil {
logger.Errorf("failed to create or retrieve rgw admin ops user. %v", err)
return
}
rgwChecker.objContext.adminOpsUserAccessKey = accessKey
rgwChecker.objContext.adminOpsUserSecretKey = secretKey
logger.Info("starting rgw healthcheck")
go rgwChecker.checkObjectStore(r.objectStoreChannels[objectstore.Name].stopChan)
}
@@ -617,6 +617,9 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) {
if args[0] == "zone" && args[1] == "get" {
return zoneGetMultisiteJSON, nil
}
if args[0] == "user" && args[1] == "create" {
return userCreateJSON, nil
}
return "", nil
},
}
+52 -33
View File
@@ -17,9 +17,12 @@ limitations under the License.
package object
import (
"context"
"fmt"
"net/http"
"time"
"github.com/ceph/go-ceph/rgw/admin"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
@@ -52,9 +55,9 @@ type bucketChecker struct {
}
// newbucketChecker creates a new HealthChecker object
func newBucketChecker(context *clusterd.Context, objContext *Context, port int32, client client.Client, namespacedName types.NamespacedName, objectStoreSpec *cephv1.ObjectStoreSpec) *bucketChecker {
func newBucketChecker(ctx *clusterd.Context, objContext *Context, port int32, client client.Client, namespacedName types.NamespacedName, objectStoreSpec *cephv1.ObjectStoreSpec) *bucketChecker {
c := &bucketChecker{
context: context,
context: ctx,
objContext: objContext,
interval: &defaultHealthCheckInterval,
port: port,
@@ -119,39 +122,51 @@ func (c *bucketChecker) checkObjectStoreHealth() error {
Always keep the bucket and the user for the health check, just do PUT and GET because bucket creation is expensive
*/
var tlsCert []byte
var s3AccessKey string
var s3SecretKey string
var tlsCert []byte
s3endpoint := buildDNSEndpoint(BuildDomainName(c.objContext.Name, c.namespacedName.Namespace), c.port, c.objectStoreSpec.IsTLSEnabled())
var err error
s3endpoint := BuildDNSEndpoint(BuildDomainName(c.objContext.Name, c.namespacedName.Namespace), c.port, c.objectStoreSpec.IsTLSEnabled())
// Generate unique user and bucket name
bucketName := genUniqueBucketName(c.objContext.UID)
userConfig := c.genUserConfig()
// Create S3 user
logger.Debugf("creating s3 user object %q for object store %q", userConfig.UserID, c.namespacedName.Name)
user, rgwerr, err := CreateUser(c.objContext, userConfig)
if err != nil {
if rgwerr == ErrorCodeFileExists {
user, _, err = GetUser(c.objContext, userConfig.UserID)
if err != nil {
return errors.Wrapf(err, "failed to get details from ceph object user %q for object store %q", userConfig.UserID, c.namespacedName.Name)
}
} else {
return errors.Wrapf(err, "failed to create object user %q. error code %d for object store %q", userConfig.UserID, rgwerr, c.namespacedName.Name)
}
}
// Set access and secret key
s3AccessKey = *user.AccessKey
s3SecretKey = *user.SecretKey
var httpClient *http.Client
if c.objectStoreSpec.IsTLSEnabled() {
tlsCert, err = GetTlsCaCert(c.objContext, c.objectStoreSpec)
if err != nil {
return errors.Wrapf(err, "failed to fetch CA cert for the user %q to establish TLS connection with the object store %q", userConfig.UserID, c.namespacedName.Name)
return errors.Wrapf(err, "failed to fetch CA cert for the user %q to establish TLS connection with the object store %q", userConfig.ID, c.namespacedName.Name)
}
httpClient.Transport = BuildTransportTLS(tlsCert)
}
// Build admin Ops API connection
co, err := admin.New(s3endpoint, c.objContext.adminOpsUserAccessKey, c.objContext.adminOpsUserSecretKey, httpClient)
if err != nil {
return errors.Wrap(err, "failed to build admin ops API connection")
}
c.objContext.adminOpsClient = co
// Create checker user
logger.Debugf("creating s3 user object %q for object store %q", userConfig.ID, c.namespacedName.Name)
var user admin.User
user, err = co.CreateUser(context.TODO(), userConfig)
if err != nil {
if errors.Is(err, admin.ErrUserExists) {
user, err = co.GetUser(context.TODO(), userConfig)
if err != nil {
return errors.Wrapf(err, "failed to get details from ceph object user %q", userConfig.ID)
}
} else {
return errors.Wrapf(err, "failed to create ceph object user %q", userConfig.ID)
}
}
// Set access and secret key
s3AccessKey = user.Keys[0].AccessKey
s3SecretKey = user.Keys[0].SecretKey
// Initiate s3 agent
logger.Debugf("initializing s3 connection for object store %q", c.namespacedName.Name)
s3client, err := NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, false, tlsCert)
@@ -189,18 +204,22 @@ func (c *bucketChecker) cleanupHealthCheck() {
bucketToDelete := genUniqueBucketName(c.objContext.UID)
logger.Infof("deleting object %q from bucket %q in object store %q", s3HealthCheckObjectKey, bucketToDelete, c.namespacedName.Name)
_, err := DeleteObjectBucket(c.objContext, bucketToDelete, true)
if err != nil {
thePurge := true
err := c.objContext.adminOpsClient.RemoveBucket(context.TODO(), admin.Bucket{Bucket: bucketToDelete, PurgeObject: &thePurge})
if errors.Is(err, admin.ErrNoSuchBucket) {
// opinion: "not found" is not an error
logger.Debugf("bucket %q does not exist", bucketToDelete)
} else {
logger.Errorf("failed to delete bucket %q for object store %q. %v", bucketToDelete, c.namespacedName.Name, err)
}
userToDelete := c.genUserConfig()
output, err := DeleteUser(c.objContext, userToDelete.UserID)
if err != nil {
logger.Errorf("failed to delete object user %q for object store %q. %s. %v", userToDelete.UserID, c.namespacedName.Name, output, err)
} else {
logger.Debugf("successfully deleted object user %q for object store %q", userToDelete.UserID, c.namespacedName.Name)
err = c.objContext.adminOpsClient.RemoveUser(context.TODO(), userToDelete)
if err != nil && !errors.Is(err, admin.ErrNoSuchUser) {
logger.Errorf("failed to delete object user %q for object store %q. %v", userToDelete.ID, c.namespacedName.Name, err)
}
logger.Debugf("successfully deleted object user %q for object store %q", userToDelete.ID, c.namespacedName.Name)
}
func toCustomResourceStatus(currentStatus *cephv1.BucketStatus, details string, health cephv1.ConditionType) *cephv1.BucketStatus {
@@ -223,12 +242,12 @@ func genUniqueBucketName(uuid string) string {
return fmt.Sprintf("%s-%s", s3HealthCheckBucketName, uuid)
}
func (c *bucketChecker) genUserConfig() ObjectUser {
func (c *bucketChecker) genUserConfig() admin.User {
userName := fmt.Sprintf("%s-%s", s3UserHealthCheckName, c.objContext.UID)
return ObjectUser{
UserID: userName,
DisplayName: &userName,
return admin.User{
ID: userName,
DisplayName: userName,
}
}
+26 -25
View File
@@ -28,8 +28,7 @@ import (
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
ceph "github.com/rook/rook/pkg/daemon/ceph/client"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/ceph/cluster/mgr"
"github.com/rook/rook/pkg/operator/ceph/config"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
@@ -591,21 +590,21 @@ func deletePools(context *Context, spec cephv1.ObjectStoreSpec, lastStore bool)
for _, pool := range pools {
name := poolName(context.Name, pool)
if err := ceph.DeletePool(context.Context, context.clusterInfo, name); err != nil {
if err := cephclient.DeletePool(context.Context, context.clusterInfo, name); err != nil {
logger.Warningf("failed to delete pool %q. %v", name, err)
}
}
// Delete erasure code profile if any
erasureCodes, err := ceph.ListErasureCodeProfiles(context.Context, context.clusterInfo)
erasureCodes, err := cephclient.ListErasureCodeProfiles(context.Context, context.clusterInfo)
if err != nil {
return errors.Wrapf(err, "failed to list erasure code profiles for cluster %s", context.clusterInfo.Namespace)
}
// cleans up the EC profile for the data pool only. Metadata pools don't support EC (only replication is supported).
ecProfileName := ceph.GetErasureCodeProfileForPool(context.Name)
ecProfileName := cephclient.GetErasureCodeProfileForPool(context.Name)
for i := range erasureCodes {
if erasureCodes[i] == ecProfileName {
if err := ceph.DeleteErasureCodeProfile(context.Context, context.clusterInfo, ecProfileName); err != nil {
if err := cephclient.DeleteErasureCodeProfile(context.Context, context.clusterInfo, ecProfileName); err != nil {
return errors.Wrapf(err, "failed to delete erasure code profile %s for object store %s", ecProfileName, context.Name)
}
break
@@ -623,7 +622,7 @@ func CreatePools(context *Context, clusterSpec *cephv1.ClusterSpec, metadataPool
var missingPools []string
for _, pool := range pools {
poolName := poolName(context.Name, pool)
_, err := ceph.GetPoolDetails(context.Context, context.clusterInfo, poolName)
_, err := cephclient.GetPoolDetails(context.Context, context.clusterInfo, poolName)
if err != nil {
logger.Debugf("failed to find pool %q. %v", poolName, err)
missingPools = append(missingPools, poolName)
@@ -638,7 +637,7 @@ func CreatePools(context *Context, clusterSpec *cephv1.ClusterSpec, metadataPool
metadataPoolPGs, err := config.GetMonStore(context.Context, context.clusterInfo).Get("mon.", "rgw_rados_pool_pg_num_min")
if err != nil {
logger.Warningf("failed to adjust the PG count for rgw metadata pools. using the general default. %v", err)
metadataPoolPGs = ceph.DefaultPGCount
metadataPoolPGs = cephclient.DefaultPGCount
}
if err := createSimilarPools(context, append(metadataPools, rootPool), clusterSpec, metadataPool, metadataPoolPGs, ""); err != nil {
@@ -647,14 +646,14 @@ func CreatePools(context *Context, clusterSpec *cephv1.ClusterSpec, metadataPool
ecProfileName := ""
if dataPool.IsErasureCoded() {
ecProfileName = ceph.GetErasureCodeProfileForPool(context.Name)
ecProfileName = cephclient.GetErasureCodeProfileForPool(context.Name)
// create a new erasure code profile for the data pool
if err := ceph.CreateErasureCodeProfile(context.Context, context.clusterInfo, ecProfileName, dataPool); err != nil {
if err := cephclient.CreateErasureCodeProfile(context.Context, context.clusterInfo, ecProfileName, dataPool); err != nil {
return errors.Wrap(err, "failed to create erasure code profile")
}
}
if err := createSimilarPools(context, []string{dataPoolName}, clusterSpec, dataPool, ceph.DefaultPGCount, ecProfileName); err != nil {
if err := createSimilarPools(context, []string{dataPoolName}, clusterSpec, dataPool, cephclient.DefaultPGCount, ecProfileName); err != nil {
return errors.Wrap(err, "failed to create data pool")
}
@@ -665,16 +664,16 @@ func createSimilarPools(context *Context, pools []string, clusterSpec *cephv1.Cl
for _, pool := range pools {
// create the pool if it doesn't exist yet
name := poolName(context.Name, pool)
if poolDetails, err := ceph.GetPoolDetails(context.Context, context.clusterInfo, name); err != nil {
if poolDetails, err := cephclient.GetPoolDetails(context.Context, context.clusterInfo, name); err != nil {
// If the ceph config has an EC profile, an EC pool must be created. Otherwise, it's necessary
// to create a replicated pool.
var err error
if poolSpec.IsErasureCoded() {
// An EC pool backing an object store does not need to enable EC overwrites, so the pool is
// created with that property disabled to avoid unnecessary performance impact.
err = ceph.CreateECPoolForApp(context.Context, context.clusterInfo, name, ecProfileName, poolSpec, pgCount, AppName, false /* enableECOverwrite */)
err = cephclient.CreateECPoolForApp(context.Context, context.clusterInfo, name, ecProfileName, poolSpec, pgCount, AppName, false /* enableECOverwrite */)
} else {
err = ceph.CreateReplicatedPoolForApp(context.Context, context.clusterInfo, clusterSpec, name, poolSpec, pgCount, AppName)
err = cephclient.CreateReplicatedPoolForApp(context.Context, context.clusterInfo, clusterSpec, name, poolSpec, pgCount, AppName)
}
if err != nil {
return errors.Wrapf(err, "failed to create pool %s for object store %s.", name, context.Name)
@@ -685,15 +684,15 @@ func createSimilarPools(context *Context, pools []string, clusterSpec *cephv1.Cl
// detect if the replication is different from the pool details
if poolDetails.Size != poolSpec.Replicated.Size {
logger.Infof("pool size is changed from %d to %d", poolDetails.Size, poolSpec.Replicated.Size)
if err := ceph.SetPoolReplicatedSizeProperty(context.Context, context.clusterInfo, poolDetails.Name, strconv.FormatUint(uint64(poolSpec.Replicated.Size), 10)); err != nil {
if err := cephclient.SetPoolReplicatedSizeProperty(context.Context, context.clusterInfo, poolDetails.Name, strconv.FormatUint(uint64(poolSpec.Replicated.Size), 10)); err != nil {
return errors.Wrapf(err, "failed to set size property to replicated pool %q to %d", poolDetails.Name, poolSpec.Replicated.Size)
}
}
}
}
// Set the pg_num_min if not the default so the autoscaler won't immediately increase the pg count
if pgCount != ceph.DefaultPGCount {
if err := ceph.SetPoolProperty(context.Context, context.clusterInfo, name, "pg_num_min", pgCount); err != nil {
if pgCount != cephclient.DefaultPGCount {
if err := cephclient.SetPoolProperty(context.Context, context.clusterInfo, name, "pg_num_min", pgCount); err != nil {
return errors.Wrapf(err, "failed to set pg_num_min on pool %q to %q", name, pgCount)
}
}
@@ -726,7 +725,7 @@ func GetObjectBucketProvisioner(c *clusterd.Context, namespace string) string {
// CheckDashboardUser returns true if the user is configure else return false
func checkDashboardUser(context *Context) (bool, error) {
args := []string{"dashboard", "get-rgw-api-access-key"}
cephCmd := ceph.NewCephCommand(context.Context, context.clusterInfo, args)
cephCmd := cephclient.NewCephCommand(context.Context, context.clusterInfo, args)
out, err := cephCmd.Run()
if string(out) != "" {
@@ -752,6 +751,8 @@ func enableRGWDashboard(context *Context) error {
DisplayName: &DashboardUser,
SystemUser: true,
}
// TODO:
// Use admin ops user instead!
u, errCode, err := CreateUser(context, user)
if err != nil || errCode != 0 {
return errors.Wrapf(err, "failed to create user %q", DashboardUser)
@@ -786,19 +787,19 @@ func enableRGWDashboard(context *Context) error {
secretArgs = []string{"dashboard", "set-rgw-api-secret-key", *u.SecretKey}
}
cephCmd := ceph.NewCephCommand(context.Context, context.clusterInfo, accessArgs)
cephCmd := cephclient.NewCephCommand(context.Context, context.clusterInfo, accessArgs)
_, err = cephCmd.Run()
if err != nil {
return errors.Wrapf(err, "failed to set user %q accesskey", DashboardUser)
}
cephCmd = ceph.NewCephCommand(context.Context, context.clusterInfo, secretArgs)
cephCmd = cephclient.NewCephCommand(context.Context, context.clusterInfo, secretArgs)
go func() {
// Setting the dashboard api secret started hanging in some clusters
// starting in ceph v15.2.8. We run it in a goroutine until the fix
// is found. We expect the ceph command to timeout so at least the goroutine exits.
logger.Info("setting the dashboard api secret key")
_, err = cephCmd.RunWithTimeout(client.CephCommandTimeout)
_, err = cephCmd.RunWithTimeout(cephclient.CephCommandTimeout)
if err != nil {
logger.Errorf("failed to set user %q secretkey. %v", DashboardUser, err)
}
@@ -829,15 +830,15 @@ func disableRGWDashboard(context *Context) {
}
args := []string{"dashboard", "reset-rgw-api-access-key"}
cephCmd := ceph.NewCephCommand(context.Context, context.clusterInfo, args)
_, err = cephCmd.RunWithTimeout(client.CephCommandTimeout)
cephCmd := cephclient.NewCephCommand(context.Context, context.clusterInfo, args)
_, err = cephCmd.RunWithTimeout(cephclient.CephCommandTimeout)
if err != nil {
logger.Warningf("failed to reset user accesskey for user %q. %v", DashboardUser, err)
}
args = []string{"dashboard", "reset-rgw-api-secret-key"}
cephCmd = ceph.NewCephCommand(context.Context, context.clusterInfo, args)
_, err = cephCmd.RunWithTimeout(client.CephCommandTimeout)
cephCmd = cephclient.NewCephCommand(context.Context, context.clusterInfo, args)
_, err = cephCmd.RunWithTimeout(cephclient.CephCommandTimeout)
if err != nil {
logger.Warningf("failed to reset user secretkey for user %q. %v", DashboardUser, err)
}
+2 -2
View File
@@ -310,8 +310,8 @@ func BuildDomainName(name, namespace string) string {
return fmt.Sprintf("%s-%s.%s.%s", AppName, name, namespace, svcDNSSuffix)
}
// buildDNSEndpoint build the dns name to reach out the service endpoint
func buildDNSEndpoint(domainName string, port int32, secure bool) string {
// BuildDNSEndpoint build the dns name to reach out the service endpoint
func BuildDNSEndpoint(domainName string, port int32, secure bool) string {
httpPrefix := "http"
if secure {
httpPrefix = "https"
+2 -2
View File
@@ -167,11 +167,11 @@ func TestBuildDomainNameAndEndpoint(t *testing.T) {
// non-secure endpoint
var port int32 = 80
ep := buildDNSEndpoint(dns, port, false)
ep := BuildDNSEndpoint(dns, port, false)
assert.Equal(t, "http://rook-ceph-rgw-my-store.rook-ceph.svc:80", ep)
// Secure endpoint
var securePort int32 = 443
ep = buildDNSEndpoint(dns, securePort, true)
ep = BuildDNSEndpoint(dns, securePort, true)
assert.Equal(t, "https://rook-ceph-rgw-my-store.rook-ceph.svc:443", ep)
}
+10 -7
View File
@@ -49,13 +49,7 @@ func NewS3Agent(accessKey, secretKey, endpoint string, debug bool, tlsCert []byt
}
tlsEnabled := false
if len(tlsCert) > 0 {
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(tlsCert)
tr := &http.Transport{
TLSClientConfig: &tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12},
}
client.Transport = tr
client.Transport = BuildTransportTLS(tlsCert)
tlsEnabled = true
}
sess, err := session.NewSession(
@@ -193,3 +187,12 @@ func (s *S3Agent) DeleteObjectInBucket(bucketname string, key string) (bool, err
}
return true, nil
}
func BuildTransportTLS(tlsCert []byte) *http.Transport {
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(tlsCert)
return &http.Transport{
TLSClientConfig: &tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12},
}
}
+4 -4
View File
@@ -86,12 +86,12 @@ func buildStatusInfo(cephObjectStore *cephv1.CephObjectStore) map[string]string
m := make(map[string]string)
if cephObjectStore.Spec.Gateway.SecurePort != 0 && cephObjectStore.Spec.Gateway.Port != 0 {
m["secureEndpoint"] = buildDNSEndpoint(BuildDomainName(cephObjectStore.Name, cephObjectStore.Namespace), cephObjectStore.Spec.Gateway.SecurePort, true)
m["endpoint"] = buildDNSEndpoint(BuildDomainName(cephObjectStore.Name, cephObjectStore.Namespace), cephObjectStore.Spec.Gateway.Port, false)
m["secureEndpoint"] = BuildDNSEndpoint(BuildDomainName(cephObjectStore.Name, cephObjectStore.Namespace), cephObjectStore.Spec.Gateway.SecurePort, true)
m["endpoint"] = BuildDNSEndpoint(BuildDomainName(cephObjectStore.Name, cephObjectStore.Namespace), cephObjectStore.Spec.Gateway.Port, false)
} else if cephObjectStore.Spec.Gateway.SecurePort != 0 {
m["endpoint"] = buildDNSEndpoint(BuildDomainName(cephObjectStore.Name, cephObjectStore.Namespace), cephObjectStore.Spec.Gateway.SecurePort, true)
m["endpoint"] = BuildDNSEndpoint(BuildDomainName(cephObjectStore.Name, cephObjectStore.Namespace), cephObjectStore.Spec.Gateway.SecurePort, true)
} else {
m["endpoint"] = buildDNSEndpoint(BuildDomainName(cephObjectStore.Name, cephObjectStore.Namespace), cephObjectStore.Spec.Gateway.Port, false)
m["endpoint"] = BuildDNSEndpoint(BuildDomainName(cephObjectStore.Name, cephObjectStore.Namespace), cephObjectStore.Spec.Gateway.Port, false)
}
return m
+27 -132
View File
@@ -18,10 +18,10 @@ package object
import (
"encoding/json"
"strconv"
"strings"
"syscall"
"github.com/ceph/go-ceph/rgw/admin"
"github.com/pkg/errors"
"github.com/rook/rook/pkg/util/exec"
)
@@ -37,47 +37,24 @@ const (
// An ObjectUser defines the details of an object store user.
type ObjectUser struct {
UserID string `json:"userId"`
DisplayName *string `json:"displayName"`
Email *string `json:"email"`
AccessKey *string `json:"accessKey"`
SecretKey *string `json:"secretKey"`
SystemUser bool `json:"systemuser"`
}
// ListUsers lists the object pool users.
func ListUsers(c *Context) ([]string, int, error) {
result, err := runAdminCommand(c, true, "user", "list")
if err != nil {
return nil, RGWErrorUnknown, errors.Wrap(err, "failed to list users")
}
var s []string
if err := json.Unmarshal([]byte(result), &s); err != nil {
return nil, RGWErrorParse, errors.Wrapf(err, "failed to read users info result=%s", result)
}
return s, RGWErrorNone, nil
}
type rgwUserInfo struct {
UserID string `json:"user_id"`
DisplayName string `json:"display_name"`
Email string `json:"email"`
Keys []struct {
AccessKey string `json:"access_key"`
SecretKey string `json:"secret_key"`
}
UserID string `json:"userId"`
DisplayName *string `json:"displayName"`
Email *string `json:"email"`
AccessKey *string `json:"accessKey"`
SecretKey *string `json:"secretKey"`
SystemUser bool `json:"systemuser"`
AdminOpsUser bool `json:"adminopsuser"`
}
// func decodeUser(data string) (*ObjectUser, int, error) {
func decodeUser(data string) (*ObjectUser, int, error) {
var user rgwUserInfo
var user admin.User
err := json.Unmarshal([]byte(data), &user)
if err != nil {
return nil, RGWErrorParse, errors.Wrapf(err, "failed to unmarshal json. %s", data)
}
rookUser := ObjectUser{UserID: user.UserID, DisplayName: &user.DisplayName, Email: &user.Email}
rookUser := ObjectUser{UserID: user.ID, DisplayName: &user.DisplayName, Email: &user.Email}
if len(user.Keys) > 0 {
rookUser.AccessKey = &user.Keys[0].AccessKey
@@ -90,6 +67,8 @@ func decodeUser(data string) (*ObjectUser, int, error) {
}
// GetUser returns the user with the given ID.
// The function is used **ONCE** only to provision so the RGW Admin Ops User
// Subsequent interaction with the API will be done with the created user
func GetUser(c *Context, id string) (*ObjectUser, int, error) {
logger.Debugf("getting s3 user %q", id)
// note: err is set for non-existent user but result output is also empty
@@ -108,6 +87,8 @@ func GetUser(c *Context, id string) (*ObjectUser, int, error) {
}
// CreateUser creates a new user with the information given.
// The function is used **ONCE** only to provision so the RGW Admin Ops User
// Subsequent interaction with the API will be done with the created user
func CreateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) {
logger.Debugf("creating s3 user %q", user.UserID)
@@ -134,6 +115,10 @@ func CreateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) {
args = append(args, "--system")
}
if user.AdminOpsUser {
args = append(args, "--caps", rgwAdminOpsUserCaps)
}
result, err := runAdminCommand(c, true, args...)
if err != nil {
if strings.Contains(result, "could not create user: unable to create user, user: ") {
@@ -143,41 +128,17 @@ func CreateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) {
if strings.Contains(result, "could not create user: unable to create user, email: ") && strings.Contains(result, " is the email address an existing user") {
return nil, RGWErrorBadData, errors.New("email already in use")
}
if strings.Contains(result, "global_init: unable to open config file from search list") {
return nil, RGWErrorUnknown, errors.New("skipping reconcile since operator is still initializing")
}
// We don't know what happened
return nil, RGWErrorUnknown, errors.Wrap(err, "failed to create s3 user")
return nil, RGWErrorUnknown, errors.Wrapf(err, "failed to create s3 user. %s", result)
}
return decodeUser(result)
}
// UpdateUser updates the user whose ID matches the user.
func UpdateUser(c *Context, user ObjectUser) (*ObjectUser, int, error) {
logger.Infof("updating s3 user %q", user.UserID)
args := []string{"user", "modify", "--uid", user.UserID}
if user.DisplayName != nil {
args = append(args, "--display-name", *user.DisplayName)
}
if user.Email != nil {
args = append(args, "--email", *user.Email)
}
body, err := runAdminCommand(c, false, args...)
if err != nil {
return nil, RGWErrorUnknown, errors.Wrap(err, "failed to update s3 user")
}
if body == "could not modify user: unable to modify user, user not found" {
return nil, RGWErrorNotFound, errors.New("s3 user not found")
}
match, err := extractJSON(body)
if err != nil {
return nil, RGWErrorParse, errors.Wrap(err, "failed to get json")
}
return decodeUser(match)
}
func ListUserBuckets(c *Context, id string, opts ...string) (string, error) {
args := []string{"bucket", "list", "--uid", id}
@@ -191,6 +152,8 @@ func ListUserBuckets(c *Context, id string, opts ...string) (string, error) {
}
// DeleteUser deletes the user with the given ID.
// Even though we should be using the Admin Ops API, we keep this on purpose until the entire migration is completed
// Used for the dashboard user
func DeleteUser(c *Context, id string, opts ...string) (string, error) {
args := []string{"user", "rm", "--uid", id}
if opts != nil {
@@ -211,71 +174,3 @@ func DeleteUser(c *Context, id string, opts ...string) (string, error) {
return result, errors.Wrapf(err, "failed to delete s3 user uid=%q", id)
}
// SetQuotaUserBucketMax will set maximum bucket quota for a user
func SetQuotaUserBucketMax(c *Context, id string, max int) (string, error) {
logger.Infof("Setting user %q max buckets to %d", id, max)
args := []string{"--quota-scope", "user", "--max-buckets", strconv.Itoa(max)}
result, err := setUserQuota(c, id, args)
if err != nil {
err = errors.Wrap(err, "failed setting bucket max")
}
return result, err
}
func setUserQuota(c *Context, id string, args []string) (string, error) {
args = append([]string{"quota", "set", "--uid", id}, args...)
result, err := runAdminCommand(c, false, args...)
if err != nil {
err = errors.Wrap(err, "failed to set max buckets for user")
}
return result, err
}
// LinkUser will link a user to a bucket
func LinkUser(c *Context, id, bucket string) (string, int, error) {
logger.Infof("Linking (user: %s) (bucket: %s)", id, bucket)
args := []string{"bucket", "link", "--uid", id, "--bucket", bucket}
result, err := runAdminCommand(c, false, args...)
if err != nil {
return "", RGWErrorUnknown, err
}
if strings.Contains(result, "bucket entry point user mismatch") {
return "", RGWErrorNotFound, err
}
return result, RGWErrorNone, nil
}
// EnableUserQuota will allows to enable quota defined for a user
func EnableUserQuota(c *Context, id string) (string, error) {
logger.Debug("Enabling user quota for %q", id)
args := []string{"quota", "enable", "--quota-scope", "user", "--uid", id}
result, err := runAdminCommand(c, false, args...)
if err != nil {
err = errors.Wrap(err, "failed to enable quota for the user")
}
return result, err
}
// SetQuotaUserObject allows to set maximum limit on objects for a user
func SetQuotaUserObjectMax(c *Context, id string, maxobjects string) (string, error) {
logger.Debugf("Setting user %q max objects to %s", id, maxobjects)
args := []string{"--quota-scope", "user", "--max-objects", maxobjects}
result, err := setUserQuota(c, id, args)
if err != nil {
err = errors.Wrap(err, "failed setting object max")
}
return result, err
}
// SetQuotaUserMaxSize allows to set maximum size for a user
func SetQuotaUserMaxSize(c *Context, id string, maxsize string) (string, error) {
logger.Debugf("Setting user %q max size to %s", id, maxsize)
args := []string{"--quota-scope", "user", "--max-size", maxsize}
result, err := setUserQuota(c, id, args)
if err != nil {
err = errors.Wrap(err, "failed setting max size")
}
return result, err
}
+53 -24
View File
@@ -20,8 +20,10 @@ package objectuser
import (
"context"
"fmt"
"net/http"
"reflect"
"github.com/ceph/go-ceph/rgw/admin"
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
"k8s.io/apimachinery/pkg/runtime"
@@ -67,9 +69,10 @@ type ReconcileObjectStoreUser struct {
scheme *runtime.Scheme
context *clusterd.Context
objContext *object.Context
userConfig object.ObjectUser
userConfig *admin.User
cephClusterSpec *cephv1.ClusterSpec
clusterInfo *cephclient.ClusterInfo
adminOpsAPI *admin.API
}
// Add creates a new CephObjectStoreUser Controller and adds it to the Manager. The Manager will set fields on the Controller
@@ -186,7 +189,7 @@ func (r *ReconcileObjectStoreUser) reconcile(request reconcile.Request) (reconci
return reconcile.Result{}, errors.Wrap(err, "failed to populate cluster info")
}
// validate initializeObjectStoreContext
// Validate the object store has been initialized
err = r.initializeObjectStoreContext(cephObjectStoreUser)
if err != nil {
if !cephObjectStoreUser.GetDeletionTimestamp().IsZero() {
@@ -207,7 +210,7 @@ func (r *ReconcileObjectStoreUser) reconcile(request reconcile.Request) (reconci
// Generate user config
userConfig := generateUserConfig(cephObjectStoreUser)
r.userConfig = userConfig
r.userConfig = &userConfig
// DELETE: the CR was deleted
if !cephObjectStoreUser.GetDeletionTimestamp().IsZero() {
@@ -267,27 +270,25 @@ func (r *ReconcileObjectStoreUser) reconcileCephUser(cephObjectStoreUser *cephv1
func (r *ReconcileObjectStoreUser) createorUpdateCephUser(u *cephv1.CephObjectStoreUser) error {
logger.Infof("creating ceph object user %q in namespace %q", u.Name, u.Namespace)
user, rgwerr, err := object.CreateUser(r.objContext, r.userConfig)
var user admin.User
var err error
user, err = r.adminOpsAPI.CreateUser(context.TODO(), *r.userConfig)
if err != nil {
if rgwerr == object.ErrorCodeFileExists {
objectUser, _, err := object.UpdateUser(r.objContext, r.userConfig)
if errors.Is(err, admin.ErrUserExists) {
user, err = r.adminOpsAPI.GetUser(context.TODO(), *r.userConfig)
if err != nil {
return errors.Wrapf(err, "failed to get details from ceph object user %q", objectUser.UserID)
return errors.Wrapf(err, "failed to get details from ceph object user %v", &r.userConfig.ID)
}
// Set access and secret key
r.userConfig.AccessKey = objectUser.AccessKey
r.userConfig.SecretKey = objectUser.SecretKey
logger.Debugf("ceph object user %q updated with display name %q", u.Name, *objectUser.DisplayName)
return nil
}
return errors.Wrapf(err, "failed to create ceph object user %q. error code %d", u.Name, rgwerr)
return errors.Wrapf(err, "failed to create ceph object user %q", u.Name)
}
// Set access and secret key
r.userConfig.AccessKey = user.AccessKey
r.userConfig.SecretKey = user.SecretKey
r.userConfig.Keys[0].AccessKey = user.Keys[0].AccessKey
r.userConfig.Keys[0].SecretKey = user.Keys[0].SecretKey
logger.Infof("created ceph object user %q", u.Name)
return nil
@@ -317,17 +318,40 @@ func (r *ReconcileObjectStoreUser) initializeObjectStoreContext(u *cephv1.CephOb
if _, ok := store.Status.Info["endpoint"]; ok {
r.objContext.Endpoint = store.Status.Info["endpoint"]
} else {
return errors.Errorf("endpoint is missing in the status Info")
return errors.New("endpoint is missing in the status Info")
}
if _, ok := store.Status.Info["secureEndpoint"]; ok {
r.objContext.Endpoint = store.Status.Info["secureEndpoint"]
}
// Fetch admin Ops API user credentials
var accessKey, secretKey string
accessKey, secretKey, err = object.GetAdminOPSUserCredentials(r.context, r.clusterInfo, objContext, store)
if err != nil {
return errors.Wrap(err, "failed to fetch rgw admin ops api user credentials")
}
// Build TLS client if needed
var httpClient *http.Client
if store.Spec.IsTLSEnabled() {
tlsCert, err := object.GetTlsCaCert(objContext, &store.Spec)
if err != nil {
return errors.Wrapf(err, "failed to fetch CA cert to establish TLS connection with the object store %q", store.Name)
}
httpClient.Transport = object.BuildTransportTLS(tlsCert)
}
adminOpsAPI, err := admin.New(r.objContext.Endpoint, accessKey, secretKey, httpClient)
if err != nil {
return errors.Wrap(err, "failed to initialized rgw admin ops client api")
}
r.adminOpsAPI = adminOpsAPI
return nil
}
func generateUserConfig(user *cephv1.CephObjectStoreUser) object.ObjectUser {
func generateUserConfig(user *cephv1.CephObjectStoreUser) admin.User {
// Set DisplayName to match Name if DisplayName is not set
displayName := user.Spec.DisplayName
if len(displayName) == 0 {
@@ -335,9 +359,10 @@ func generateUserConfig(user *cephv1.CephObjectStoreUser) object.ObjectUser {
}
// create the user
userConfig := object.ObjectUser{
UserID: user.Name,
DisplayName: &displayName,
userConfig := admin.User{
ID: user.Name,
DisplayName: displayName,
Keys: make([]admin.UserKeySpec, 1),
}
return userConfig
@@ -356,8 +381,8 @@ func generateStatusInfo(u *cephv1.CephObjectStoreUser) map[string]string {
func (r *ReconcileObjectStoreUser) generateCephUserSecret(u *cephv1.CephObjectStoreUser) *corev1.Secret {
// Store the keys in a secret
secrets := map[string]string{
"AccessKey": *r.userConfig.AccessKey,
"SecretKey": *r.userConfig.SecretKey,
"AccessKey": r.userConfig.Keys[0].AccessKey,
"SecretKey": r.userConfig.Keys[0].SecretKey,
"Endpoint": r.objContext.Endpoint,
}
secret := &corev1.Secret{
@@ -470,9 +495,13 @@ func (r *ReconcileObjectStoreUser) getRgwPodList(cephObjectStoreUser *cephv1.Cep
// Delete the user
func (r *ReconcileObjectStoreUser) deleteUser(u *cephv1.CephObjectStoreUser) error {
output, err := object.DeleteUser(r.objContext, u.Name)
err := r.adminOpsAPI.RemoveUser(context.TODO(), admin.User{ID: u.Name})
if err != nil {
return errors.Wrapf(err, "failed to delete ceph object user %q. %v", u.Name, output)
if errors.Is(err, admin.ErrNoSuchUser) {
logger.Warningf("user %q does not exist, nothing to remove", u.Name)
return nil
}
return errors.Wrapf(err, "failed to delete ceph object user %q.", u.Name)
}
logger.Infof("ceph object user %q deleted successfully", u.Name)
@@ -280,22 +280,31 @@ func TestCephObjectStoreUserController(t *testing.T) {
// SUCCESS! The CephCluster is ready
// Rgw object exists and pods are running
//
rgwPod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{
Name: "rook-ceph-rgw-my-store-a-5fd6fb4489-xv65v",
Namespace: namespace,
Labels: map[string]string{k8sutil.AppAttr: appName, "rgw": "my-store"}}}
// TODO: add client API Mock when available in the library
//
// rgwPod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{
// Name: "rook-ceph-rgw-my-store-a-5fd6fb4489-xv65v",
// Namespace: namespace,
// Labels: map[string]string{k8sutil.AppAttr: appName, "rgw": "my-store"}}}
// Get the updated object.
logger.Info("STARTING PHASE 5")
err = r.client.Create(context.TODO(), rgwPod)
assert.NoError(t, err)
res, err = r.Reconcile(ctx, req)
assert.NoError(t, err)
assert.False(t, res.Requeue)
err = r.client.Get(context.TODO(), req.NamespacedName, objectUser)
assert.NoError(t, err)
assert.Equal(t, "Ready", objectUser.Status.Phase, objectUser)
logger.Info("PHASE 5 DONE")
// // Get the updated object.
// logger.Info("STARTING PHASE 5")
// // Create RGW pod
// err = r.client.Create(context.TODO(), rgwPod)
// assert.NoError(t, err)
// // Mock HTTP call
// r.adminOpsAPI, err = admin.New(r.objContext.Endpoint, "53S6B9S809NUP19IJ2K3", "1bXPegzsGClvoGAiJdHQD1uOW2sQBLAZM9j9VtXR", nil)
// assert.NoError(t, err)
// // Run reconcile
// res, err = r.Reconcile(ctx, req)
// assert.NoError(t, err)
// assert.False(t, res.Requeue)
// err = r.client.Get(context.TODO(), req.NamespacedName, objectUser)
// assert.NoError(t, err)
// assert.Equal(t, "Ready", objectUser.Status.Phase, objectUser)
// logger.Info("PHASE 5 DONE")
}
func TestBuildUpdateStatusInfo(t *testing.T) {
+1 -1
View File
@@ -264,7 +264,7 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile
reconcileResponse, err = r.reconcileCreatePool(clusterInfo, &cephCluster.Spec, cephBlockPool)
if err != nil {
if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
logger.Info("skipping reconcile since operator is still initializing")
logger.Info(opcontroller.OperatorNotInitializedMessage)
return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil
}
updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure, nil)