From bcbe707bcba2d663ac2a1727779864af5c7fd02c Mon Sep 17 00:00:00 2001 From: Jiffin Tony Thottan Date: Mon, 1 Mar 2021 23:18:55 +0530 Subject: [PATCH] ceph: fix healthcheck incase ssl enabled for rgw In case ssl enabled for RGW, bucket healthcheck won't work since access is denied from the server. In that case configure S3 agent using "InsecureSkipVerify: true" option. Fixes: 7288 Signed-off-by: Jiffin Tony Thottan --- pkg/apis/ceph.rook.io/v1/object.go | 4 ++++ .../ceph/object/bucket/provisioner.go | 6 ++--- pkg/operator/ceph/object/controller.go | 12 +++++----- pkg/operator/ceph/object/health.go | 23 +++++++++++------- pkg/operator/ceph/object/s3-handlers.go | 24 +++++++++++++++---- tests/integration/ceph_base_object_test.go | 2 +- 6 files changed, 48 insertions(+), 23 deletions(-) diff --git a/pkg/apis/ceph.rook.io/v1/object.go b/pkg/apis/ceph.rook.io/v1/object.go index ca5afdccb..31b470ebb 100644 --- a/pkg/apis/ceph.rook.io/v1/object.go +++ b/pkg/apis/ceph.rook.io/v1/object.go @@ -30,6 +30,10 @@ func (s *ObjectStoreSpec) IsMultisite() bool { return s.Zone.Name != "" } +func (s *ObjectStoreSpec) IsTLSEnabled() bool { + return s.Gateway.SecurePort != 0 && s.Gateway.SSLCertificateRef != "" +} + func (s *ObjectStoreSpec) IsExternal() bool { return len(s.Gateway.ExternalRgwEndpoints) != 0 } diff --git a/pkg/operator/ceph/object/bucket/provisioner.go b/pkg/operator/ceph/object/bucket/provisioner.go index f25eea2af..e87dc6fed 100644 --- a/pkg/operator/ceph/object/bucket/provisioner.go +++ b/pkg/operator/ceph/object/bucket/provisioner.go @@ -73,7 +73,7 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje return nil, errors.Wrap(err, "Provision: can't create ceph user") } - s3svc, err := cephObject.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), true) + s3svc, err := cephObject.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), true, nil) if err != nil { p.deleteOBCResourceLogError("") return nil, err @@ -147,7 +147,7 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu return nil, errors.Wrapf(err, "could not get user (user: %s)", stats.Owner) } - s3svc, err := cephObject.NewS3Agent(*objectUser.AccessKey, *objectUser.SecretKey, p.getObjectStoreEndpoint(), true) + s3svc, err := cephObject.NewS3Agent(*objectUser.AccessKey, *objectUser.SecretKey, p.getObjectStoreEndpoint(), true, nil) if err != nil { p.deleteOBCResourceLogError("") return nil, err @@ -242,7 +242,7 @@ func (p Provisioner) Revoke(ob *bktv1alpha1.ObjectBucket) error { return nil } - s3svc, err := cephObject.NewS3Agent(*user.AccessKey, *user.SecretKey, p.getObjectStoreEndpoint(), true) + s3svc, err := cephObject.NewS3Agent(*user.AccessKey, *user.SecretKey, p.getObjectStoreEndpoint(), true, nil) if err != nil { return err } diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index 86b627493..ae5995175 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -20,7 +20,6 @@ import ( "context" "fmt" "reflect" - "strconv" "strings" "syscall" "time" @@ -512,18 +511,19 @@ func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjec // Set the monitoring flag so we don't start more than one go routine r.objectStoreChannels[objectstore.Name].monitoringRunning = true - var port string + var port int32 - if objectstore.Spec.Gateway.SecurePort != 0 && objectstore.Spec.Gateway.SSLCertificateRef != "" { - port = strconv.Itoa(int(objectstore.Spec.Gateway.SecurePort)) + if objectstore.Spec.IsTLSEnabled() { + port = objectstore.Spec.Gateway.SecurePort } else if objectstore.Spec.Gateway.Port != 0 { - port = strconv.Itoa(int(objectstore.Spec.Gateway.Port)) + port = objectstore.Spec.Gateway.Port + } else { logger.Error("At least one of Port or SecurePort should be non-zero") return } - rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec.HealthCheck) + rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec) logger.Info("starting rgw healthcheck") go rgwChecker.checkObjectStore(r.objectStoreChannels[objectstore.Name].stopChan) } diff --git a/pkg/operator/ceph/object/health.go b/pkg/operator/ceph/object/health.go index af16928d1..8cd985e8b 100644 --- a/pkg/operator/ceph/object/health.go +++ b/pkg/operator/ceph/object/health.go @@ -17,6 +17,7 @@ limitations under the License. package object import ( + "context" "fmt" "time" @@ -24,6 +25,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -46,14 +48,14 @@ type bucketChecker struct { objContext *Context interval *time.Duration serviceIP string - port string + port int32 client client.Client namespacedName types.NamespacedName - healthCheckSpec *cephv1.BucketHealthCheckSpec + objectStoreSpec *cephv1.ObjectStoreSpec } // newbucketChecker creates a new HealthChecker object -func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP, port string, client client.Client, namespacedName types.NamespacedName, healthCheckSpec *cephv1.BucketHealthCheckSpec) *bucketChecker { +func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP string, port int32, client client.Client, namespacedName types.NamespacedName, objectStoreSpec *cephv1.ObjectStoreSpec) *bucketChecker { c := &bucketChecker{ context: context, objContext: objContext, @@ -62,11 +64,11 @@ func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP, port: port, namespacedName: namespacedName, client: client, - healthCheckSpec: healthCheckSpec, + objectStoreSpec: objectStoreSpec, } // allow overriding the check interval - checkInterval := healthCheckSpec.Bucket.Interval + checkInterval := objectStoreSpec.HealthCheck.Bucket.Interval if checkInterval != nil { logger.Infof("ceph rgw status check interval for object store %q is %q", namespacedName.Name, checkInterval.Duration.String()) c.interval = &checkInterval.Duration @@ -121,10 +123,11 @@ func (c *bucketChecker) checkObjectStoreHealth() error { Always keep the bucket and the user for the health check, just do PUT and GET because bucket creation is expensive */ - + ctx := context.TODO() var s3AccessKey string var s3SecretKey string - s3endpoint := fmt.Sprintf("%s:%s", BuildDomainName(c.objContext.Name, c.namespacedName.Namespace), c.port) + var sslCert []byte + s3endpoint := buildDNSEndpoint(BuildDomainName(c.objContext.Name, c.namespacedName.Namespace), c.port, c.objectStoreSpec.IsTLSEnabled()) // Generate unique user and bucket name bucketName := genUniqueBucketName(c.objContext.UID) @@ -147,9 +150,13 @@ func (c *bucketChecker) checkObjectStoreHealth() error { s3AccessKey = *user.AccessKey s3SecretKey = *user.SecretKey + if c.objectStoreSpec.IsTLSEnabled() { + sslSecretCert, _ := c.context.Clientset.CoreV1().Secrets(c.namespacedName.Namespace).Get(ctx, c.objectStoreSpec.Gateway.SSLCertificateRef, metav1.GetOptions{}) + sslCert = sslSecretCert.Data[certKeyName] + } // Initiate s3 agent logger.Debugf("initializing s3 connection for object store %q", c.namespacedName.Name) - s3client, err := NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, false) + s3client, err := NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, false, sslCert) if err != nil { return errors.Wrap(err, "failed to initialize s3 connection") } diff --git a/pkg/operator/ceph/object/s3-handlers.go b/pkg/operator/ceph/object/s3-handlers.go index 0452843c5..e042d8acb 100644 --- a/pkg/operator/ceph/object/s3-handlers.go +++ b/pkg/operator/ceph/object/s3-handlers.go @@ -18,6 +18,8 @@ package object import ( "bytes" + "crypto/tls" + "crypto/x509" "net/http" "strings" "time" @@ -35,13 +37,27 @@ type S3Agent struct { Client *s3.S3 } -func NewS3Agent(accessKey, secretKey, endpoint string, debug bool) (*S3Agent, error) { +func NewS3Agent(accessKey, secretKey, endpoint string, debug bool, tlsCert []byte) (*S3Agent, error) { const cephRegion = "us-east-1" logLevel := aws.LogOff if debug { logLevel = aws.LogDebug } + client := http.Client{ + Timeout: time.Second * 15, + } + tlsEnabled := false + if len(tlsCert) > 0 { + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(tlsCert) + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12}, + } + client.Transport = tr + tlsEnabled = true + } sess, err := session.NewSession( aws.NewConfig(). WithRegion(cephRegion). @@ -49,10 +65,8 @@ func NewS3Agent(accessKey, secretKey, endpoint string, debug bool) (*S3Agent, er WithEndpoint(endpoint). WithS3ForcePathStyle(true). WithMaxRetries(5). - WithDisableSSL(true). - WithHTTPClient(&http.Client{ - Timeout: time.Second * 15, - }). + WithDisableSSL(!tlsEnabled). + WithHTTPClient(&client). WithLogLevel(logLevel), ) if err != nil { diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go index e4bb4d09d..f95ffe6b0 100644 --- a/tests/integration/ceph_base_object_test.go +++ b/tests/integration/ceph_base_object_test.go @@ -147,7 +147,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName) s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName) s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName) - s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true) + s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil) assert.Nil(s.T(), err) logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)