diff --git a/pkg/apis/ceph.rook.io/v1/object.go b/pkg/apis/ceph.rook.io/v1/object.go index ca5afdccb..31b470ebb 100644 --- a/pkg/apis/ceph.rook.io/v1/object.go +++ b/pkg/apis/ceph.rook.io/v1/object.go @@ -30,6 +30,10 @@ func (s *ObjectStoreSpec) IsMultisite() bool { return s.Zone.Name != "" } +func (s *ObjectStoreSpec) IsTLSEnabled() bool { + return s.Gateway.SecurePort != 0 && s.Gateway.SSLCertificateRef != "" +} + func (s *ObjectStoreSpec) IsExternal() bool { return len(s.Gateway.ExternalRgwEndpoints) != 0 } diff --git a/pkg/operator/ceph/object/bucket/provisioner.go b/pkg/operator/ceph/object/bucket/provisioner.go index f25eea2af..e87dc6fed 100644 --- a/pkg/operator/ceph/object/bucket/provisioner.go +++ b/pkg/operator/ceph/object/bucket/provisioner.go @@ -73,7 +73,7 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje return nil, errors.Wrap(err, "Provision: can't create ceph user") } - s3svc, err := cephObject.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), true) + s3svc, err := cephObject.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), true, nil) if err != nil { p.deleteOBCResourceLogError("") return nil, err @@ -147,7 +147,7 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu return nil, errors.Wrapf(err, "could not get user (user: %s)", stats.Owner) } - s3svc, err := cephObject.NewS3Agent(*objectUser.AccessKey, *objectUser.SecretKey, p.getObjectStoreEndpoint(), true) + s3svc, err := cephObject.NewS3Agent(*objectUser.AccessKey, *objectUser.SecretKey, p.getObjectStoreEndpoint(), true, nil) if err != nil { p.deleteOBCResourceLogError("") return nil, err @@ -242,7 +242,7 @@ func (p Provisioner) Revoke(ob *bktv1alpha1.ObjectBucket) error { return nil } - s3svc, err := cephObject.NewS3Agent(*user.AccessKey, *user.SecretKey, p.getObjectStoreEndpoint(), true) + s3svc, err := cephObject.NewS3Agent(*user.AccessKey, *user.SecretKey, p.getObjectStoreEndpoint(), true, nil) if err != nil { return err } diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index 86b627493..ae5995175 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -20,7 +20,6 @@ import ( "context" "fmt" "reflect" - "strconv" "strings" "syscall" "time" @@ -512,18 +511,19 @@ func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjec // Set the monitoring flag so we don't start more than one go routine r.objectStoreChannels[objectstore.Name].monitoringRunning = true - var port string + var port int32 - if objectstore.Spec.Gateway.SecurePort != 0 && objectstore.Spec.Gateway.SSLCertificateRef != "" { - port = strconv.Itoa(int(objectstore.Spec.Gateway.SecurePort)) + if objectstore.Spec.IsTLSEnabled() { + port = objectstore.Spec.Gateway.SecurePort } else if objectstore.Spec.Gateway.Port != 0 { - port = strconv.Itoa(int(objectstore.Spec.Gateway.Port)) + port = objectstore.Spec.Gateway.Port + } else { logger.Error("At least one of Port or SecurePort should be non-zero") return } - rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec.HealthCheck) + rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec) logger.Info("starting rgw healthcheck") go rgwChecker.checkObjectStore(r.objectStoreChannels[objectstore.Name].stopChan) } diff --git a/pkg/operator/ceph/object/health.go b/pkg/operator/ceph/object/health.go index af16928d1..8cd985e8b 100644 --- a/pkg/operator/ceph/object/health.go +++ b/pkg/operator/ceph/object/health.go @@ -17,6 +17,7 @@ limitations under the License. package object import ( + "context" "fmt" "time" @@ -24,6 +25,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -46,14 +48,14 @@ type bucketChecker struct { objContext *Context interval *time.Duration serviceIP string - port string + port int32 client client.Client namespacedName types.NamespacedName - healthCheckSpec *cephv1.BucketHealthCheckSpec + objectStoreSpec *cephv1.ObjectStoreSpec } // newbucketChecker creates a new HealthChecker object -func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP, port string, client client.Client, namespacedName types.NamespacedName, healthCheckSpec *cephv1.BucketHealthCheckSpec) *bucketChecker { +func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP string, port int32, client client.Client, namespacedName types.NamespacedName, objectStoreSpec *cephv1.ObjectStoreSpec) *bucketChecker { c := &bucketChecker{ context: context, objContext: objContext, @@ -62,11 +64,11 @@ func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP, port: port, namespacedName: namespacedName, client: client, - healthCheckSpec: healthCheckSpec, + objectStoreSpec: objectStoreSpec, } // allow overriding the check interval - checkInterval := healthCheckSpec.Bucket.Interval + checkInterval := objectStoreSpec.HealthCheck.Bucket.Interval if checkInterval != nil { logger.Infof("ceph rgw status check interval for object store %q is %q", namespacedName.Name, checkInterval.Duration.String()) c.interval = &checkInterval.Duration @@ -121,10 +123,11 @@ func (c *bucketChecker) checkObjectStoreHealth() error { Always keep the bucket and the user for the health check, just do PUT and GET because bucket creation is expensive */ - + ctx := context.TODO() var s3AccessKey string var s3SecretKey string - s3endpoint := fmt.Sprintf("%s:%s", BuildDomainName(c.objContext.Name, c.namespacedName.Namespace), c.port) + var sslCert []byte + s3endpoint := buildDNSEndpoint(BuildDomainName(c.objContext.Name, c.namespacedName.Namespace), c.port, c.objectStoreSpec.IsTLSEnabled()) // Generate unique user and bucket name bucketName := genUniqueBucketName(c.objContext.UID) @@ -147,9 +150,13 @@ func (c *bucketChecker) checkObjectStoreHealth() error { s3AccessKey = *user.AccessKey s3SecretKey = *user.SecretKey + if c.objectStoreSpec.IsTLSEnabled() { + sslSecretCert, _ := c.context.Clientset.CoreV1().Secrets(c.namespacedName.Namespace).Get(ctx, c.objectStoreSpec.Gateway.SSLCertificateRef, metav1.GetOptions{}) + sslCert = sslSecretCert.Data[certKeyName] + } // Initiate s3 agent logger.Debugf("initializing s3 connection for object store %q", c.namespacedName.Name) - s3client, err := NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, false) + s3client, err := NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, false, sslCert) if err != nil { return errors.Wrap(err, "failed to initialize s3 connection") } diff --git a/pkg/operator/ceph/object/s3-handlers.go b/pkg/operator/ceph/object/s3-handlers.go index 0452843c5..e042d8acb 100644 --- a/pkg/operator/ceph/object/s3-handlers.go +++ b/pkg/operator/ceph/object/s3-handlers.go @@ -18,6 +18,8 @@ package object import ( "bytes" + "crypto/tls" + "crypto/x509" "net/http" "strings" "time" @@ -35,13 +37,27 @@ type S3Agent struct { Client *s3.S3 } -func NewS3Agent(accessKey, secretKey, endpoint string, debug bool) (*S3Agent, error) { +func NewS3Agent(accessKey, secretKey, endpoint string, debug bool, tlsCert []byte) (*S3Agent, error) { const cephRegion = "us-east-1" logLevel := aws.LogOff if debug { logLevel = aws.LogDebug } + client := http.Client{ + Timeout: time.Second * 15, + } + tlsEnabled := false + if len(tlsCert) > 0 { + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(tlsCert) + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12}, + } + client.Transport = tr + tlsEnabled = true + } sess, err := session.NewSession( aws.NewConfig(). WithRegion(cephRegion). @@ -49,10 +65,8 @@ func NewS3Agent(accessKey, secretKey, endpoint string, debug bool) (*S3Agent, er WithEndpoint(endpoint). WithS3ForcePathStyle(true). WithMaxRetries(5). - WithDisableSSL(true). - WithHTTPClient(&http.Client{ - Timeout: time.Second * 15, - }). + WithDisableSSL(!tlsEnabled). + WithHTTPClient(&client). WithLogLevel(logLevel), ) if err != nil { diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go index e4bb4d09d..f95ffe6b0 100644 --- a/tests/integration/ceph_base_object_test.go +++ b/tests/integration/ceph_base_object_test.go @@ -147,7 +147,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName) s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName) s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName) - s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true) + s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil) assert.Nil(s.T(), err) logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)