ceph: fix healthcheck incase ssl enabled for rgw

In case ssl enabled for RGW, bucket healthcheck won't work since access is denied from the server.
In that case configure S3 agent using "InsecureSkipVerify: true" option.

Fixes: 7288
Signed-off-by: Jiffin Tony Thottan <thottanjiffin@gmail.com>
This commit is contained in:
Jiffin Tony Thottan committed 2021-04-20 14:23:48 +05:30
1 parent 76902a9775
commit bcbe707bcb
6 files changed
+48 -23

No files matched your search

+4
View File
@@ -30,6 +30,10 @@ func (s *ObjectStoreSpec) IsMultisite() bool {
return s.Zone.Name != ""
}
func (s *ObjectStoreSpec) IsTLSEnabled() bool {
return s.Gateway.SecurePort != 0 && s.Gateway.SSLCertificateRef != ""
}
func (s *ObjectStoreSpec) IsExternal() bool {
return len(s.Gateway.ExternalRgwEndpoints) != 0
}
@@ -73,7 +73,7 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje
return nil, errors.Wrap(err, "Provision: can't create ceph user")
}
s3svc, err := cephObject.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), true)
s3svc, err := cephObject.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), true, nil)
if err != nil {
p.deleteOBCResourceLogError("")
return nil, err
@@ -147,7 +147,7 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu
return nil, errors.Wrapf(err, "could not get user (user: %s)", stats.Owner)
}
s3svc, err := cephObject.NewS3Agent(*objectUser.AccessKey, *objectUser.SecretKey, p.getObjectStoreEndpoint(), true)
s3svc, err := cephObject.NewS3Agent(*objectUser.AccessKey, *objectUser.SecretKey, p.getObjectStoreEndpoint(), true, nil)
if err != nil {
p.deleteOBCResourceLogError("")
return nil, err
@@ -242,7 +242,7 @@ func (p Provisioner) Revoke(ob *bktv1alpha1.ObjectBucket) error {
return nil
}
s3svc, err := cephObject.NewS3Agent(*user.AccessKey, *user.SecretKey, p.getObjectStoreEndpoint(), true)
s3svc, err := cephObject.NewS3Agent(*user.AccessKey, *user.SecretKey, p.getObjectStoreEndpoint(), true, nil)
if err != nil {
return err
}
+6 -6
View File
@@ -20,7 +20,6 @@ import (
"context"
"fmt"
"reflect"
"strconv"
"strings"
"syscall"
"time"
@@ -512,18 +511,19 @@ func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjec
// Set the monitoring flag so we don't start more than one go routine
r.objectStoreChannels[objectstore.Name].monitoringRunning = true
var port string
var port int32
if objectstore.Spec.Gateway.SecurePort != 0 && objectstore.Spec.Gateway.SSLCertificateRef != "" {
port = strconv.Itoa(int(objectstore.Spec.Gateway.SecurePort))
if objectstore.Spec.IsTLSEnabled() {
port = objectstore.Spec.Gateway.SecurePort
} else if objectstore.Spec.Gateway.Port != 0 {
port = strconv.Itoa(int(objectstore.Spec.Gateway.Port))
port = objectstore.Spec.Gateway.Port
} else {
logger.Error("At least one of Port or SecurePort should be non-zero")
return
}
rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec.HealthCheck)
rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec)
logger.Info("starting rgw healthcheck")
go rgwChecker.checkObjectStore(r.objectStoreChannels[objectstore.Name].stopChan)
}
+15 -8
View File
@@ -17,6 +17,7 @@ limitations under the License.
package object
import (
"context"
"fmt"
"time"
@@ -24,6 +25,7 @@ import (
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -46,14 +48,14 @@ type bucketChecker struct {
objContext *Context
interval *time.Duration
serviceIP string
port string
port int32
client client.Client
namespacedName types.NamespacedName
healthCheckSpec *cephv1.BucketHealthCheckSpec
objectStoreSpec *cephv1.ObjectStoreSpec
}
// newbucketChecker creates a new HealthChecker object
func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP, port string, client client.Client, namespacedName types.NamespacedName, healthCheckSpec *cephv1.BucketHealthCheckSpec) *bucketChecker {
func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP string, port int32, client client.Client, namespacedName types.NamespacedName, objectStoreSpec *cephv1.ObjectStoreSpec) *bucketChecker {
c := &bucketChecker{
context: context,
objContext: objContext,
@@ -62,11 +64,11 @@ func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP,
port: port,
namespacedName: namespacedName,
client: client,
healthCheckSpec: healthCheckSpec,
objectStoreSpec: objectStoreSpec,
}
// allow overriding the check interval
checkInterval := healthCheckSpec.Bucket.Interval
checkInterval := objectStoreSpec.HealthCheck.Bucket.Interval
if checkInterval != nil {
logger.Infof("ceph rgw status check interval for object store %q is %q", namespacedName.Name, checkInterval.Duration.String())
c.interval = &checkInterval.Duration
@@ -121,10 +123,11 @@ func (c *bucketChecker) checkObjectStoreHealth() error {
Always keep the bucket and the user for the health check, just do PUT and GET because bucket creation is expensive
*/
ctx := context.TODO()
var s3AccessKey string
var s3SecretKey string
s3endpoint := fmt.Sprintf("%s:%s", BuildDomainName(c.objContext.Name, c.namespacedName.Namespace), c.port)
var sslCert []byte
s3endpoint := buildDNSEndpoint(BuildDomainName(c.objContext.Name, c.namespacedName.Namespace), c.port, c.objectStoreSpec.IsTLSEnabled())
// Generate unique user and bucket name
bucketName := genUniqueBucketName(c.objContext.UID)
@@ -147,9 +150,13 @@ func (c *bucketChecker) checkObjectStoreHealth() error {
s3AccessKey = *user.AccessKey
s3SecretKey = *user.SecretKey
if c.objectStoreSpec.IsTLSEnabled() {
sslSecretCert, _ := c.context.Clientset.CoreV1().Secrets(c.namespacedName.Namespace).Get(ctx, c.objectStoreSpec.Gateway.SSLCertificateRef, metav1.GetOptions{})
sslCert = sslSecretCert.Data[certKeyName]
}
// Initiate s3 agent
logger.Debugf("initializing s3 connection for object store %q", c.namespacedName.Name)
s3client, err := NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, false)
s3client, err := NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, false, sslCert)
if err != nil {
return errors.Wrap(err, "failed to initialize s3 connection")
}
+19 -5
View File
@@ -18,6 +18,8 @@ package object
import (
"bytes"
"crypto/tls"
"crypto/x509"
"net/http"
"strings"
"time"
@@ -35,13 +37,27 @@ type S3Agent struct {
Client *s3.S3
}
func NewS3Agent(accessKey, secretKey, endpoint string, debug bool) (*S3Agent, error) {
func NewS3Agent(accessKey, secretKey, endpoint string, debug bool, tlsCert []byte) (*S3Agent, error) {
const cephRegion = "us-east-1"
logLevel := aws.LogOff
if debug {
logLevel = aws.LogDebug
}
client := http.Client{
Timeout: time.Second * 15,
}
tlsEnabled := false
if len(tlsCert) > 0 {
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(tlsCert)
tr := &http.Transport{
TLSClientConfig: &tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12},
}
client.Transport = tr
tlsEnabled = true
}
sess, err := session.NewSession(
aws.NewConfig().
WithRegion(cephRegion).
@@ -49,10 +65,8 @@ func NewS3Agent(accessKey, secretKey, endpoint string, debug bool) (*S3Agent, er
WithEndpoint(endpoint).
WithS3ForcePathStyle(true).
WithMaxRetries(5).
WithDisableSSL(true).
WithHTTPClient(&http.Client{
Timeout: time.Second * 15,
}).
WithDisableSSL(!tlsEnabled).
WithHTTPClient(&client).
WithLogLevel(logLevel),
)
if err != nil {
+1 -1
View File
@@ -147,7 +147,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true)
s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil)
assert.Nil(s.T(), err)
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)