Merge pull request #7331 from thotz/healthcheckwhensslenabled
ceph: fix healthcheck in case ssl enabled for rgw
This commit is contained in:
6 files changed
+52
-30
No files matched your search
@@ -30,6 +30,10 @@ func (s *ObjectStoreSpec) IsMultisite() bool {
|
||||
return s.Zone.Name != ""
|
||||
}
|
||||
|
||||
func (s *ObjectStoreSpec) IsTLSEnabled() bool {
|
||||
return s.Gateway.SecurePort != 0 && s.Gateway.SSLCertificateRef != ""
|
||||
}
|
||||
|
||||
func (s *ObjectStoreSpec) IsExternal() bool {
|
||||
return len(s.Gateway.ExternalRgwEndpoints) != 0
|
||||
}
|
||||
|
||||
@@ -73,7 +73,7 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje
|
||||
return nil, errors.Wrap(err, "Provision: can't create ceph user")
|
||||
}
|
||||
|
||||
s3svc, err := cephObject.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), true)
|
||||
s3svc, err := cephObject.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), true, nil)
|
||||
if err != nil {
|
||||
p.deleteOBCResourceLogError("")
|
||||
return nil, err
|
||||
@@ -147,7 +147,7 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu
|
||||
return nil, errors.Wrapf(err, "could not get user (user: %s)", stats.Owner)
|
||||
}
|
||||
|
||||
s3svc, err := cephObject.NewS3Agent(*objectUser.AccessKey, *objectUser.SecretKey, p.getObjectStoreEndpoint(), true)
|
||||
s3svc, err := cephObject.NewS3Agent(*objectUser.AccessKey, *objectUser.SecretKey, p.getObjectStoreEndpoint(), true, nil)
|
||||
if err != nil {
|
||||
p.deleteOBCResourceLogError("")
|
||||
return nil, err
|
||||
@@ -242,7 +242,7 @@ func (p Provisioner) Revoke(ob *bktv1alpha1.ObjectBucket) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
s3svc, err := cephObject.NewS3Agent(*user.AccessKey, *user.SecretKey, p.getObjectStoreEndpoint(), true)
|
||||
s3svc, err := cephObject.NewS3Agent(*user.AccessKey, *user.SecretKey, p.getObjectStoreEndpoint(), true, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -339,7 +338,6 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
|
||||
objContext := NewContext(r.context, r.clusterInfo, cephObjectStore.Name)
|
||||
objContext.UID = string(cephObjectStore.UID)
|
||||
|
||||
var serviceIP string
|
||||
var err error
|
||||
|
||||
if cephObjectStore.Spec.IsExternal() {
|
||||
@@ -347,7 +345,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
|
||||
|
||||
// RECONCILE SERVICE
|
||||
logger.Info("reconciling object store service")
|
||||
serviceIP, err = cfg.reconcileService(cephObjectStore)
|
||||
_, err = cfg.reconcileService(cephObjectStore)
|
||||
if err != nil {
|
||||
return r.setFailedStatus(namespacedName, "failed to reconcile service", err)
|
||||
}
|
||||
@@ -384,7 +382,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
|
||||
|
||||
// RECONCILE SERVICE
|
||||
logger.Debug("reconciling object store service")
|
||||
serviceIP, err = cfg.reconcileService(cephObjectStore)
|
||||
serviceIP, err := cfg.reconcileService(cephObjectStore)
|
||||
if err != nil {
|
||||
return r.setFailedStatus(namespacedName, "failed to reconcile service", err)
|
||||
}
|
||||
@@ -414,7 +412,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
|
||||
|
||||
// Start monitoring
|
||||
if !cephObjectStore.Spec.HealthCheck.Bucket.Disabled {
|
||||
r.startMonitoring(cephObjectStore, objContext, serviceIP, namespacedName)
|
||||
r.startMonitoring(cephObjectStore, objContext, namespacedName)
|
||||
}
|
||||
|
||||
return reconcile.Result{}, nil
|
||||
@@ -502,7 +500,7 @@ func (r *ReconcileCephObjectStore) verifyObjectBucketCleanup(objectstore *cephv1
|
||||
return opcontroller.WaitForRequeueIfFinalizerBlocked, false
|
||||
}
|
||||
|
||||
func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjectStore, objContext *Context, serviceIP string, namespacedName types.NamespacedName) {
|
||||
func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjectStore, objContext *Context, namespacedName types.NamespacedName) {
|
||||
// Start monitoring object store
|
||||
if r.objectStoreChannels[objectstore.Name].monitoringRunning {
|
||||
logger.Debug("external rgw endpoint monitoring go routine already running!")
|
||||
@@ -512,18 +510,19 @@ func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjec
|
||||
// Set the monitoring flag so we don't start more than one go routine
|
||||
r.objectStoreChannels[objectstore.Name].monitoringRunning = true
|
||||
|
||||
var port string
|
||||
var port int32
|
||||
|
||||
if objectstore.Spec.Gateway.SecurePort != 0 && objectstore.Spec.Gateway.SSLCertificateRef != "" {
|
||||
port = strconv.Itoa(int(objectstore.Spec.Gateway.SecurePort))
|
||||
if objectstore.Spec.IsTLSEnabled() {
|
||||
port = objectstore.Spec.Gateway.SecurePort
|
||||
} else if objectstore.Spec.Gateway.Port != 0 {
|
||||
port = strconv.Itoa(int(objectstore.Spec.Gateway.Port))
|
||||
port = objectstore.Spec.Gateway.Port
|
||||
|
||||
} else {
|
||||
logger.Error("At least one of Port or SecurePort should be non-zero")
|
||||
return
|
||||
}
|
||||
|
||||
rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec.HealthCheck)
|
||||
rgwChecker := newBucketChecker(r.context, objContext, port, r.client, namespacedName, &objectstore.Spec)
|
||||
logger.Info("starting rgw healthcheck")
|
||||
go rgwChecker.checkObjectStore(r.objectStoreChannels[objectstore.Name].stopChan)
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ limitations under the License.
|
||||
package object
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
@@ -24,6 +25,7 @@ import (
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
)
|
||||
@@ -45,28 +47,26 @@ type bucketChecker struct {
|
||||
context *clusterd.Context
|
||||
objContext *Context
|
||||
interval *time.Duration
|
||||
serviceIP string
|
||||
port string
|
||||
port int32
|
||||
client client.Client
|
||||
namespacedName types.NamespacedName
|
||||
healthCheckSpec *cephv1.BucketHealthCheckSpec
|
||||
objectStoreSpec *cephv1.ObjectStoreSpec
|
||||
}
|
||||
|
||||
// newbucketChecker creates a new HealthChecker object
|
||||
func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP, port string, client client.Client, namespacedName types.NamespacedName, healthCheckSpec *cephv1.BucketHealthCheckSpec) *bucketChecker {
|
||||
func newBucketChecker(context *clusterd.Context, objContext *Context, port int32, client client.Client, namespacedName types.NamespacedName, objectStoreSpec *cephv1.ObjectStoreSpec) *bucketChecker {
|
||||
c := &bucketChecker{
|
||||
context: context,
|
||||
objContext: objContext,
|
||||
interval: &defaultHealthCheckInterval,
|
||||
serviceIP: serviceIP,
|
||||
port: port,
|
||||
namespacedName: namespacedName,
|
||||
client: client,
|
||||
healthCheckSpec: healthCheckSpec,
|
||||
objectStoreSpec: objectStoreSpec,
|
||||
}
|
||||
|
||||
// allow overriding the check interval
|
||||
checkInterval := healthCheckSpec.Bucket.Interval
|
||||
checkInterval := objectStoreSpec.HealthCheck.Bucket.Interval
|
||||
if checkInterval != nil {
|
||||
logger.Infof("ceph rgw status check interval for object store %q is %q", namespacedName.Name, checkInterval.Duration.String())
|
||||
c.interval = &checkInterval.Duration
|
||||
@@ -121,10 +121,11 @@ func (c *bucketChecker) checkObjectStoreHealth() error {
|
||||
|
||||
Always keep the bucket and the user for the health check, just do PUT and GET because bucket creation is expensive
|
||||
*/
|
||||
|
||||
ctx := context.TODO()
|
||||
var s3AccessKey string
|
||||
var s3SecretKey string
|
||||
s3endpoint := fmt.Sprintf("%s:%s", BuildDomainName(c.objContext.Name, c.namespacedName.Namespace), c.port)
|
||||
var sslCert []byte
|
||||
s3endpoint := buildDNSEndpoint(BuildDomainName(c.objContext.Name, c.namespacedName.Namespace), c.port, c.objectStoreSpec.IsTLSEnabled())
|
||||
|
||||
// Generate unique user and bucket name
|
||||
bucketName := genUniqueBucketName(c.objContext.UID)
|
||||
@@ -147,9 +148,13 @@ func (c *bucketChecker) checkObjectStoreHealth() error {
|
||||
s3AccessKey = *user.AccessKey
|
||||
s3SecretKey = *user.SecretKey
|
||||
|
||||
if c.objectStoreSpec.IsTLSEnabled() {
|
||||
sslSecretCert, _ := c.context.Clientset.CoreV1().Secrets(c.namespacedName.Namespace).Get(ctx, c.objectStoreSpec.Gateway.SSLCertificateRef, metav1.GetOptions{})
|
||||
sslCert = sslSecretCert.Data[certKeyName]
|
||||
}
|
||||
// Initiate s3 agent
|
||||
logger.Debugf("initializing s3 connection for object store %q", c.namespacedName.Name)
|
||||
s3client, err := NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, false)
|
||||
s3client, err := NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, false, sslCert)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to initialize s3 connection")
|
||||
}
|
||||
|
||||
@@ -18,6 +18,8 @@ package object
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -35,13 +37,27 @@ type S3Agent struct {
|
||||
Client *s3.S3
|
||||
}
|
||||
|
||||
func NewS3Agent(accessKey, secretKey, endpoint string, debug bool) (*S3Agent, error) {
|
||||
func NewS3Agent(accessKey, secretKey, endpoint string, debug bool, tlsCert []byte) (*S3Agent, error) {
|
||||
const cephRegion = "us-east-1"
|
||||
|
||||
logLevel := aws.LogOff
|
||||
if debug {
|
||||
logLevel = aws.LogDebug
|
||||
}
|
||||
client := http.Client{
|
||||
Timeout: time.Second * 15,
|
||||
}
|
||||
tlsEnabled := false
|
||||
if len(tlsCert) > 0 {
|
||||
caCertPool := x509.NewCertPool()
|
||||
caCertPool.AppendCertsFromPEM(tlsCert)
|
||||
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12},
|
||||
}
|
||||
client.Transport = tr
|
||||
tlsEnabled = true
|
||||
}
|
||||
sess, err := session.NewSession(
|
||||
aws.NewConfig().
|
||||
WithRegion(cephRegion).
|
||||
@@ -49,10 +65,8 @@ func NewS3Agent(accessKey, secretKey, endpoint string, debug bool) (*S3Agent, er
|
||||
WithEndpoint(endpoint).
|
||||
WithS3ForcePathStyle(true).
|
||||
WithMaxRetries(5).
|
||||
WithDisableSSL(true).
|
||||
WithHTTPClient(&http.Client{
|
||||
Timeout: time.Second * 15,
|
||||
}).
|
||||
WithDisableSSL(!tlsEnabled).
|
||||
WithHTTPClient(&client).
|
||||
WithLogLevel(logLevel),
|
||||
)
|
||||
if err != nil {
|
||||
|
||||
@@ -147,7 +147,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
|
||||
s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
|
||||
s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
|
||||
s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
|
||||
s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true)
|
||||
s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil)
|
||||
assert.Nil(s.T(), err)
|
||||
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
|
||||
|
||||
|
||||
Reference in new issue
Block a user